diff --git a/.github/workflows/vale.yml b/.github/workflows-backup/old-vale.yml.bak similarity index 56% rename from .github/workflows/vale.yml rename to .github/workflows-backup/old-vale.yml.bak index bad1eb18c..ec3c4da5d 100644 --- a/.github/workflows/vale.yml +++ b/.github/workflows-backup/old-vale.yml.bak @@ -15,7 +15,18 @@ jobs: - name: Install Asciidoctor run: sudo apt-get install -y asciidoctor - - name: Run Vale + - name: Vale Style Check + uses: errata-ai/vale-action@reviewdog + env: + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + with: + fail_on_error: false + reporter: github-pr-check + filter_mode: added + files: latest/ug + continue-on-error: true + + - name: Enforce AWS Brand uses: errata-ai/vale-action@reviewdog env: GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} @@ -24,4 +35,5 @@ jobs: reporter: github-pr-check filter_mode: added files: latest/ug - continue-on-error: false \ No newline at end of file + vale_flags: "--config brand.vale.ini" + continue-on-error: true \ No newline at end of file diff --git a/.gitignore b/.gitignore index 7bc098742..a4d20af83 100755 --- a/.gitignore +++ b/.gitignore @@ -76,5 +76,7 @@ build *.xls *.xlsx *.xpr +.idea/* vale/styles/AsciiDoc/ vale/styles/RedHat/ +.amazonq/ diff --git a/.vale.ini b/.vale.ini index 5f93659a1..858da4d52 100644 --- a/.vale.ini +++ b/.vale.ini @@ -12,5 +12,6 @@ BasedOnStyles = RedHat, AsciiDoc, EksDocs RedHat.GitLinks = OFF AsciiDoc.UnsetAttributes = OFF RedHat.CaseSensitiveTerms = suggestion +RedHat.Contractions = OFF RedHat.TermsErrors = warning RedHat.Spacing = warning diff --git a/.vscode/asciidoc.code-snippets b/.vscode/asciidoc.code-snippets index cdceb00ea..bb24f9976 100644 --- a/.vscode/asciidoc.code-snippets +++ b/.vscode/asciidoc.code-snippets @@ -68,7 +68,7 @@ "----\n", "====" ], - "description": "adoc step tablist" + "description": "adoc step with tablist of console, eksctl, awscli, CFN" }, "adoc region tablist": { "prefix": "tab-region", @@ -152,16 +152,14 @@ "NODE_ROOT Section": { "prefix": "adoc-topic", "body": [ - "//!!NODE_ROOT
", "[.topic]", "[[${1:page-id},${1:page-id}.title]]", "= ${2:page title goes here}", - ":info_doctype: section", "", "include::../attributes.txt[]", "", ], - "description": "Creates a NODE_ROOT section template with topic class and ID" + "description": "Creates a section template with topic class and ID" }, "Include with Leveloffset": { "prefix": "adoc-inc", @@ -181,5 +179,15 @@ "====" ], "description": "Creates a collapsible section in AsciiDoc" + }, + "AsciiDoc Platform Version Table": { + "prefix": "adoc-pv", + "body": [ + "| `${1:K8s version}`", + "| `eks.${2:platform version}`", + "| New platform version with security fixes and enhancements.", + "| ${3:Date available in all regions}" + ], + "description": "Creates entry in platform versions table" } } \ No newline at end of file diff --git a/CODEOWNERS b/CODEOWNERS index f0fc64426..09cfc7a16 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,32 +1,5 @@ # default rule, unless a more specific rule applies -* @eks-admins - -# lower approval for specific non-content folders -/.vscode/ fincd@amazon.com +* @awsdocs/eks-admins # default rule for user guide -/latest/ug/ @eks-contributors - -# guide section assignments -/latest/ug/clusters/ gcline@amazon.com -/latest/ug/manage-access/ gcline@amazon.com -/latest/ug/workloads/ gcline@amazon.com -/latest/ug/networking/ fincd@amazon.com -/latest/ug/storage/ pgasca@amazon.com -/latest/ug/nodes/ pgasca@amazon.com -/latest/ug/connector/ fincd@amazon.com -/latest/ug/contribute/ gcline@amazon.com -/latest/ug/getting-started/ mcngs@amazon.com -/latest/ug/integrations/ gcline@amazon.com -/latest/ug/observability/ pgasca@amazon.com -/latest/ug/outposts/ fincd@amazon.com -/latest/ug/security/ fincd@amazon.com -/latest/ug/what-is/ mcngs@amazon.com - -# use default rule -# /latest/ug/troubleshooting/ -# /latest/ug/images/ -# /latest/ug/images_BJS/ -# /latest/ug/YAML -# /latest/ug/iam_policies/ -# /latest/ug/diagrams/ \ No newline at end of file +/latest/ug/ @awsdocs/eks-contributors diff --git a/Config b/Config index 6a1d1ce19..c9664a2d8 100755 --- a/Config +++ b/Config @@ -12,7 +12,7 @@ package.AmazonEKSDocs = { }; scope = webservices; - build-system = happytrails; + build-system = zonbooktrails; build-environment = { chroot = basic; network-access = blocked; @@ -20,11 +20,11 @@ package.AmazonEKSDocs = { build-tools = { 3.0 = { - HappyTrails = 3.2; + ZonBookTrails = 1.0; AWSEC2ContainerChecklist = 1.0; AWSDevDocsQuotasShare = 1.0; - JavaBuildAndTestMin = jdk8; - ZonBook = 4.0; + AWSDocsSdkExamplesPublic = 1.0; + ZonBook = 5.0; AWSDevDocsChecklistBJS = 2.0; }; }; diff --git a/README-AUTOMATION.md b/README-AUTOMATION.md new file mode 100644 index 000000000..19862b751 --- /dev/null +++ b/README-AUTOMATION.md @@ -0,0 +1,14 @@ +# README-AUTOMATION.md +This file includes details about generating automated content in this package. + +## GitHub Sync Script +**RUN INSTRUCTIONS:** [Instructions for running the GitHub sync script (Quip/WorkDocs link)](https://quip-amazon.com/PL0vAsCa3p10/How-to-Run-the-GitHub-Sync-Script-AmazonEKSDocs) + +**SCRIPT/CODE LOCATION:** [`sync-github.sh`](./sync-github.sh) +The script is located at the top level of this package. + +**DOC PAGES:** [Amazon EKS User Guide – GitHub Repository](https://github.com/awsdocs/amazon-eks-user-guide) +The script keeps the public GitHub repo (`awsdocs/amazon-eks-user-guide`) in sync with the internal GitFarm repository for the Amazon EKS User Guide. Unlike most other AWS documentation, the EKS User Guide is open source. For this reason, it is important to run the sync script **regularly (at least once or twice a week, if not more often)** to ensure contributions and changes remain aligned between the public and internal repositories. + +## Node repair tables +The node repair tables are autogenerated using scripts run by the `eks-node-runtime` team. Any nececessary edits to the descriptions can be ported by employees to the source code. \ No newline at end of file diff --git a/README.md b/README.md index 180508b68..965fa47a7 100644 --- a/README.md +++ b/README.md @@ -2,15 +2,13 @@ Welcome to the Amazon EKS User Guide repository. This repository contains the open source version of the [Amazon EKS User Guide](https://docs.aws.amazon.com/eks/latest/userguide/). -## Important Update +## New Contribution Experience -This repository will be temporarily taken down to prepare for a new contributor experience. The repository will return at the same url by mid-November. +You can now edit the EKS User Guide source directly. The AsciiDoc markup language meets the needs of the AWS Platform, while also being easy to learn. -## New Contribution Experience Coming Soon +Use the "Edit this page on GitHub" links in the right sidebar of the EKS User Guide to submit changes. -We are temporarily taking down the current GitHub repository to prepare for an enhanced contribution experience. The new version will be available in mid-November with the following improvements: - -- **AsciiDoc-Powered Documentation**: The guide will use AsciiDoc, an intuitive yet powerful authoring language similar to Markdown that offers: +- **AsciiDoc-Powered Documentation**: The docs now use AsciiDoc, an intuitive yet powerful authoring language similar to Markdown that offers: - Advanced formatting capabilities - Robust cross-referencing - Enhanced security controls @@ -26,9 +24,7 @@ We are temporarily taking down the current GitHub repository to prepare for an e For more information about the new experience, see [Contribute](https://docs.aws.amazon.com/eks/latest/userguide/contribute.html) in the Amazon EKS User Guide. -We look forward to your contributions when we launch the new GitHub experience. The improved platform will make it easier than ever to help us enhance the Amazon EKS documentation. - - +We look forward to your contributions with the new GitHub experience. The improved platform makes it easier than ever to help us enhance the Amazon EKS documentation. ## License Summary diff --git a/brand.vale.ini b/brand.vale.ini new file mode 100644 index 000000000..ad48fba7f --- /dev/null +++ b/brand.vale.ini @@ -0,0 +1,14 @@ +StylesPath = vale/styles + +MinAlertLevel = error + +# Packages = RedHat, AsciiDoc + +#Vocab = EksDocsVocab + +# Ignore files in dirs starting with `.` to avoid raising errors for `.vale/fixtures/*/testinvalid.adoc` files +[[!.]*.adoc] +BasedOnStyles = EksDocs +EksDocs.ExternalDomains = ON + + diff --git a/build.xml b/build.xml index d8c709f32..be3181b4e 100755 --- a/build.xml +++ b/build.xml @@ -1,6 +1,6 @@ - - + + This is the entry point for happy trails builds (package builder and eclipse). - + \ No newline at end of file diff --git a/check_xrefs.py b/check_xrefs.py new file mode 100755 index 000000000..fdf8ab81e --- /dev/null +++ b/check_xrefs.py @@ -0,0 +1,301 @@ +#!/usr/bin/env python3 +""" +AsciiDoc Cross-Reference Checker + +This script analyzes all .adoc files in a directory to find broken cross-references. +It supports both xref: and <<>> syntax and checks against explicit and auto-generated section IDs. +""" + +import re +import os +from pathlib import Path +from concurrent.futures import ProcessPoolExecutor, as_completed +from collections import defaultdict +from dataclasses import dataclass +from typing import Set, List, Tuple +import sys + + +@dataclass +class XRefInfo: + """Information about a cross-reference""" + file_path: str + line_number: int + xref_id: str + xref_type: str # 'xref' or 'angle_bracket' + + +@dataclass +class FileAnalysis: + """Analysis results for a single file""" + file_path: str + section_ids: Set[str] + xrefs: List[XRefInfo] + errors: List[str] + + +def normalize_id(text: str) -> str: + """ + Normalize a section header to an auto-generated ID. + Based on AsciiDoc rules with idseparator: - + """ + # Convert to lowercase + text = text.lower() + # Remove formatting and special chars, replace spaces with hyphens + text = re.sub(r'[^\w\s-]', '', text) + text = re.sub(r'\s+', '-', text) + # Remove multiple consecutive hyphens + text = re.sub(r'-+', '-', text) + # Remove leading/trailing hyphens + text = text.strip('-') + return text + + +def extract_section_ids(content: str, lines: List[str]) -> Set[str]: + """ + Extract all section IDs from file content. + Supports: + - [[id]] syntax (standalone or inline) + - [#id] syntax (standalone or inline) + - Auto-generated IDs from section headers + """ + section_ids = set() + + # Pattern for explicit [[id]] or [[id,title]] syntax (standalone or inline) + # This pattern works for both "[[id]]" on its own line and "=== Title [[id]]" inline + explicit_bracket_pattern = re.compile(r'\[\[([^\]]+)\]\]') + for match in explicit_bracket_pattern.finditer(content): + # Handle [[id,title]] syntax - ID is the part before the comma + id_text = match.group(1) + section_id = id_text.split(',')[0].strip() + section_ids.add(section_id) + + # Pattern for [#id] syntax (standalone or inline) + explicit_hash_pattern = re.compile(r'\[#([^\]]+)\]') + for match in explicit_hash_pattern.finditer(content): + section_id = match.group(1).split(',')[0].strip() + section_ids.add(section_id) + + # Pattern for section headers (=, ==, ===, etc.) + # Auto-generate IDs from section titles + section_header_pattern = re.compile(r'^(=+)\s+(.+)$', re.MULTILINE) + for match in section_header_pattern.finditer(content): + header_text = match.group(2).strip() + # Remove inline IDs like [[id]] or [#id] from the header text before auto-generating ID + header_text = re.sub(r'\[\[[^\]]+\]\]', '', header_text) + header_text = re.sub(r'\[#[^\]]+\]', '', header_text) + # Remove inline formatting like *bold*, _italic_, etc. + header_text = re.sub(r'\*\*?([^*]+)\*\*?', r'\1', header_text) + header_text = re.sub(r'__?([^_]+)__?', r'\1', header_text) + header_text = re.sub(r'`([^`]+)`', r'\1', header_text) + # Remove links + header_text = re.sub(r'https?://[^\s\[]+', '', header_text) + header_text = re.sub(r'link:[^\[]+\[[^\]]*\]', '', header_text) + + auto_id = normalize_id(header_text) + if auto_id: + section_ids.add(auto_id) + + return section_ids + + +def extract_xrefs(content: str, file_path: str) -> List[XRefInfo]: + """ + Extract all cross-references from file content. + Supports: + - xref:id[...] syntax + - <> syntax + - <> syntax + """ + xrefs = [] + lines = content.split('\n') + + # Pattern for xref:id[...] syntax + xref_pattern = re.compile(r'xref:([a-zA-Z0-9_-]+)(?:\[[^\]]*\])?') + + # Pattern for <> or <> syntax + angle_bracket_pattern = re.compile(r'<<([a-zA-Z0-9_-]+)(?:,[^>]*)?>>') + + for line_num, line in enumerate(lines, 1): + # Find xref: references + for match in xref_pattern.finditer(line): + xref_id = match.group(1) + xrefs.append(XRefInfo( + file_path=file_path, + line_number=line_num, + xref_id=xref_id, + xref_type='xref' + )) + + # Find <<>> references + for match in angle_bracket_pattern.finditer(line): + xref_id = match.group(1) + xrefs.append(XRefInfo( + file_path=file_path, + line_number=line_num, + xref_id=xref_id, + xref_type='angle_bracket' + )) + + return xrefs + + +def analyze_file(file_path: Path) -> FileAnalysis: + """ + Analyze a single .adoc file for section IDs and cross-references. + """ + errors = [] + + try: + with open(file_path, 'r', encoding='utf-8') as f: + content = f.read() + lines = content.split('\n') + + section_ids = extract_section_ids(content, lines) + xrefs = extract_xrefs(content, str(file_path)) + + return FileAnalysis( + file_path=str(file_path), + section_ids=section_ids, + xrefs=xrefs, + errors=errors + ) + + except Exception as e: + errors.append(f"Error reading {file_path}: {str(e)}") + return FileAnalysis( + file_path=str(file_path), + section_ids=set(), + xrefs=[], + errors=errors + ) + + +def find_adoc_files(directory: str) -> List[Path]: + """Find all .adoc files in the directory recursively.""" + path = Path(directory) + return list(path.rglob('*.adoc')) + + +def main(): + """Main function to orchestrate the cross-reference checking.""" + + # Configuration + directory = 'latest/ug/' + + if not os.path.exists(directory): + print(f"Error: Directory '{directory}' not found") + sys.exit(1) + + print(f"Analyzing .adoc files in {directory}...") + + # Find all .adoc files + adoc_files = find_adoc_files(directory) + print(f"Found {len(adoc_files)} .adoc files") + + # Analyze files in parallel + all_section_ids = defaultdict(set) # id -> set of files that define it + all_xrefs = [] + file_errors = [] + + print("\nAnalyzing files in parallel...") + + with ProcessPoolExecutor() as executor: + # Submit all files for analysis + future_to_file = { + executor.submit(analyze_file, file_path): file_path + for file_path in adoc_files + } + + # Collect results as they complete + completed = 0 + for future in as_completed(future_to_file): + completed += 1 + if completed % 50 == 0: + print(f" Processed {completed}/{len(adoc_files)} files...") + + try: + result = future.result() + + # Collect section IDs + for section_id in result.section_ids: + all_section_ids[section_id].add(result.file_path) + + # Collect xrefs + all_xrefs.extend(result.xrefs) + + # Collect errors + if result.errors: + file_errors.extend(result.errors) + + except Exception as e: + file_path = future_to_file[future] + file_errors.append(f"Error processing {file_path}: {str(e)}") + + print(f" Processed {len(adoc_files)}/{len(adoc_files)} files") + + # Report file processing errors + if file_errors: + print("\n" + "="*80) + print("FILE PROCESSING ERRORS") + print("="*80) + for error in file_errors: + print(f" {error}") + + # Check for broken xrefs + print("\n" + "="*80) + print("CHECKING CROSS-REFERENCES") + print("="*80) + print(f"Total section IDs found: {len(all_section_ids)}") + print(f"Total xrefs found: {len(all_xrefs)}") + + broken_xrefs = [] + for xref in all_xrefs: + if xref.xref_id not in all_section_ids: + broken_xrefs.append(xref) + + # Report results + print("\n" + "="*80) + print("RESULTS") + print("="*80) + + if not broken_xrefs: + print("✓ No broken cross-references found!") + else: + print(f"✗ Found {len(broken_xrefs)} broken cross-references:\n") + + # Group by file for better readability + broken_by_file = defaultdict(list) + for xref in broken_xrefs: + broken_by_file[xref.file_path].append(xref) + + for file_path in sorted(broken_by_file.keys()): + print(f"\n{file_path}:") + for xref in sorted(broken_by_file[file_path], key=lambda x: x.line_number): + xref_syntax = f"xref:{xref.xref_id}[...]" if xref.xref_type == 'xref' else f"<<{xref.xref_id}>>" + print(f" Line {xref.line_number}: {xref_syntax}") + + # Summary statistics + print("\n" + "="*80) + print("SUMMARY") + print("="*80) + print(f"Files analyzed: {len(adoc_files)}") + print(f"Section IDs found: {len(all_section_ids)}") + print(f"Cross-references found: {len(all_xrefs)}") + print(f"Broken cross-references: {len(broken_xrefs)}") + + # Check for duplicate section IDs + duplicates = {id: files for id, files in all_section_ids.items() if len(files) > 1} + if duplicates: + print(f"\n⚠ Warning: Found {len(duplicates)} duplicate section IDs:") + for section_id, files in sorted(duplicates.items()): + print(f"\n ID '{section_id}' defined in {len(files)} files:") + for file_path in sorted(files): + print(f" - {file_path}") + + # Exit with error code if broken xrefs found + sys.exit(1 if broken_xrefs else 0) + + +if __name__ == '__main__': + main() diff --git a/eks-docs.code-workspace b/eks-docs.code-workspace index 9f126cbfe..dd5503a15 100644 --- a/eks-docs.code-workspace +++ b/eks-docs.code-workspace @@ -22,6 +22,8 @@ "asciidoc.preview.scrollPreviewWithEditor": true, "asciidoc.preview.scrollEditorWithPreview": true, "asciidoc.antora.showEnableAntoraPrompt": false + // vale extension 0.30.0+ requires an absolute path to the .vale.ini, can't find in the workspace root + // "vale.valeCLI.config": "" }, "extensions": { "recommendations": [ diff --git a/github-sync.sh b/github-sync.sh new file mode 100755 index 000000000..5e5aedd7a --- /dev/null +++ b/github-sync.sh @@ -0,0 +1,97 @@ +#!/bin/bash + +set -e # Exit on any error + +# Configuration +GITHUB_SSH_URL="git@github.com:awsdocs/amazon-eks-user-guide.git" + +# Color codes for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# Helper function for printing status messages +print_status() { + echo -e "${GREEN}==>${NC} $1" +} + +print_warning() { + echo -e "${YELLOW}Warning:${NC} $1" +} + +print_error() { + echo -e "${RED}ERROR:${NC} $1" + exit 1 +} + +# Check if we're in a git repository +if ! git rev-parse --is-inside-work-tree > /dev/null 2>&1; then + print_error "Not in a git repository" +fi + +# Check if current branch is mainline +current_branch=$(git symbolic-ref --short HEAD) +if [ "$current_branch" != "mainline" ]; then + print_error "Must be on 'mainline' branch to sync. Currently on '$current_branch'" +fi + +# Check for origin remote +if ! git remote | grep -q "^origin$"; then + print_error "Remote 'origin' not found" +fi + +# Check for GitHub remote, add if missing +if ! git remote | grep -q "^github$"; then + print_status "GitHub remote not found. Adding it..." + if ! git remote add github "$GITHUB_SSH_URL"; then + print_error "Failed to add GitHub remote" + fi +fi + +# Test GitHub authentication +print_status "Testing GitHub authentication..." +if ! git ls-remote github &>/dev/null; then + print_error "GitHub authentication failed. Please check your SSH keys and permissions" +fi + +# Check for uncommitted changes + if ! git diff-index --quiet HEAD --; then + print_warning "You have uncommitted changes. Please commit or stash them before syncing." + exit 1 + fi + +print_status "Fetching from GitHub remote..." +if ! git fetch github; then + print_error "Failed to fetch from GitHub remote. Check your internet connection and repository permissions" +fi + +print_status "Attempting to merge github/mainline..." +if ! git merge github/mainline --no-edit; then + print_warning "Merge conflicts detected. Please:" + echo "1. Resolve the conflicts" + echo "2. Complete the merge with 'git commit'" + echo "3. Run this script again to finish syncing" + exit 1 +fi + +print_status "Pushing changes to GitHub..." +if ! git push github; then + print_error "Failed to push to GitHub remote. Possible causes:" + echo "- You don't have push permissions" + echo "- The remote branch is protected" + echo "- There are new changes on the remote that you need to pull first" + exit 1 +fi + +print_status "Pushing changes to origin..." +if ! git push origin; then + print_error "Failed to push to origin remote. Possible causes:" + echo "- You don't have push permissions" + echo "- The remote branch is protected" + echo "- There are new changes on the remote that you need to pull first" + exit 1 +fi + +# If we got here, everything worked +print_status "Successfully synced mainline branch between remotes!" \ No newline at end of file diff --git a/latest/ug/attributes.txt b/latest/ug/attributes.txt index 784e706d0..8c96743dc 100644 --- a/latest/ug/attributes.txt +++ b/latest/ug/attributes.txt @@ -1,230 +1,19 @@ -// EKS-specific attributes +:tcx5-waiver: pass:[ ] -:eksctl-min-version: 0.199.0 +// eksctl version +:eksctl-min-version: 0.215.0 -// EKS Auto Mode +// EKS Auto Mode versions :auto-cli-v2-version: 2.12.3 :auto-cli-v1-version: 1.27.160 -// Words Geoffrey often spells wrong or doesn't like to type +// Kubernetes versions +:k8s-n: 1.33 +:k8s-n-1: 1.32 +:k8s-n-2: 1.31 +:k8s-n-3: 1.30 -:ret: retrieve -:resp: responsibility -:det: determine -:cap: capability -:caps: capabilites -:recd: recommended -:config: configuration -:ind: indicate -:ena: enable -:dis: disable - -// AWS shared content - -// Region specific -// Entities that differ depending on the Region build such as China +// Entities that differ depending on the AWS Region build such as China :arn-aws: pass:q[[.shared]``region.arn``] - -// Service names -:amazon-cloudwatch: pass:q[[.shared]``CWlong``] -:amazon-rds: pass:q[[.shared]``RDS``] -:amazon-route-53: pass:q[[.shared]``R53long``] -:amazon-route-53-resolver: pass:q[[.shared]``R53Rlong``] -:amazon-s3: pass:q[[.shared]``S3``] -:amazon-virtual-private-cloud: pass:q[[.shared]``VPClong``] -:amazon-vpc: pass:q[[.shared]``VPC``] -:amazon-elastic-block-store: pass:q[[.shared]``EBSlong``] -:amazon-elastic-file-system: pass:q[[.shared]``EFSlong``] -:amazon-efs: pass:q[[.shared]``EFS``] -:amazon-web-services: pass:q[[.shared]``AWSlong``] :aws: pass:q[[.shared]``AWS``] -:application-load-balancer: pass:q[[.shared]``ALB``] -:application-load-balancers: pass:q[[.shared]``ALBs``] -:aws-account: pass:q[[.shared]``AWS-account``] -:aws-accounts: pass:q[[.shared]``AWS-accounts``] -:aws-always: pass:q[[.shared]``AWS-always``] -:aws-artifact: pass:q[[.shared]``ART``] -:aws-billing: pass:q[[.shared]``Billinglong``] -:aws-billing-cost-management: pass:q[[.shared]``ABlong``] -:aws-cloud: pass:q[[.shared]``AWS-Cloud``] -:aws-cloudtrail: pass:q[[.shared]``CTlong``] -:aws-command-line-interface: pass:q[[.shared]``CLIlong``] -:aws-config: pass:q[[.shared]``CC``] -:aws-cost-explorer: pass:q[[.shared]``AWSCostExplorerServicelong``] -:aws-direct-connect: pass:q[[.shared]``AWS-DC``] -:aws-identity-and-access-management: pass:q[[.shared]``IAMlong``] -:aws-kms: pass:q[[.shared]``KMS``] -:aws-key-management-service: pass:q[[.shared]``KMSlong``] -:aws-kms-key: pass:q[[.shared]``kms-key-long``] -:aws-kms-keys: pass:q[[.shared]``kms-keys-long``] -:aws-license-manager: pass:q[[.shared]``LIClong``] :aws-management-console: pass:q[[.shared]``consolelong``] -:aws-organizations: pass:q[[.shared]``AOlong``] -:aws-marketplace: pass:q[[.shared]``MKT``] -:aws-region: pass:q[[.shared]``AWS-Region``] -:aws-regions: pass:q[[.shared]``AWS-Regions``] -:aws-security-token-service: pass:q[[.shared]``STSlong``] -:aws-service: pass:q[[.shared]``AWS-service``] -:aws-services: pass:q[[.shared]``AWS-services``] -:aws-service-quotas: pass:q[[.shared]``SQ``] -:aws-support: pass:q[[.shared]``SUP``] -:aws-sts: pass:q[[.shared]``STS``] -:aws-transit-gateway: pass:q[[.shared]``AWSTGlong``] -:aws-vpn: pass:q[[.shared]``VPN``] -:classic-load-balancer: pass:q[[.shared]``CLB``] -:classic-load-balancers: pass:q[[.shared]``CLBs``] -:cli: pass:q[[.shared]``CLI``] -:cloudtrail: pass:q[[.shared]``CT``] -:cloudwatch: pass:q[[.shared]``CW``] -:cluster: pass:q[[.shared]``cluster``] -:cluster-cap: pass:q[[.shared]``Cluster``] -:ebs: pass:q[[.shared]``EBS``] -:ec2: pass:q[[.shared]``EC2``] -:ec2-auto-scaling: pass:q[[.shared]``ASlong``] -:elastic-load-balancing: pass:q[[.shared]``ELB``] -:iam: pass:q[[.shared]``IAM``] -:kms-key: pass:q[[.shared]``kms-key``] -:kms-keys: pass:q[[.shared]``kms-keys``] -:license-manager: pass:q[[.shared]``LIC``] -:organizations: pass:q[[.shared]``AO``] -:privatelink: pass:q[[.shared]``privatelink``] -:rosa-service-name-long: pass:q[[.shared]``ROSAlong``] -:rosa-service-name-short: pass:q[[.shared]``ROSA``] -:route-53: pass:q[[.shared]``R53``] -:route-53-resolver: pass:q[[.shared]``R53R``] -:sts: pass:q[[.shared]``STSshort``] -:transit-gateway: pass:q[[.shared]``AWSSTG``] -:cloudformation: pass:q[[.shared]``CFN``] -:outposts: pass:q[[.shared]``OUTlong``] -:eks-a: pass:q[[.shared]``EKS-A``] - -//AWS Regions - -:us-east-1-name: US East (N. Virginia) Region -:us-east-1-region: US East (N. Virginia) -:us-east-1-code: us-east-1 - -:us-east-2-name: US East (Ohio) Region -:us-east-2-region: US East (Ohio) -:us-east-2-code: us-east-2 - -:us-west-1-name: US West (N. California) Region -:us-west-1-region: US West (N. California) -:us-west-1-code: us-west-1 - -:us-west-2-name: US West (Oregon) Region -:us-west-2-region: US West (Oregon) -:us-west-2-code: us-west-2 - -:af-capetown-name: Africa (Cape Town) Region -:af-capetown-region: Africa (Cape Town) -:af-capetown-code: af-south-1 - -:ap-hongkong-name: Asia Pacific (Hong Kong) Region -:ap-hongkong-region: Asia Pacific (Hong Kong) -:ap-hongkong-code: ap-east-1 - -:ap-hyderabad-name: Asia Pacific (Hyderabad) Region -:ap-hyderabad-region: Asia Pacific (Hyderabad) -:ap-hyderabad-code: ap-south-2 - -:ap-jakarta-name: Asia Pacific (Jakarta) Region -:ap-jakarta-region: Asia Pacific (Jakarta) -:ap-jakarta-code: ap-southeast-3 - -:ap-melbourne-name: Asia Pacific (Melbourne) Region -:ap-melbourne-region: Asia Pacific (Melbourne) -:ap-melbourne-code: ap-southeast-4 - -:ap-mumbai-name: Asia Pacific (Mumbai) Region -:ap-mumbai-region: Asia Pacific (Mumbai) -:ap-mumbai-code: ap-south-1 - -:ap-osaka-name: Asia Pacific (Osaka) Region -:ap-osaka-region: Asia Pacific (Osaka) -:ap-osaka-code: ap-northeast-3 - -:ap-seoul-name: Asia Pacific (Seoul) Region -:ap-seoul-region: Asia Pacific (Seoul) -:ap-seoul-code: ap-northeast-2 - -:ap-singapore-name: Asia Pacific (Singapore) Region -:ap-singapore-region: Asia Pacific (Singapore) -:ap-singapore-code: ap-southeast-1 - -:ap-sydney-name: Asia Pacific (Sydney) Region -:ap-sydney-region: Asia Pacific (Sydney) -:ap-sydney-code: ap-southeast-2 - -:ap-tokyo-name: Asia Pacific (Tokyo) Region -:ap-tokyo-region: Asia Pacific (Tokyo) -:ap-tokyo-code: ap-northeast-1 - -:ca-central-name: Canada (Central) Region -:ca-central-region: Canada (Central) -:ca-central-code: ca-central-1 - -:eu-frankfort-name: Europe (Frankfort) Region -:eu-frankfort-region: Europe (Frankfort) -:eu-frankfort-code: eu-central-1 - -:eu-ireland-name: Europe (Ireland) Region -:eu-ireland-region: Europe (Ireland) -:eu-ireland-code: eu-west-1 - -:eu-london-name: Europe (London) Region -:eu-london-region: Europe (London) -:eu-london-code: eu-west-2 - -:eu-milan-name: Europe (Milan) Region -:eu-milan-region: Europe (Milan) -:eu-milan-code: eu-south-1 - -:eu-paris-name: Europe (Paris) Region -:eu-paris-region: Europe (Paris) -:eu-paris-code: eu-west-3 - -:eu-spain-name: Europe (Spain) Region -:eu-spain-region: Europe (Spain) -:eu-spain-code: eu-south-2 - -:eu-stockholm-name: Europe (Stockholm) Region -:eu-stockholm-region: Europe (Stockholm) -:eu-stockholm-code: eu-north-1 - -:eu-zurich-name: Europe (Zurich) Region -:eu-zurich-region: Europe (Zurich) -:eu-zurich-code: eu-central-2 - -:me-bahrain-name: Middle East (Bahrain) Region -:me-bahrain-region: Middle East (Bahrain) -:me-bahrain-code: me-south-1 - -:me-uae-name: Middle East (UAE) Region -:me-uae-region: Middle East (UAE) -:me-uae-code: me-central-1 - -:sa-saopaulo-name: South America (São Paulo) Region -:sa-saopaulo-region: South America (São Paulo) -:sa-saopaulo-code: sa-east-1 - -:govcloud-us: {aws} GovCloud (US) - -:us-gov-east-1-name: {aws} GovCloud (US-East) Region -:us-gov-east-1-region: {aws} GovCloud (US-East) -:us-gov-east-1-code: us-gov-east-1 - -:us-gov-west-1-name: {aws} GovCloud (US-West) Region -:us-gov-east-1-region: {aws} GovCloud (US-West) -:us-gov-east-1-code: us-gov-west-1 - -// EKS Auto Mode attributes - -:yec: your EKS Auto Mode cluster -:yaa: your {aws} account -:emi: EC2 managed instance -:eam: EKS Auto Mode -:mng: managed node group -:e2i: EC2 Instance -:k8s: Kubernetes -:k8s-n: 1.31 \ No newline at end of file diff --git a/latest/ug/automode/api-reference.adoc b/latest/ug/automode/api-reference.adoc index e0e73f46c..f1bc94240 100644 --- a/latest/ug/automode/api-reference.adoc +++ b/latest/ug/automode/api-reference.adoc @@ -1,41 +1,26 @@ - [.topic] = EKA Auto Mode API Reference WIP -:info_doctype: section - -:ind: indicate -:Ind: Indicate -:ena: enable -:dis: disable -:cap: capability -:caps: capabilities -:yec: your EKS Auto Mode cluster -:yaa: your {aws} account -:emi: EC2 Managed Instance -:eam: EKS Auto Mode -:lbi: load balancing -:bs: block storage :fmis: For more information, see :in-guide: in the EKS User Guide :generic-update-request: For example, enable the capability. -:generic-status-request: For example, if the {cap} is {ena}d or {dis}d. -:generic-describe-cap: {Ind}s the current configuration of the {cap} on {yec}. {generic-status-request} +:generic-status-request: For example, if the capability is enabled or disabled. +:generic-describe-cap: Indicates the current configuration of the capability on your EKS Auto Mode cluster. {generic-status-request} :generic-config-request: Request to update the configuration of the -:comp-cap-desc: If the compute {cap} is enabled, {eam} will create and delete {emi}s in {yaa}. -:comp-cap-link: {fmis} {eam} compute {cap} {in-guide}. +:comp-cap-desc: If the compute capability is enabled, EKS Auto Mode will create and delete EC2 managed instances in your {aws} account. +:comp-cap-link: {fmis} EKS Auto Mode compute capability {in-guide}. -:lb-cap-desc: If the load balancing {cap} is enabled, {eam} will create and delete {emi}s in {yaa}. -:lb-cap-link: {fmis} {eam} load balancing {cap} {in-guide}. +:lb-cap-desc: If the load balancing capability is enabled, EKS Auto Mode will create and delete EC2 managed instances in your {aws} account. +:lb-cap-link: {fmis} EKS Auto Mode load balancing capability {in-guide}. -:ebs-cap-desc: If the {bs} {cap} is enabled, {eam} will create and delete EBS volumes in {yaa}. -:ebs-cap-link: {fmis} {eam} {bs} {cap} {in-guide}. +:ebs-cap-desc: If the block storage capability is enabled, EKS Auto Mode will create and delete EBS volumes in your {aws} account. +:ebs-cap-link: {fmis} EKS Auto Mode block storage capability {in-guide}. :iam-link: {fmis} the IAM Reference {in-guide}. -:launch-limitation: Currently, you cannot selectively enable or disable {eam} {caps}. The compute {cap}, {bs} {cap}, and {lbi} {cap} must all be enabled or disabled. You must enable or disable all three capabilities in the same API request. +:launch-limitation: Currently, you cannot selectively enable or disable EKS Auto Mode capabilities. The compute capability, block storage capability, and load balancing capability must all be enabled or disabled. You must enable or disable all three capabilities in the same API request. == Capabilities @@ -52,7 +37,7 @@ // Storage * BlockStorage$controllerRole -** The IAM role used by {eam} to manage EBS volumes. {iam-link} +** The IAM role used by EKS Auto Mode to manage EBS volumes. {iam-link} // missing compute cap? @@ -63,12 +48,12 @@ // Load Balancing * ElasticLoadBalancing$enabled -** {ind}s if the {lbi} {cap} is enabled on {yec}. {lb-cap-desc} +** indicates if the load balancing capability is enabled on your EKS Auto Mode cluster. {lb-cap-desc} // Storage * BlockStorage$enabled -** {ind}s if the {bs} {cap} is enabled on {yec}. {ebs-cap-desc} +** indicates if the block storage capability is enabled on your EKS Auto Mode cluster. {ebs-cap-desc} //missing compute cap? @@ -78,12 +63,12 @@ // Compute * CreateClusterRequest$computeConfig -** Enable or disable the compute {cap} of {eam} when creating {yec}. {comp-cap-desc} +** Enable or disable the compute capability of EKS Auto Mode when creating your EKS Auto Mode cluster. {comp-cap-desc} // Storage * CreateClusterRequest$storageConfig -** Enable or disable the {bs} {cap} of {eam} when creating {yec}. {ebs-cap-desc} +** Enable or disable the block storage capability of EKS Auto Mode when creating your EKS Auto Mode cluster. {ebs-cap-desc} == Cluster$ Config @@ -102,36 +87,36 @@ == ConfigRequest * ComputeConfigRequest -** {generic-config-request} the compute {cap} of your {eam}. {generic-update-request} {comp-cap-link} +** {generic-config-request} the compute capability of your EKS Auto Mode. {generic-update-request} {comp-cap-link} * StorageConfigRequest -** {generic-config-request} the storage {cap} of your {eam}. {generic-update-request} {ebs-cap-link} +** {generic-config-request} the storage capability of your EKS Auto Mode. {generic-update-request} {ebs-cap-link} === Load Balancing * KubernetesNetworkConfigRequest$elasticLoadBalancing -** Request to {ena} or {dis} the {lbi} {cap} on {yec}. {lb-cap-link} +** Request to enable or disable the load balancing capability on your EKS Auto Mode cluster. {lb-cap-link} === Compute * ComputeConfigRequest$enabled -** Request to {ena} or {dis} the compute {cap} on {yec}. {comp-cap-desc} +** Request to enable or disable the compute capability on your EKS Auto Mode cluster. {comp-cap-desc} * ComputeConfigRequest$nodePools -** Configuration for node pools that defines the compute resources for {yec}. {fmis} {eam} Node Pools {in-guide}. +** Configuration for node pools that defines the compute resources for your EKS Auto Mode cluster. {fmis} EKS Auto Mode Node Pools {in-guide}. * ComputeConfigRequest$nodeRoleArn -** The ARN of the IAM Role EKS will assign to {emi}s in {yec}. This value cannot be changed after the compute {cap} of {eam} is enabled. {iam-link} +** The ARN of the IAM Role EKS will assign to EC2 managed instances in your EKS Auto Mode cluster. This value cannot be changed after the compute capability of EKS Auto Mode is enabled. {iam-link} === Storage * StorageConfigRequest$blockStorage -** Request to configure EBS Block Storage settings for {yec}. +** Request to configure EBS Block Storage settings for your EKS Auto Mode cluster. == ConfigResponse @@ -139,23 +124,23 @@ // Compute * ComputeConfigResponse -** {ind}s {status-of-request} the compute {cap} of {yec}. +** indicates {status-of-request} the compute capability of your EKS Auto Mode cluster. // Storage * StorageConfigResponse -** {ind}s {status-of-request} the {bs} {cap} of {yec}. +** indicates {status-of-request} the block storage capability of your EKS Auto Mode cluster. === Response pointers to objects // Storage * StorageConfigResponse$blockStorage -** {ind}s the current configuration of the {bs} {cap} on {yec}. {generic-status-request} +** indicates the current configuration of the block storage capability on your EKS Auto Mode cluster. {generic-status-request} // Load Balancing * $elasticLoadBalancing -** {ind}s the current configuration of the {lbi} {cap} on {yec}. {generic-status-request} +** indicates the current configuration of the load balancing capability on your EKS Auto Mode cluster. {generic-status-request} === Compute Details @@ -163,16 +148,16 @@ // Compute * ComputeConfigResponse$enabled -** {ind}s if the compute {cap} is enabled on {yec}. {comp-cap-desc} +** indicates if the compute capability is enabled on your EKS Auto Mode cluster. {comp-cap-desc} // Compute * ComputeConfigResponse$nodePools -** {ind}s the current configuration of node pools in {yec}. {fmis} {eam} Node Pools {in-guide}. +** indicates the current configuration of node pools in your EKS Auto Mode cluster. {fmis} EKS Auto Mode Node Pools {in-guide}. // Compute * ComputeConfigResponse$nodeRoleArn -** The ARN of the IAM Role EKS will assign to {emi}s in {yec}. +** The ARN of the IAM Role EKS will assign to EC2 managed instances in your EKS Auto Mode cluster. == UpdateClusterConfigRequest @@ -180,11 +165,11 @@ // Storage * UpdateClusterConfigRequest$storageConfig -** {update-config} the {bs} {cap} of {yec}. {generic-update-request} +** {update-config} the block storage capability of your EKS Auto Mode cluster. {generic-update-request} // Compute * UpdateClusterConfigRequest$computeConfig -** {update-config} the compute {cap} of {yec}. {generic-update-request} +** {update-config} the compute capability of your EKS Auto Mode cluster. {generic-update-request} //where is LB? diff --git a/latest/ug/automode/associate-workload.adoc b/latest/ug/automode/associate-workload.adoc index 232512ed5..08f2e9c3b 100644 --- a/latest/ug/automode/associate-workload.adoc +++ b/latest/ug/automode/associate-workload.adoc @@ -1,42 +1,38 @@ -//!!NODE_ROOT
- include::../attributes.txt[] [.topic] -[[associate-workload,associate-workload.title]] +[#associate-workload] = Control if a workload is deployed on EKS Auto Mode nodes -:info_doctype: section -:info_title: Control if a workload is deployed on EKS Auto Mode nodes -:info_titleabbrev: Control workload deployment -:info_abstract: Control if a workload is deployed on EKS Auto Mode nodes +:info_titleabbrev: Control deployment -When running workloads in an EKS cluster with {eam}, you might need to control whether specific workloads run on {eam} nodes or other compute types. This topic describes how to use node selectors and affinity rules to ensure your workloads are scheduled on the intended compute infrastructure. +When running workloads in an EKS cluster with EKS Auto Mode, you might need to control whether specific workloads run on EKS Auto Mode nodes or other compute types. This topic describes how to use node selectors and affinity rules to ensure your workloads are scheduled on the intended compute infrastructure. -The examples in this topic demonstrate how to use the `eks.amazonaws.com/compute-type` label to either require or prevent workload deployment on {eam} nodes. This is particularly useful in mixed-mode clusters where you're running both {eam} and other compute types, such as self-managed Karpenter provisioners or EKS Managed Node Groups. +The examples in this topic demonstrate how to use the `eks.amazonaws.com/compute-type` label to either require or prevent workload deployment on EKS Auto Mode nodes. This is particularly useful in mixed-mode clusters where you're running both EKS Auto Mode and other compute types, such as self-managed Karpenter provisioners or EKS Managed Node Groups. -{eam} nodes have set the value of the label `eks.amazonaws.com/compute-type` to `auto`. You can use this label to control if a workload is deployed to nodes managed by {eam}. +EKS Auto Mode nodes have set the value of the label `eks.amazonaws.com/compute-type` to `auto`. You can use this label to control if a workload is deployed to nodes managed by EKS Auto Mode. -== Require a workload is deployed to {eam} nodes +== Require a workload is deployed to EKS Auto Mode nodes [NOTE] ==== -This `nodeSelector` value is not required for {eam}. This `nodeSelector` value is only relevant if you are running a cluster in a mixed mode, node types not managed by {eam}. For example, you may have static compute capacity deployed to your cluster with EKS Managed Node Groups, and have dynamic compute capacity managed by {eam}. +This `nodeSelector` value is not required for EKS Auto Mode. This `nodeSelector` value is only relevant if you are running a cluster in a mixed mode, node types not managed by EKS Auto Mode. For example, you may have static compute capacity deployed to your cluster with EKS Managed Node Groups, and have dynamic compute capacity managed by EKS Auto Mode. ==== -You can add this `nodeSelector` to Deployments or other workloads to require Kubernetes schedule them onto {eam} nodes. +You can add this `nodeSelector` to Deployments or other workloads to require Kubernetes schedule them onto EKS Auto Mode nodes. [source,yaml] ---- apiVersion: apps/v1 kind: Deployment spec: - nodeSelector: - eks.amazonaws.com/compute-type: auto + template: + nodeSelector: + eks.amazonaws.com/compute-type: auto ---- -== Require a workload is not deployed to {eam} nodes +== Require a workload is not deployed to EKS Auto Mode nodes -You can add this `nodeAffinity` to Deployments or other workloads to require Kubernetes *not* schedule them onto {eam} nodes. +You can add this `nodeAffinity` to Deployments or other workloads to require Kubernetes *not* schedule them onto EKS Auto Mode nodes. [source,yaml] ---- diff --git a/latest/ug/automode/auto-accelerated.adoc b/latest/ug/automode/auto-accelerated.adoc new file mode 100644 index 000000000..6ad25ab79 --- /dev/null +++ b/latest/ug/automode/auto-accelerated.adoc @@ -0,0 +1,278 @@ +[.topic] +[#auto-accelerated] += Deploy an accelerated workload +:info_titleabbrev: Deploy accelerated workload + +include::../attributes.txt[] + + +This tutorial demonstrates how Amazon EKS Auto Mode simplifies launching hardware-accelerated workloads. Amazon EKS Auto Mode streamlines operations beyond the cluster itself by automating key infrastructure components providing compute, networking, load balancing, storage, and Identity Access and Management capabilities out of the box. + +Amazon EKS Auto Mode includes the drivers and device plugins required for certain instance types, such as NVIDIA and {aws} Neuron drivers. You do not have to install or update these components. + +EKS Auto Mode automatically manages drivers for these accelerators: + +* link:ai/machine-learning/trainium/[{aws} Trainium, type="marketing"] +* link:ai/machine-learning/inferentia/[{aws} Inferentia, type="marketing"] +* link:ec2/latest/instancetypes/ac.html[NVIDIA GPUs on Amazon EC2 accelerated instances, type="documentation"] + +NOTE: EKS Auto Mode includes the NVIDIA device plugin for Kubernetes. This plugin runs automatically and isn't visible as a daemon set in your cluster. + +Additional networking support: + +* link:hpc/efa/[Elastic Fabric Adapter (EFA), type="marketing"] + +Amazon EKS Auto Mode eliminates the toil of accelerator driver and device plugin management. + +You can also benefit from cost savings by scaling the cluster to zero. You can configure EKS Auto Mode to terminate instances when no workloads are running. This is useful for batch based inference workloads. + +The following provides an example of how to launch accelerated workloads with Amazon EKS Auto Mode. + +== Prerequisites + +* A Kubernetes cluster with Amazon EKS Auto Mode configured. +* A `default` EKS Node class as created when the `general-purpose` or `system` Managed Node Pools are enabled. + +== Step 1: Deploy a GPU workload + +In this example, you will create a NodePool for NVIDIA based workloads that requires 45GB GPU memory. With EKS Auto Mode, you use Kubernetes scheduling constraints to define your instance requirements. + +To deploy the Amazon EKS Auto Mode `NodePool` and the sample `workload`, review the following NodePool and Pod definition and save as `nodepool-gpu.yaml` and `pod.yaml`: + +*nodepool-gpu.yaml* + +[source,yaml] +---- +apiVersion: karpenter.sh/v1 +kind: NodePool +metadata: + name: gpu +spec: + disruption: + budgets: + - nodes: 10% + consolidateAfter: 1h + consolidationPolicy: WhenEmpty + template: + metadata: {} + spec: + nodeClassRef: + group: eks.amazonaws.com + kind: NodeClass + name: default + requirements: + - key: "karpenter.sh/capacity-type" + operator: In + values: ["on-demand"] + - key: "kubernetes.io/arch" + operator: In + values: ["amd64"] + - key: "eks.amazonaws.com/instance-family" + operator: In + values: + - g6e + - g6 + taints: + - key: nvidia.com/gpu + effect: NoSchedule + terminationGracePeriod: 24h0m0s +---- + +*pod.yaml* + +[source,yaml] +---- +apiVersion: v1 +kind: Pod +metadata: + name: nvidia-smi +spec: + nodeSelector: + eks.amazonaws.com/compute-type: auto + restartPolicy: OnFailure + containers: + - name: nvidia-smi + image: public.ecr.aws/amazonlinux/amazonlinux:2023-minimal + args: + - "nvidia-smi" + resources: + requests: + memory: "30Gi" + cpu: "3500m" + nvidia.com/gpu: 1 + limits: + memory: "30Gi" + nvidia.com/gpu: 1 + tolerations: + - key: nvidia.com/gpu + effect: NoSchedule + operator: Exists +---- + +Note the `eks.amazonaws.com/compute-type: auto` selector requires the workload be deployed on an Amazon EKS Auto Mode node. The NodePool also sets a taint that only allows pods with tolerations for Nvidia GPUs to be scheduled. + +Apply the NodePool and workload to your cluster. + +[source,bash] +---- +kubectl apply -f nodepool-gpu.yaml +kubectl apply -f pod.yaml +---- + +You should see the following output: + +[source,bash] +---- +nodepool.karpenter.sh/gpu configured created +pod/nvidia-smi created +---- + +Wait a few seconds, and check the nodes in your cluster. You should now see a new node provisioned in your Amazon EKS Auto Mode cluster: + +[source,bash] +---- +> kubectl get nodes + +NAME TYPE CAPACITY ZONE NODE READY AGE +gpu-dnknr g6e.2xlarge on-demand us-west-2b i-02315c7d7643cdee6 True 76s +---- + +== Step 2: Validate + +You can see Amazon EKS Auto Mode launched a `g6e.2xlarge` rather than an `g6.2xlarge` as the workload required an instance with l40s `GPU`, according to the following Kubernetes scheduling constraints: + +[source,yaml] +---- +... + nodeSelector: + eks.amazonaws.com/instance-gpu-name: l40s +... + requests: + memory: "30Gi" + cpu: "3500m" + nvidia.com/gpu: 1 + limits: + memory: "30Gi" + nvidia.com/gpu: 1 +---- + +Now, look at the containers logs, by running the following command: + +[source,bash] +---- +kubectl logs nvidia-smi +---- + +Sample output: + +[source,bash] +---- ++---------------------------------------------------------------------------------------+ +| NVIDIA-SMI 535.230.02 Driver Version: 535.230.02 CUDA Version: 12.2 | +|-----------------------------------------+----------------------+----------------------+ +| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC | +| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. | +| | | MIG M. | +|=========================================+======================+======================| +| 0 NVIDIA L40S On | 00000000:30:00.0 Off | 0 | +| N/A 27C P8 23W / 350W | 0MiB / 46068MiB | 0% Default | +| | | N/A | ++-----------------------------------------+----------------------+----------------------+ + ++---------------------------------------------------------------------------------------+ +| Processes: | +| GPU GI CI PID Type Process name GPU Memory | +| ID ID Usage | +|=======================================================================================| +| No running processes found | ++---------------------------------------------------------------------------------------+ +---- + +You can see that the container has detected it's running on an instance with an `NVIDIA` GPU and that you've not had to install any device drivers, as this is managed by Amazon EKS Auto Mode. + +== Step 3: Clean-up + +To remove all objects created, use `kubectl` to delete the sample deployment and NodePool so the node is terminated: + +---- +kubectl delete -f nodepool-gpu.yaml +kubectl delete -f pod.yaml +---- + + +== Example NodePools Reference + +=== Create an NVIDIA NodePool + +The following NodePool defines: + +* Only launch instances of `g6e` and `g6` family +* Consolidate nodes when empty for 1 hour +** The 1 hour value for `consolodateAfter` supports spiky workloads and reduce node churn. You can tune `consolidateAfter` based on your workload requirements. + +*Example NodePool with GPU instance family and consolidation* + +[source,yaml] +---- +apiVersion: karpenter.sh/v1 +kind: NodePool +metadata: + name: gpu +spec: + disruption: + budgets: + - nodes: 10% + consolidateAfter: 1h + consolidationPolicy: WhenEmpty + template: + metadata: {} + spec: + nodeClassRef: + group: eks.amazonaws.com + kind: NodeClass + name: default + requirements: + - key: "karpenter.sh/capacity-type" + operator: In + values: ["on-demand"] + - key: "kubernetes.io/arch" + operator: In + values: ["amd64"] + - key: "eks.amazonaws.com/instance-family" + operator: In + values: + - g6e + - g6 + terminationGracePeriod: 24h0m0s +---- + +Instead of to setting the `eks.amazonaws.com/instance-gpu-name` you might use `eks.amazonaws.com/instance-family` to specify the instance family. For other well-known labels which influence scheduling review, see <>. + +If you have specific storage requirements you can tune the nodes ephemeral storage `iops`, `size` and `throughput` by creating your own xref:create-node-class[NodeClass] to reference in the NodePool. Learn more about the xref:create-node-class[configurable NodeClass options]. + +*Example storage configuration for NodeClass* + +[source,yaml] +---- +apiVersion: eks.amazonaws.com/v1 +kind: NodeClass +metadata: + name: gpu +spec: + ephemeralStorage: + iops: 3000 + size: 80Gi + throughput: 125 +---- + +=== Define an {aws} Trainium and {aws} Inferentia NodePool + +The following NodePool has an `eks.amazonaws.com/instance-category` set that says, only launch instances of Inferentia and Trainium family: + +---- + - key: "eks.amazonaws.com/instance-category" + operator: In + values: + - inf + - trn +---- diff --git a/latest/ug/automode/auto-change.adoc b/latest/ug/automode/auto-change.adoc new file mode 100644 index 000000000..f0393ff5c --- /dev/null +++ b/latest/ug/automode/auto-change.adoc @@ -0,0 +1,97 @@ +[.topic] +[#auto-change] += Review EKS Auto Mode release notes +:info_titleabbrev: Release notes + +include::../attributes.txt[] + +This page documents updates to Amazon EKS Auto Mode. You can periodically check this page for announcements about features, bug fixes, known issues, and deprecated functionality. + +To receive notifications of all source file changes to this specific documentation page, you can subscribe to the following URL with an RSS reader: + +[source,none] +---- +https://github.com/awsdocs/amazon-eks-user-guide/commits/mainline/latest/ug/automode/auto-change.adoc.atom +---- + +== October 23, 2025 + +*Feature:* Users with clusters in US regions can now request to use FIPS compatible AMIs by specifying `spec.advancedSecurity.fips` in their NodeClass definition. + +== October 1, 2025 + +*Feature:* EKS Auto Mode now supports deploying nodes to {aws} Local Zones. For more information, see <>. + +== September 30, 2025 + +*Feature:* Added support for instanceProfile to the NodeClass `spec.instanceProfile` which is mutually exclusive from the `spec.role` field. + +== September 29, 2025 + +DRA is not currently supported by EKS Auto Mode. + +== September 10, 2025 + +*Chore:* Events fired from the Auto Mode Compute controller will now use the name `eks-auto-mode/compute` instead of `karpenter`. + + +== August 24, 2025 + +*Bug Fix:* VPCs that used a DHCP option set with a custom domain name that contained capital letters would cause Nodes to fail to join the cluster due to generating an invalid hostname. This has been resolved and domain names with capital letters now work correctly. + + +== August 15, 2025 + +*Bug Fix:* The Pod Identity Agent will now only listen on the IPv4 Link Local address in an IPv4 EKS cluster to avoid issues where the Pod can't reach the IPv6 address. + +== August 6, 2025 + +*Feature:* Added new configuration on the NodeClass `spec.advancedNetworking.associatePublicIPAddress` which can be used to prevent public IP addresses from being assigned to EKS Auto Mode Nodes + + +== June 30, 2025 + +*Feature:* The Auto Mode NodeClass now uses the configured custom KMS key to encrypt the read-only root volume of the instance, in addition to the read/write data volume. Previously, the custom KMS key was only used to encrypt the data volume. + + +== June 20, 2025 + +*Feature:* Support for controlling deployment of workloads into EC2 On-Demand Capacity Reservations (ODCRs). This adds the optional key `capacityReservationSelectorTerms` to the NodeClass, allowing you to explicitly control which ODCRs your workloads use. For more information, see <>. + +== June 13, 2025 + +*Feature:* Support for separate pod subnets in the `NodeClass`. This adds the optional keys ``podSubnetSelectorTerms` and `podSecurityGroupSelectorTerms` to set the subnets and security groups for the pods. For more information, see <>. + +== April 30, 2025 + +*Feature:* Support for forward network proxies in the `NodeClass`. This adds the optional key `advancedNetworking` to set your HTTPS proxy. For more information, see <>. + +== April 18, 2025 + +*Feature:* Support for resolving .local domains (typically reserved for Multicast DNS) via unicast DNS. + +== April 11, 2025 + +*Feature:* Added `certificateBundles` and `ephemeralStorage.kmsKeyID` to `NodeClass`. For more information, see <>. + +*Feature:* Improved image pull speed, particularly for instance types with local instance storage that can take advantage of the faster image decompression. + +*Bug Fix:* Resolved a race condition which caused FailedCreatePodSandBox , Error while dialing: dial tcp 127.0.0.1:50051: connect: connection refused to sometimes occur for Pods scheduling to a Node immediately at startup. + + +== April 4, 2025 + +**Feature:** Increase `registryPullQPS` from 5 to 25 and `registryBurst` from 10 to 50 to reduce client enforced image pull throttling (`Failed to pull image xyz: pull QPS exceeded`) + + +== March 31, 2025 + +**Bug Fix:** Fixes an issue where if a Core DNS Pod is running on an Auto Mode node, DNS queries from Pods on the node would hit that Core DNS Pod instead of the node local DNS server. DNS queries from Pods on an Auto Mode node will always go to the node local DNS. + +== March 21, 2025 + +**Bug Fix:** Auto Mode nodes now resolve `kube-dns.kube-system.svc.cluster.local` correctly when there isn't a `kube-dns` service installed in the cluster. Addresses GitHub issue https://github.com/aws/containers-roadmap/issues/2546[#2546]. + +== March 14, 2025 + +**Feature**: `IPv4` egress enabled in `IPv6` clusters. `IPv4` traffic egressing from `IPv6` Auto Mode clusters will now be automatically translated to the `v4` address of the node primary ENI. diff --git a/latest/ug/automode/auto-cis.adoc b/latest/ug/automode/auto-cis.adoc new file mode 100644 index 000000000..e35727f0c --- /dev/null +++ b/latest/ug/automode/auto-cis.adoc @@ -0,0 +1,101 @@ +include::../attributes.txt[] + +[.topic] +[#auto-cis] += Generate CIS compliance reports from Kubernetes nodes using kubectl debug +:info_titleabbrev: Generate CIS report + +This topic describes how to generate CIS (Center for Internet Security) compliance reports for Amazon EKS nodes using the `kubectl debug` command. +The command allows you to temporarily create a debugging container on a Kubernetes node and run CIS compliance checks using the `apiclient` tool. The `apiclient` tool is part of Bottlerocket OS, the OS used by EKS Auto Mode nodes. + +== Prerequisites + +Before you begin, ensure you have: + +* Access to an Amazon EKS cluster with `kubectl` configured (version must be at least v1.32.0; type `kubectl version` to check). +* The appropriate IAM permissions to debug nodes. +* A valid profile that allows debug operations (e.g., `sysadmin`). + +For more information about using debugging profiles with `kubectl`, see https://kubernetes.io/docs/tasks/debug/debug-application/debug-running-pod/#debugging-profiles[Debugging a Pod or Node while applying a profile] in the Kubernetes documentation. + +== Procedure + +. Determine the {aws} Instance ID of the node you want to run the report on. Use the following command to list the nodes in the cluster. The instance ID is found in the name column, and begins with `i-`: ++ +[source,bash] +---- +kubectl get nodes +---- ++ +[source] +---- +NAME STATUS ROLES AGE VERSION +i-0ea0ba0f8ef9ad609 Ready 62s v1.30.10-eks-1a9dacd +---- +. Run the following command, replacing `` with the instance ID of the node you want to query: ++ +[source,bash] +---- +kubectl debug node/ -it --profile=sysadmin --image=public.ecr.aws/amazonlinux/amazonlinux:2023 -- bash -c "yum install -q -y util-linux-core; nsenter -t 1 -m apiclient report cis --level 1 --format text" +---- ++ +Components of this command include: ++ +* `kubectl debug node/` -- Creates a debugging session on the specified EC2 instance ID. +* `-it` -- Allocates a TTY (command line shell) and keeps stdin open for interactive usage. +* `--profile=sysadmin` -- Uses the specified `kubectl` profile with appropriate permissions. +* `--image=public.ecr.aws/amazonlinux/amazonlinux:2023` -- Uses `amazonlinux:2023` as the container image for debugging. +* `bash -c "..."` -- Executes the following commands in a bash shell: +** `yum install -q -y util-linux-core` -- Quietly installs the required utilities package. +** `nsenter -t 1 -m` -- Runs `nsenter` to enter the namespace of the host process (PID 1). +** `apiclient report cis --level 1 --format text` -- Runs the CIS compliance report at level 1 with text output. + +. Review the report text output. + +== Interpreting the output + +The command generates a text-based report showing the compliance status of various CIS controls. The output includes: + +* Individual CIS control IDs +* Description of each control +* Pass, Fail, or Skip status for each check +* Details that explain any compliance issues + +Here is an example of output from the report run on a Bottlerocket instance: + +[source] +---- +Benchmark name: CIS Bottlerocket Benchmark +Version: v1.0.0 +Reference: https://www.cisecurity.org/benchmark/bottlerocket +Benchmark level: 1 +Start time: 2025-04-11T01:40:39.055623436Z + +[SKIP] 1.2.1 Ensure software update repositories are configured (Manual) +[PASS] 1.3.1 Ensure dm-verity is configured (Automatic)[PASS] 1.4.1 Ensure setuid programs do not create core dumps (Automatic) +[PASS] 1.4.2 Ensure address space layout randomization (ASLR) is enabled (Automatic) +[PASS] 1.4.3 Ensure unprivileged eBPF is disabled (Automatic) +[PASS] 1.5.1 Ensure SELinux is configured (Automatic) +[SKIP] 1.6 Ensure updates, patches, and additional security software are installed (Manual) +[PASS] 2.1.1.1 Ensure chrony is configured (Automatic) +[PASS] 3.2.5 Ensure broadcast ICMP requests are ignored (Automatic) +[PASS] 3.2.6 Ensure bogus ICMP responses are ignored (Automatic) +[PASS] 3.2.7 Ensure TCP SYN Cookies is enabled (Automatic) +[SKIP] 3.4.1.3 Ensure IPv4 outbound and established connections are configured (Manual) +[SKIP] 3.4.2.3 Ensure IPv6 outbound and established connections are configured (Manual) +[PASS] 4.1.1.1 Ensure journald is configured to write logs to persistent disk (Automatic) +[PASS] 4.1.2 Ensure permissions on journal files are configured (Automatic) + +Passed: 11 +Failed: 0 +Skipped: 4 +Total checks: 15 +---- + +For information about the benchmark, see https://www.cisecurity.org/benchmark/kubernetes/[Kubernetes Benchmark] from the Center for Internet Security (CIS). + +== Related resources + +* https://bottlerocket.dev/en/os/1.34.x/api/reporting/cis/[Bottlerocket CIS Benchmark] in Bottlerocket OS Documentation. +* https://kubernetes.io/docs/tasks/debug/debug-application/debug-running-pod/[Debug Running Pods] in the Kubernetes Documentation. +* https://www.cisecurity.org/benchmark/kubernetes/[Kubernetes Benchmark] from the Center for Internet Security (CIS) diff --git a/latest/ug/automode/auto-configure-alb.adoc b/latest/ug/automode/auto-configure-alb.adoc index c5fbc1fe0..839f435ec 100644 --- a/latest/ug/automode/auto-configure-alb.adoc +++ b/latest/ug/automode/auto-configure-alb.adoc @@ -1,36 +1,96 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[auto-configure-alb,auto-configure-alb.title]] +[#auto-configure-alb] = Create an IngressClass to configure an Application Load Balancer -:info_doctype: section :info_titleabbrev: Create ingress class -include::../attributes.txt[] - EKS Auto Mode automates routine tasks for load balancing, including exposing cluster apps to the internet. {aws} suggests using Application Load Balancers (ALB) to serve HTTP and HTTPS traffic. Application Load Balancers can route requests based on the content of the request. For more information on Application Load Balancers, see link:elasticloadbalancing/latest/userguide/what-is-load-balancing.html["What is Elastic Load Balancing?",type="documentation"] EKS Auto Mode creates and configures Application Load Balancers (ALBs). For example, EKS Auto Mode creates a load balancer when you create an `Ingress` Kubernetes objects and configures it to route traffic to your cluster workload. -**Overview** +*Overview* +. Create a workload that you want to expose to the internet. . Create an `IngressClassParams` resource, specifying {aws} specific configuration values such as the certificate to use for SSL/TLS and VPC Subnets. . Create an `IngressClass` resource, specifying that EKS Auto Mode will be the controller for the resource. . Create an `Ingress` resource that associates a HTTP path and port with a cluster workload. -. EKS Auto Mode will create an Application Load Balancer that points to the workload specified in the `Ingress` resource, using the load balancer configuration specified in the `IngressClassParams` resource. -## Prerequisites +EKS Auto Mode will create an Application Load Balancer that points to the workload specified in the `Ingress` resource, using the load balancer configuration specified in the `IngressClassParams` resource. + +== Prerequisites * EKS Auto Mode Enabled on an Amazon EKS Cluster * Kubectl configured to connect to your cluster ** You can use `kubectl apply -f ` to apply the sample configuration YAML files below to your cluster. -## Step 1: Create IngressClassParams +[NOTE] +==== +EKS Auto Mode requires subnet tags to identify public and private subnets. + +If you created your cluster with `eksctl`, you already have these tags. -Create an `IngressClassParams` object to specify {aws} specific configuration options for the Application Load Balancer. Use the reference below to update the sample YAML file. +Learn how to <>. +==== -Note the name you set for the `IngressClassParams` resource, you will need it in the next step. +== Step 1: Create a workload + +To begin, create a workload that you want to expose to the internet. This can be any Kubernetes resource that serves HTTP traffic, such as a Deployment or a Service. + +This example uses a simple HTTP service called `service-2048` that listens on port `80`. Create this service and its deployment by applying the following manifest, `2048-deployment-service.yaml`: + +```yaml +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deployment-2048 +spec: + selector: + matchLabels: + app.kubernetes.io/name: app-2048 + replicas: 2 + template: + metadata: + labels: + app.kubernetes.io/name: app-2048 + spec: + containers: + - image: public.ecr.aws/l6m2t8p7/docker-2048:latest + imagePullPolicy: Always + name: app-2048 + ports: + - containerPort: 80 +--- +apiVersion: v1 +kind: Service +metadata: + name: service-2048 +spec: + ports: + - port: 80 + targetPort: 80 + protocol: TCP + type: NodePort + selector: + app.kubernetes.io/name: app-2048 +``` + +Apply the configuration to your cluster: +```bash +kubectl apply -f 2048-deployment-service.yaml +``` + +The resources listed above will be created in the default namespace. You can verify this by running the following command: +```bash +kubectl get all -n default +``` + +== Step 2: Create IngressClassParams + +Create an `IngressClassParams` object to specify {aws} specific configuration options for the Application Load Balancer. In this example, we create an `IngressClassParams` resource named `alb` (which you will use in the next step) that specifies the load balancer scheme as `internet-facing` in a file called `alb-ingressclassparams.yaml`. ```yaml apiVersion: eks.amazonaws.com/v1 @@ -40,12 +100,13 @@ metadata: spec: scheme: internet-facing ``` +Apply the configuration to your cluster: +```bash +kubectl apply -f alb-ingressclassparams.yaml +``` +== Step 3: Create IngressClass - - -## Step 2: Create IngressClass - -Create an `IngressClass` that references the {aws} specific configuration values set in the `IngressClassParams` resource. Note the name of the `IngressClass` . In this example, both the `IngressClass` and `IngressClassParams` are named `alb`. +Create an `IngressClass` that references the {aws} specific configuration values set in the `IngressClassParams` resource in a file named `alb-ingressclass.yaml`. Note the name of the `IngressClass`. In this example, both the `IngressClass` and `IngressClassParams` are named `alb`. Use the `is-default-class` annotation to control if `Ingress` resources should use this class by default. @@ -70,9 +131,14 @@ spec: For more information on configuration options, see <>. -## Step 3: Create Ingress +Apply the configuration to your cluster: +```bash +kubectl apply -f alb-ingressclass.yaml +``` + +== Step 4: Create Ingress -Create an `Ingress` resource. The purpose of this resource is to associate paths and ports on the Application Load Balancer with workloads in your cluster. +Create an `Ingress` resource in a file named `alb-ingress.yaml`. The purpose of this resource is to associate paths and ports on the Application Load Balancer with workloads in your cluster. For this example, we create an `Ingress` resource named `2048-ingress` that routes traffic to a service named `service-2048` on port 80. For more information about configuring this resource, see https://kubernetes.io/docs/concepts/services-networking/ingress/[Ingress] in the Kubernetes Documentation. @@ -92,82 +158,100 @@ spec: pathType: ImplementationSpecific backend: service: - name: + name: service-2048 port: number: 80 ``` +Apply the configuration to your cluster: +```bash +kubectl apply -f alb-ingress.yaml +``` - -## Step 4: Check Status +== Step 5: Check Status Use `kubectl` to find the status of the `Ingress`. It can take a few minutes for the load balancer to become available. -Use the name of the `Ingress` resource you set in the previous step. +Use the name of the `Ingress` resource you set in the previous step. For example: -``` -kubectl get ingress +```bash +kubectl get ingress 2048-ingress ``` Once the resource is ready, retrieve the domain name of the load balancer. -``` -kubectl get ingress api-ingress -o jsonpath='{.status.loadBalancer.ingress[0].hostname}' +```bash +kubectl get ingress 2048-ingress -o jsonpath='{.status.loadBalancer.ingress[0].hostname}' ``` To view the service in a web browser, review the port and path specified in the `Ingress` rescue. -## Step 5: Cleanup +== Step 6: Cleanup To clean up the load balancer, use the following command: -``` -kubectl delete ingress +```bash +kubectl delete ingress 2048-ingress +kubectl delete ingressclass alb +kubectl delete ingressclassparams alb ``` EKS Auto Mode will automatically delete the associated load balancer in your {aws} account. -[[ingress-reference,ingress-reference.title]] +[#ingress-reference] == IngressClassParams Reference The table below is a quick reference for commonly used configuration options. -[cols="3*", options="header"] +[%header,cols="3"] |=== -|Field |Description |Example Value +|Field +|Description +|Example value + |`scheme` |Defines whether the ALB is internal or internet-facing |`internet-facing` + |`namespaceSelector` |Restricts which namespaces can use this IngressClass |`environment: prod` + |`group.name` |Groups multiple Ingresses to share a single ALB |`retail-apps` + |`ipAddressType` |Sets IP address type for the ALB |`dualstack` + |`subnets.ids` |List of subnet IDs for ALB deployment |`subnet-xxxx, subnet-yyyy` + |`subnets.tags` |Tag filters to select subnets for ALB |`Environment: prod` + |`certificateARNs` |ARNs of SSL certificates to use -|`arn:aws:acm:region:account:certificate/id` +|`{arn-aws}acm:region:account:certificate/id` + |`tags` |Custom tags for {aws} resources |`Environment: prod, Team: platform` + |`loadBalancerAttributes` |Load balancer specific attributes |`idle_timeout.timeout_seconds: 60` + |=== == Considerations * You cannot use Annotations on an IngressClass to configure load balancers with EKS Auto Mode. +* You cannot set link:elasticloadbalancing/latest/APIReference/API_ListenerAttribute.html["ListenerAttribute",type="documentation"] with EKS Auto Mode. * You must update the Cluster IAM Role to enable tag propagation from Kubernetes to {aws} Load Balancer resources. For more information, see <>. * For information about associating resources with either EKS Auto Mode or the self-managed {aws} Load Balancer Controller, see <>. * For information about fixing issues with load balancers, see <>. @@ -177,33 +261,34 @@ The following tables provide a detailed comparison of changes in IngressClassPar === IngressClassParams -[options="header"] |=== | Previous | New | Description + | `elbv2.k8s.aws/v1beta1` | `eks.amazonaws.com/v1` | API version change | `spec.certificateArn` | `spec.certificateARNs` | Support for multiple certificate ARNs | `spec.subnets.tags` | `spec.subnets.matchTags` | Changed subnet matching schema -| `spec.listeners.listenerAttributes` | `spec.listeners.attributes` | Simplified attribute naming +| `spec.listeners.listenerAttributes` | Not supported | Not yet supported by EKS Auto Mode |=== === Ingress annotations -[options="header"] |=== | Previous | New | Description + | `kubernetes.io/ingress.class` | Not supported | Use `spec.ingressClassName` on Ingress objects | `alb.ingress.kubernetes.io/group.name` | Not supported | Specify groups in IngressClass only | `alb.ingress.kubernetes.io/waf-acl-id` | Not supported | Use WAF v2 instead | `alb.ingress.kubernetes.io/web-acl-id` | Not supported | Use WAF v2 instead | `alb.ingress.kubernetes.io/shield-advanced-protection` | Not supported | Shield integration disabled +| `alb.ingress.kubernetes.io/auth-type: oidc` | Not supported | OIDC Auth Type is currently not supported |=== === TargetGroupBinding -[options="header"] |=== | Previous | New | Description + | `elbv2.k8s.aws/v1beta1` | `eks.amazonaws.com/v1` | API version change | `spec.targetType` optional | `spec.targetType` required | Explicit target type specification | `spec.networking.ingress.from` | Not supported | No longer supports NLB without security groups -|=== +|=== \ No newline at end of file diff --git a/latest/ug/automode/auto-configure-nlb.adoc b/latest/ug/automode/auto-configure-nlb.adoc index 916fc6934..2f1858290 100644 --- a/latest/ug/automode/auto-configure-nlb.adoc +++ b/latest/ug/automode/auto-configure-nlb.adoc @@ -1,17 +1,24 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[auto-configure-nlb,auto-configure-nlb.title]] +[#auto-configure-nlb] = Use Service Annotations to configure Network Load Balancers -:info_doctype: section :info_titleabbrev: Create service -include::../attributes.txt[] - Learn how to configure Network Load Balancers (NLB) in Amazon EKS using Kubernetes service annotations. This topic explains the annotations supported by EKS Auto Mode for customizing NLB behavior, including internet accessibility, health checks, SSL/TLS termination, and IP targeting modes. When you create a Kubernetes service of type `LoadBalancer` in EKS Auto Mode, EKS automatically provisions and configures an {aws} Network Load Balancer based on the annotations you specify. This declarative approach allows you to manage load balancer configurations directly through your Kubernetes manifests, maintaining infrastructure as code practices. -EKS Auto Mode handles Network Load Balancer provisioning by default for all services of type LoadBalancer - no additional controller installation or configuration is required. The `loadBalancerClass: eks.amazonaws.com/nlb `specification is automatically set as the cluster default, streamlining the deployment process while maintaining compatibility with existing Kubernetes workloads. +EKS Auto Mode handles Network Load Balancer provisioning by default for all services of type LoadBalancer - no additional controller installation or configuration is required. The `loadBalancerClass: eks.amazonaws.com/nlb` specification is automatically set as the cluster default, streamlining the deployment process while maintaining compatibility with existing Kubernetes workloads. + +[NOTE] +==== +EKS Auto Mode requires subnet tags to identify public and private subnets. + +If you created your cluster with `eksctl`, you already have these tags. + +Learn how to <>. +==== == Sample Service @@ -53,9 +60,12 @@ All of the following annotations need to be prefixed with `service.beta.kubernet [role="no-scroll"] -[cols="1,1,1", options="header"] +[%header,cols="3"] |=== -|Field |Description |Example +|Field +|Description +|Example + |`aws-load-balancer-type` |Specifies the load balancer type. Use `external` for new deployments. @@ -87,7 +97,7 @@ All of the following annotations need to be prefixed with `service.beta.kubernet |`aws-load-balancer-ssl-cert` |ARN of the SSL certificate from {aws} Certificate Manager for HTTPS/TLS. -|`arn:aws:acm:region:account:certificate/cert-id` +|`{arn-aws}acm:region:account:certificate/cert-id` |`aws-load-balancer-ssl-ports` |Specifies which ports should use SSL/TLS. @@ -104,6 +114,7 @@ All of the following annotations need to be prefixed with `service.beta.kubernet |`aws-load-balancer-ip-address-type` |Specifies whether the load balancer uses IPv4 or dual-stack (IPv4 + IPv6). |`ipv4` or `dualstack` + |=== == Considerations @@ -117,9 +128,9 @@ When migrating to EKS Auto Mode for load balancing, several changes in service a === Service annotations -[options="header"] |=== | Previous | New | Description + | `service.beta.kubernetes.io/load-balancer-source-ranges` | Not supported | Use `spec.loadBalancerSourceRanges` on Service | `service.beta.kubernetes.io/aws-load-balancer-type` | Not supported | Use `spec.loadBalancerClass` on Service | `service.beta.kubernetes.io/aws-load-balancer-internal` | Not supported | Use `service.beta.kubernetes.io/aws-load-balancer-scheme` @@ -134,22 +145,16 @@ When migrating to EKS Auto Mode for load balancing, several changes in service a To migrate from deprecated load balancer attribute annotations, consolidate these settings into the `service.beta.kubernetes.io/aws-load-balancer-attributes` annotation. This annotation accepts a comma-separated list of key-value pairs for various load balancer attributes. For example, to specify proxy protocol, access logging, and cross-zone load balancing, use the following format: ```yaml -service.beta.kubernetes.io/aws-load-balancer-attributes: | - proxy_protocol.v2.enabled=true - access_logs.s3.enabled=true - access_logs.s3.bucket=my-bucket - access_logs.s3.prefix=my-prefix - load_balancing.cross_zone.enabled=true - +service.beta.kubernetes.io/aws-load-balancer-attributes: access_logs.s3.enabled=true,access_logs.s3.bucket=my-bucket,access_logs.s3.prefix=my-prefix,load_balancing.cross_zone.enabled=true ``` This consolidated format provides a more consistent and flexible way to configure load balancer attributes while reducing the number of individual annotations needed. Review your existing Service configurations and update them to use this consolidated format. === TargetGroupBinding -[options="header"] |=== | Previous | New | Description + | `elbv2.k8s.aws/v1beta1` | `eks.amazonaws.com/v1` | API version change | `spec.targetType` optional | `spec.targetType` required | Explicit target type specification | `spec.networking.ingress.from` | Not supported | No longer supports NLB without security groups diff --git a/latest/ug/automode/auto-controls.adoc b/latest/ug/automode/auto-controls.adoc new file mode 100644 index 000000000..7d1ac679c --- /dev/null +++ b/latest/ug/automode/auto-controls.adoc @@ -0,0 +1,86 @@ +include::../attributes.txt[] + +[.topic] +[#auto-controls] += Update organization controls for EKS Auto Mode +:info_titleabbrev: Update AMI controls + +Some organization controls can prevent EKS Auto Mode from functioning correctly. If so, you must update these controls to allow EKS Auto Mode to have the permissions required to manage EC2 instances on your behalf. + +EKS Auto Mode uses a service role for launching the EC2 Instances that back EKS Auto Mode Nodes. A service role is an link:IAM/latest/UserGuide/id_roles.html[IAM role,type="documentation"] which is created in your account that a service assumes to perform actions on your behalf. link:organizations/latest/userguide/orgs_manage_policies_scps.html[Service Control Policies,type="documentation"] (SCPs) always apply to actions performed with service roles. This allows an SCP to inhibit Auto Mode's operations. The most common occurrence is when an SCP is used to restrict the Amazon Machine Images (AMIs) that can be launched. To allow EKS Auto Mode to function, modify the SCP to permit launching AMIs from EKS Auto Mode accounts. + +You can also use the link:AWSEC2/latest/UserGuide/ec2-allowed-amis.html[EC2 Allowed AMIs,type="documentation"] feature to limit the visibility of AMIs in other accounts. If you use this feature, you must expand the image criteria to also include the EKS Auto Mode AMI accounts in the regions of interest. + +== Example SCP to block all AMIs except for EKS Auto Mode AMIs + +The SCP below prevents calling `ec2:RunInstances` unless the AMI belongs to the EKS Auto Mode AMI account for us-west-2 or us-east-1. + +[NOTE] +==== +It's important *not* to use the `ec2:Owner` context key. Amazon owns the EKS Auto Mode AMI accounts and the value for this key will always be `amazon`. Constructing an SCP that allows launching AMIs if the `ec2:Owner` is `amazon` will allow launching any Amazon owned AMI, not just those for EKS Auto Mode.* +==== + +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:b3fc497a-510a-4dcf-8f20-f6a340f72dfd[] +---- + +== EKS Auto Mode AMI accounts + +{aws} accounts that vary by region host EKS Auto Mode public AMIs. + +|=== +|{aws} Region |Account +|af-south-1 |471112993317 +|ap-east-1 |590183728416 +|ap-northeast-1 |851725346105 +|ap-northeast-2 |992382805010 +|ap-northeast-3 |891377407544 +|ap-south-1 |975049899075 +|ap-south-2 |590183737426 +|ap-southeast-1 |339712723301 +|ap-southeast-2 |58264376476 +|ap-southeast-3 |471112941769 +|ap-southeast-4 |590183863144 +|ap-southeast-5 |654654202513 +|ap-southeast-7 |533267217478 +|ca-central-1 |992382439851 +|ca-west-1 |767397959864 +|eu-central-1 |891376953411 +|eu-central-2 |381492036002 +|eu-north-1 |339712696471 +|eu-south-1 |975049955519 +|eu-south-2 |471112620929 +|eu-west-1 |381492008532 +|eu-west-2 |590184142468 +|eu-west-3 |891376969258 +|il-central-1 |590183797093 +|me-central-1 |637423494195 +|me-south-1 |905418070398 +|mx-central-1 |211125506622 +|sa-east-1 |339712709251 +|us-east-1 |992382739861 +|us-east-2 |975050179949 +|us-west-1 |975050035094 +|us-west-2 |767397842682 +|=== + +== Associate Public IP address + +When `ec2:RunInstances` is called the `AssociatePublicIpAddress` field for an instance launch is determined automatically by the type of subnet that the instance is being launched into. +An SCP may be used to enforce that this value is explicitly set to false, regardless of the type of subnet being launched into. +In this case the NodeClass field `spec.advancedNetworking.associatePublicIPAddress` can also be set to false to satisfy the requirements of the SCP. + +```json + { + "Sid": "DenyPublicEC2IPAddesses", + "Effect": "Deny", + "Action": "ec2:RunInstances", + "Resource": "arn:aws:ec2:*:*:network-interface/*", + "Condition": { + "BoolIfExists": { + "ec2:AssociatePublicIpAddress": "true" + } + } + } +``` diff --git a/latest/ug/automode/auto-disable.adoc b/latest/ug/automode/auto-disable.adoc index ce6df4116..75500b5a0 100644 --- a/latest/ug/automode/auto-disable.adoc +++ b/latest/ug/automode/auto-disable.adoc @@ -1,10 +1,8 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[auto-disable,auto-disable.title]] +[#auto-disable] = Disable EKS Auto Mode -:info_doctype: section - -include::../attributes.txt[] You can disable EKS Auto Mode on an existing EKS Cluster. This is a destructive operation. @@ -21,7 +19,7 @@ Steps below describe how to remove a leaked Security Group if that should happen == Disable EKS Auto Mode ({aws} Console) -. Open your cluster overview page in the {aws} Management Console. +. Open your cluster overview page in the {aws-management-console}. . Under *EKS Auto Mode* select *Manage* . Toggle *EKS Auto Mode* to `off`. @@ -29,7 +27,7 @@ If any managed Security Group is not deleted at the end of this process, you can == Disable EKS Auto Mode ({aws} CLI) -Use the following command to disable {eam} on an existing cluster. +Use the following command to disable EKS Auto Mode on an existing cluster. You need to have the `aws` CLI installed, and be logged in with sufficent permissions to manage EKS clusters. For more information, see <>. @@ -60,4 +58,4 @@ To then delete the Security Group: [source,cli] ---- aws ec2 delete-security-group --group-name= ----- +---- \ No newline at end of file diff --git a/latest/ug/automode/auto-elb-example.adoc b/latest/ug/automode/auto-elb-example.adoc index 10edb9ed6..032868442 100644 --- a/latest/ug/automode/auto-elb-example.adoc +++ b/latest/ug/automode/auto-elb-example.adoc @@ -1,16 +1,9 @@ -//!!NODE_ROOT
+include::../attributes.txt[] [.topic] -[[auto-elb-example,auto-elb-example.title]] +[#auto-elb-example] = Deploy a Sample Load Balancer Workload to EKS Auto Mode -:info_doctype: section -:info_title: Deploy a sample load balancer workload to EKS Auto Mode -:info_titleabbrev: Deploy load balancer workload -:info_abstract: Deploy a sample load balancer workload to EKS Auto Mode - - -include::../attributes.txt[] - +:info_titleabbrev: Deploy load balancer This guide walks you through deploying a containerized version of the 2048 game on Amazon EKS, complete with load balancing and internet accessibility. @@ -75,7 +68,9 @@ spec: cpu: "0.5" ---- -**Key components:** +NOTE: If you receive an error loading the image `public.ecr.aws/l6m2t8p7/docker-2048:latest`, confirm your Node IAM role has sufficent permissions to pull images from ECR. For more information, see <>. Also, the `docker-2048` image in the example is an `x86_64` image and will not run on other architectures. + +*Key components:* - Deploys 5 replicas of the application - Uses a public ECR image @@ -107,12 +102,11 @@ spec: - port: 80 targetPort: 80 protocol: TCP - type: NodePort selector: app.kubernetes.io/name: app-2048 ---- -**Key components:** +*Key components:* - Creates a NodePort service - Maps port 80 to the container's port 80 @@ -136,7 +130,6 @@ First, create the `IngressClass`. Create a file named `04-ingressclass.yaml`: apiVersion: networking.k8s.io/v1 kind: IngressClass metadata: - namespace: game-2048 labels: app.kubernetes.io/name: LoadBalancerController name: alb @@ -144,6 +137,15 @@ spec: controller: eks.amazonaws.com/alb ---- +[NOTE] +==== +EKS Auto Mode requires subnet tags to identify public and private subnets. + +If you created your cluster with `eksctl`, you already have these tags. + +Learn how to <>. +==== + Then create the Ingress resource. Create a file named `05-ingress.yaml`: [source,yaml] @@ -170,7 +172,7 @@ spec: number: 80 ---- -**Key components:** +*Key components:* - Creates an internet-facing ALB - Uses IP target type for direct pod routing @@ -231,7 +233,7 @@ This will delete all resources in the namespace, including the deployment, servi ** Configures target groups for the pods ** Sets up routing rules to direct traffic to the service -[[auto-elb-troubleshooting,auto-elb-troubleshooting.title]] +[#auto-elb-troubleshooting] == Troubleshooting If the game doesn't load: diff --git a/latest/ug/automode/auto-enable-existing.adoc b/latest/ug/automode/auto-enable-existing.adoc index 1e0dab076..83a8cb78d 100644 --- a/latest/ug/automode/auto-enable-existing.adoc +++ b/latest/ug/automode/auto-enable-existing.adoc @@ -1,14 +1,9 @@ -//!!NODE_ROOT
+include::../attributes.txt[] [.topic] -[[auto-enable-existing,auto-enable-existing.title]] +[#auto-enable-existing] = Enable EKS Auto Mode on an existing cluster -:info_doctype: section -:info_title: Enable EKS Auto Mode on an existing cluster :info_titleabbrev: Enable on cluster -:info_abstract: Enable EKS Auto Mode on an existing cluster - -include::../attributes.txt[] This topic describes how to enable Amazon EKS Auto Mode on your existing Amazon EKS clusters. Enabling Auto Mode on an existing cluster requires updating IAM permissions and configuring core EKS Auto Mode settings. Once enabled, you can begin migrating your existing compute workloads to take advantage of Auto Mode's simplified operations and automated infrastructure management. @@ -18,20 +13,21 @@ Verify you have the minimum required version of certain Amazon EKS Add-ons insta ==== -Before you begin, ensure you have administrator access to your Amazon EKS cluster and permissions to modify IAM roles. The steps in this topic guide you through enabling Auto Mode using either the {aws} Management Console or {aws} CLI. +Before you begin, ensure you have administrator access to your Amazon EKS cluster and permissions to modify IAM roles. The steps in this topic guide you through enabling Auto Mode using either the {aws-management-console} or {aws} CLI. -== {aws} Management Console +[#auto-enable-existing-console] +== {aws-management-console} You must be logged into the {aws} console with permission to manage IAM, EKS, and EC2 resources. [NOTE] ==== -The Cluster IAM role of an EKS Cluster cannot be changed after the cluster is created. {eam} requires additional permissions on this role. You must attach additional policies to the current role. +The Cluster IAM role of an EKS Cluster cannot be changed after the cluster is created. EKS Auto Mode requires additional permissions on this role. You must attach additional policies to the current role. ==== -=== Update Cluster IAM Role +=== Update Cluster IAM role -. Open your cluster overview page in the {aws} Management Console. +. Open your cluster overview page in the {aws-management-console}. . Under *Cluster IAM role ARN*, select *View in IAM*. . From the *Add Permissions* dropdown, select *Attach Policies*. . Use the *Search* box to find and select the following policies: @@ -44,40 +40,26 @@ The Cluster IAM role of an EKS Cluster cannot be changed after the cluster is cr . From the *Trust relationships* tab, select *Edit trust policy* . Insert the following Cluster IAM Role trust policy, and select *Update policy* -[source,json] +[source,json,subs="verbatim,attributes"] ---- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Service": "eks.amazonaws.com" - }, - "Action": [ - "sts:AssumeRole", - "sts:TagSession" - ] - } - ] -} +include::iam_snippet:6352e330-4f08-4077-ba62-101ab3e4580f[] ---- -=== Enable {eam} +=== Enable EKS Auto Mode -. Open your cluster overview page in the {aws} Management Console. +. Open your cluster overview page in the {aws-management-console}. . Under *EKS Auto Mode* select *Manage* . Toggle *EKS Auto Mode* to on. . From the *EKS Node Pool* dropdown, select the default node pools you want to create. -** Learn more about Node Pools in {eam}. For more information, see <>. -. If you have previously created an {eam} Node IAM role this {aws} account, select it in the *Node IAM Role* dropdown. If you have not created this role before, select *Create {recd} Role* and follow the steps. +** Learn more about Node Pools in EKS Auto Mode. For more information, see <>. +. If you have previously created an EKS Auto Mode Node IAM role this {aws} account, select it in the *Node IAM Role* dropdown. If you have not created this role before, select *Create recommended Role* and follow the steps. == {aws} CLI === Prerequisites -* The Cluster IAM Role of the existing EKS Cluster must include sufficent permissiosn for {eam}, such as the following policies: +* The Cluster IAM Role of the existing EKS Cluster must include sufficent permissiosn for EKS Auto Mode, such as the following policies: ** `AmazonEKSComputePolicy` ** `AmazonEKSBlockStoragePolicy` ** `AmazonEKSLoadBalancingPolicy` @@ -88,7 +70,7 @@ The Cluster IAM role of an EKS Cluster cannot be changed after the cluster is cr === Procedure -Use the following commands to enable {eam} on an existing cluster. +Use the following commands to enable EKS Auto Mode on an existing cluster. [NOTE] ==== @@ -104,8 +86,8 @@ aws eks update-cluster-config \ --storage-config '{"blockStorage":{"enabled": true}}' ---- -[[auto-addons-required,auto-addons-required.title]] -== Required Add-on Versions +[#auto-addons-required] +== Required add-on versions If you're planning to enable EKS Auto Mode on an existing cluster, you may need to update certain add-ons. Please note: @@ -114,16 +96,17 @@ If you're planning to enable EKS Auto Mode on an existing cluster, you may need If you have any of the following add-ons installed, ensure they are at least at the specified minimum version: -[cols="1,1"] +[%header,cols="2"] |=== -| Add-on Name | Minimum Required Version +|Add-on name +|Minimum required version + -| Amazon VPC CNI plugin for Kubernetes -| v1.19.0-eksbuild.1 +|Amazon VPC CNI plugin for Kubernetes +|v1.19.0-eksbuild.1 -| Kube-proxy +|Kube-proxy a| -* v1.25.16-eksbuild.22 * v1.26.15-eksbuild.19 * v1.27.16-eksbuild.14 * v1.28.15-eksbuild.4 @@ -131,14 +114,14 @@ a| * v1.30.6-eksbuild.3 * v1.31.2-eksbuild.3 -| Amazon EBS CSI driver -| v1.37.0-eksbuild.1 +|Amazon EBS CSI driver +|v1.37.0-eksbuild.1 -| CSI snapshot controller -| v8.1.0-eksbuild.2 +|CSI snapshot controller +|v8.1.0-eksbuild.2 -| EKS Pod Identity Agent -| v1.3.4-eksbuild.1 +|EKS Pod Identity Agent +|v1.3.4-eksbuild.1 |=== @@ -147,4 +130,4 @@ For more information, see <>. == Next Steps * To migrate Manage Node Group workloads, see <>. -* To migrate from Self-Managed Karpenter, see <>. +* To migrate from Self-Managed Karpenter, see <>. \ No newline at end of file diff --git a/latest/ug/automode/auto-glossary.adoc b/latest/ug/automode/auto-glossary.adoc index d2bfcb7fb..9dac6db29 100644 --- a/latest/ug/automode/auto-glossary.adoc +++ b/latest/ug/automode/auto-glossary.adoc @@ -1,14 +1,9 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[auto-glossary,auto-glossary.title]] +[#auto-glossary] = Glossary -:info_doctype: section -:info_title: Glossary of terms for EKS Auto Mode :info_titleabbrev: Glossary -:info_abstract: Glossary of terms for EKS Auto Mode - - -include::../attributes.txt[] IAM Role:: An IAM identity that you can create in your {aws} account that has specific permissions. You can use IAM roles to delegate access to users, applications, or services that don't normally have access to your {aws} resources. @@ -74,4 +69,4 @@ EKS Auto Mode -- Block Storage Capability:: A feature that manages Amazon EBS volumes in Amazon EKS Auto Mode clusters. This capability automatically handles volume provisioning and lifecycle management for pods that need persistent storage. EKS Auto Mode -- Load Balancing Capability:: -A feature that manages Application Load Balancers and Network Load Balancers in Amazon EKS Auto Mode clusters. This capability automatically configures load balancers based on your service requirements. +A feature that manages Application Load Balancers and Network Load Balancers in Amazon EKS Auto Mode clusters. This capability automatically configures load balancers based on your service requirements. \ No newline at end of file diff --git a/latest/ug/automode/auto-kms.adoc b/latest/ug/automode/auto-kms.adoc new file mode 100644 index 000000000..92f33eaea --- /dev/null +++ b/latest/ug/automode/auto-kms.adoc @@ -0,0 +1,102 @@ +include::../attributes.txt[] + +[.topic] +[#auto-kms] += Enable EBS Volume Encryption with Customer Managed KMS Keys for EKS Auto Mode +:info_titleabbrev: Encrypt root volumes + +You can encrypt the ephemeral root volume for EKS Auto Mode instances with a customer managed KMS key. + +Amazon EKS Auto Mode uses service-linked roles to delegate permissions to other {aws} services when managing encrypted EBS volumes for your Kubernetes clusters. This topic describes how to set up the key policy that you need when specifying a customer managed key for Amazon EBS encryption with EKS Auto Mode. + +Considerations: + +* EKS Auto Mode does not need additional authorization to use the default {aws} managed key to protect the encrypted volumes in your account. +* This topic covers encrypting ephemeral volumes, the root volumes for EC2 instances. For more information about encrypting data volumes used for workloads, see <>. + +== Overview + +The following {aws} KMS keys can be used for Amazon EBS root volume encryption when EKS Auto Mode launches instances: + +* *{aws} managed key* – An encryption key in your account that Amazon EBS creates, owns, and manages. This is the default encryption key for a new account. + +* *Customer managed key* – A custom encryption key that you create, own, and manage. + +[NOTE] +==== +The key must be symmetric. Amazon EBS does not support asymmetric customer managed keys. +==== + +== Step 1: Configure the key policy + +Your KMS keys must have a key policy that allows EKS Auto Mode to launch instances with Amazon EBS volumes encrypted with a customer managed key. + +Configure your key policy with the following structure: + + +[NOTE] +==== +This policy only includes permissions for EKS Auto Mode. The key policy may need additional permissions if other identities need to use the key or manage grants. +==== + +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:c0f99b58-37c6-4e50-b643-48423c997b6d[] +---- + +Make sure to replace `` with your actual {aws} account ID. + +When configuring the key policy: + +* The `ClusterServiceRole` must have the necessary IAM permissions to use the KMS key for encryption operations +* The `kms:GrantIsForAWSResource` condition ensures that grants can only be created for {aws} services + +== Step 2: Configure NodeClass with your customer managed key + +After configuring the key policy, reference the KMS key in your EKS Auto Mode NodeClass configuration: + +[source,yaml,subs="verbatim,attributes"] +---- +apiVersion: eks.amazonaws.com/v1 +kind: NodeClass +metadata: + name: my-node-class +spec: + # Insert existing configuration + + ephemeralStorage: + size: "80Gi" # Range: 1-59000Gi or 1-64000G or 1-58Ti or 1-64T + iops: 3000 # Range: 3000-16000 + throughput: 125 # Range: 125-1000 + + # KMS key for encryption + kmsKeyID: "{arn-aws}kms:::key/" +---- + +Replace the placeholder values with your actual values: + +* `` with your {aws} region +* `` with your {aws} account ID +* `` with your KMS key ID + +You can specify the KMS key using any of the following formats: + +* KMS Key ID: `1a2b3c4d-5e6f-1a2b-3c4d-5e6f1a2b3c4d` +* KMS Key ARN: `{arn-aws}kms:us-west-2:111122223333:key/1a2b3c4d-5e6f-1a2b-3c4d-5e6f1a2b3c4d` +* Key Alias Name: `alias/eks-auto-mode-key` +* Key Alias ARN: `{arn-aws}kms:us-west-2:111122223333:alias/eks-auto-mode-key` + +Apply the NodeClass configuration using kubectl: + +[source,bash] +---- +kubectl apply -f nodeclass.yaml +---- + +== Related Resources + +* xref:create-node-class[Create a Node Class for Amazon EKS] +* View more information in the {aws} Key Management Service Developer Guide +** https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-services.html[Permissions for {aws} services in key policies] +** https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-modifying.html[Change a key policy] +** https://docs.aws.amazon.com/kms/latest/developerguide/grants.html[Grants in {aws} KMS] diff --git a/latest/ug/automode/auto-learn-iam.adoc b/latest/ug/automode/auto-learn-iam.adoc index 020547d0d..d1b63ce9f 100644 --- a/latest/ug/automode/auto-learn-iam.adoc +++ b/latest/ug/automode/auto-learn-iam.adoc @@ -1,13 +1,9 @@ -//!!NODE_ROOT
+include::../attributes.txt[] [.topic] -[[auto-learn-iam,auto-learn-iam.title]] +[#auto-learn-iam] = Learn about identity and access in EKS Auto Mode -:info_titleabbrev: Identity & access -:info_doctype: section - -include::../attributes.txt[] - +:info_titleabbrev: Identity and access This topic describes the Identity and Access Management (IAM) roles and permissions required to use EKS Auto Mode. EKS Auto Mode uses two primary IAM roles: a Cluster IAM Role and a Node IAM Role. These roles work in conjunction with EKS Pod Identity and EKS access entries to provide comprehensive access management for your EKS clusters. @@ -15,17 +11,17 @@ When you configure EKS Auto Mode, you will need to set up these IAM roles with s In EKS Auto Mode, {aws} IAM roles are automatically mapped to Kubernetes permissions through EKS access entries, removing the need for manual configuration of `aws-auth` ConfigMaps or custom bindings. When you create a new auto mode cluster, EKS automatically creates the corresponding Kubernetes permissions using Access entries, ensuring that {aws} services and cluster components have the appropriate access levels within both the {aws} and Kubernetes authorization systems. This automated integration reduces configuration complexity and helps prevent permission-related issues that commonly occur when managing EKS clusters. -[[auto-learn-cluster-iam-role,auto-learn-cluster-iam-role.title]] +[#auto-learn-cluster-iam-role] == Cluster IAM role The Cluster IAM role is an {aws} Identity and Access Management (IAM) role used by Amazon EKS to manage permissions for Kubernetes clusters. This role grants Amazon EKS the necessary permissions to interact with other {aws} services on behalf of your cluster, and is automatically configured with Kubernetes permissions using EKS access entries. * You must attach {aws} IAM policies to this role. -* {eam} attaches Kubernetes permissions to this role automatically using EKS access entries. -* With {eam}, {aws} suggests creating a single Cluster IAM Role per {aws} account. +* EKS Auto Mode attaches Kubernetes permissions to this role automatically using EKS access entries. +* With EKS Auto Mode, {aws} suggests creating a single Cluster IAM Role per {aws} account. * {aws} suggests naming this role `AmazonEKSAutoClusterRole`. * This role requires permissions for multiple {aws} services to manage resources including EBS volumes, Elastic Load Balancers, and EC2 instances. -* The suggested configuration for this role includes multiple {aws} managed IAM policies, related to the different {caps} of {eam}. +* The suggested configuration for this role includes multiple {aws} managed IAM policies, related to the different capabilities of EKS Auto Mode. ** `AmazonEKSComputePolicy` ** `AmazonEKSBlockStoragePolicy` ** `AmazonEKSLoadBalancingPolicy` @@ -41,15 +37,15 @@ For more information about Kubernetes access, see: * <> -[[auto-learn-node-iam-role,auto-learn-node-iam-role.title]] +[#auto-learn-node-iam-role] == Node IAM role The Node IAM role is an {aws} Identity and Access Management (IAM) role used by Amazon EKS to manage permissions for worker nodes in Kubernetes clusters. This role grants EC2 instances running as Kubernetes nodes the necessary permissions to interact with {aws} services and resources, and is automatically configured with Kubernetes RBAC permissions using EKS access entries. * You must attach {aws} IAM policies to this role. -* {eam} attaches Kubernetes RBAC permissions to this role automatically using EKS access entries. +* EKS Auto Mode attaches Kubernetes RBAC permissions to this role automatically using EKS access entries. * {aws} suggests naming this role `AmazonEKSAutoNodeRole`. -* With {eam}, {aws} suggests creating a single Node IAM Role per {aws} account. +* With EKS Auto Mode, {aws} suggests creating a single Node IAM Role per {aws} account. * This role has limited permissions. The key permissions include assuming a Pod Identity Role, and pulling images from ECR. * {aws} suggests the following {aws} managed IAM policies: ** `AmazonEKSWorkerNodeMinimalPolicy` @@ -78,119 +74,22 @@ For more information, see: * <> -[[tag-prop,tag-prop.title]] +[#tag-prop] == Custom {aws} tags for EKS Auto resources By default, the managed policies related to EKS Auto Mode do not permit applying user defined tags to Auto Mode provisioned {aws} resources. If you want to apply user defined tags to {aws} resources, you must attach additional permissions to the Cluster IAM Role with sufficient permissions to create and modify tags on {aws} resources. Below is an example of a policy that will allow unrestricted tagging access: -[[auto-tag-policy,auto-tag-policy.title]] +[#auto-tag-policy] .View custom tag policy example [%collapsible, expand-section="_collapse_all_"] ==== -[source,json] +[source,json,subs="verbatim,attributes"] ---- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "Compute", - "Effect": "Allow", - "Action": [ - "ec2:CreateFleet", - "ec2:RunInstances", - "ec2:CreateLaunchTemplate" - ], - "Resource": "*", - "Condition": { - "StringEquals": { - "aws:RequestTag/eks:eks-cluster-name": "${aws:PrincipalTag/eks:eks-cluster-name}" - }, - "StringLike": { - "aws:RequestTag/eks:kubernetes-node-class-name": "*", - "aws:RequestTag/eks:kubernetes-node-pool-name": "*" - } - } - }, - { - "Sid": "Storage", - "Effect": "Allow", - "Action": [ - "ec2:CreateVolume", - "ec2:CreateSnapshot" - ], - "Resource": [ - "arn:aws:ec2:*:*:volume/*", - "arn:aws:ec2:*:*:snapshot/*" - ], - "Condition": { - "StringEquals": { - "aws:RequestTag/eks:eks-cluster-name": "${aws:PrincipalTag/eks:eks-cluster-name}" - } - } - }, - { - "Sid": "Networking", - "Effect": "Allow", - "Action": "ec2:CreateNetworkInterface", - "Resource": "*", - "Condition": { - "StringEquals": { - "aws:RequestTag/eks:eks-cluster-name": "${aws:PrincipalTag/eks:eks-cluster-name}" - }, - "StringLike": { - "aws:RequestTag/eks:kubernetes-cni-node-name": "*" - } - } - }, - { - "Sid": "LoadBalancer", - "Effect": "Allow", - "Action": [ - "elasticloadbalancing:CreateLoadBalancer", - "elasticloadbalancing:CreateTargetGroup", - "elasticloadbalancing:CreateListener", - "elasticloadbalancing:CreateRule", - "ec2:CreateSecurityGroup" - ], - "Resource": "*", - "Condition": { - "StringEquals": { - "aws:RequestTag/eks:eks-cluster-name": "${aws:PrincipalTag/eks:eks-cluster-name}" - } - } - }, - { - "Sid": "ShieldProtection", - "Effect": "Allow", - "Action": [ - "shield:CreateProtection" - ], - "Resource": "*", - "Condition": { - "StringEquals": { - "aws:RequestTag/eks:eks-cluster-name": "${aws:PrincipalTag/eks:eks-cluster-name}" - } - } - }, - { - "Sid": "ShieldTagResource", - "Effect": "Allow", - "Action": [ - "shield:TagResource" - ], - "Resource": "arn:aws:shield::*:protection/*", - "Condition": { - "StringEquals": { - "aws:RequestTag/eks:eks-cluster-name": "${aws:PrincipalTag/eks:eks-cluster-name}" - } - } - } - ] -} +include::iam_snippet:a9e17b2c-8295-44b9-ac98-e7bbc0d9a0de[] ---- ==== == Access Policy Reference -For more information about the Kubernetes permissions used by EKS Auto Mode, see <>. +For more information about the Kubernetes permissions used by EKS Auto Mode, see <>. \ No newline at end of file diff --git a/latest/ug/automode/auto-local-zone.adoc b/latest/ug/automode/auto-local-zone.adoc new file mode 100644 index 000000000..d7fc83f0c --- /dev/null +++ b/latest/ug/automode/auto-local-zone.adoc @@ -0,0 +1,155 @@ +include::../attributes.txt[] + +[.topic] +[#auto-local-zone] += Deploy EKS Auto Mode nodes onto Local Zones +:info_titleabbrev: Use Local Zones + +EKS Auto Mode provides simplified cluster management with automatic node provisioning. {aws} Local Zones extend {aws} infrastructure to geographic locations closer to your end users, reducing latency for latency-sensitive applications. This guide walks you through the process of deploying EKS Auto Mode nodes onto {aws} Local Zones, enabling you to run containerized applications with lower latency for users in specific geographic areas. + +This guide also demonstrates how to use Kubernetes taints and tolerations to ensure that only specific workloads run on your Local Zone nodes, helping you control costs and optimize resource usage. + +== Prerequisites + +Before you begin deploying EKS Auto Mode nodes onto Local Zones, ensure you have the following prerequisites in place: + +* <> +* link:local-zones/latest/ug/getting-started.html#getting-started-find-local-zone[Opted-in to local zone in your {aws} account,type="documentation"] + +== Step 1: Create Local Zone Subnet + +The first step in deploying EKS Auto Mode nodes to a Local Zone is creating a subnet in that Local Zone. This subnet provides the network infrastructure for your nodes and allows them to communicate with the rest of your VPC. Follow the link:local-zones/latest/ug/getting-started.html#getting-started-create-local-zone-subnet[Create a Local Zone subnet,type="documentation"] instructions (in the {aws} Local Zones User Guide) to create a subnet in your chosen Local Zone. + +TIP: Make a note of the name of your local zone subnet. + +== Step 2: Create NodeClass for Local Zone Subnet + +After creating your Local Zone subnet, you need to define a NodeClass that references this subnet. The NodeClass is a Kubernetes custom resource that specifies the infrastructure attributes for your nodes, including which subnets, security groups, and storage configurations to use. In the example below, we create a NodeClass called "local-zone" that targets a local zone subnet based on its name. You can also use the subnet ID. You'll need to adapt this configuration to target your Local Zone subnet. + +For more information, see <>. + +```yaml +apiVersion: eks.amazonaws.com/v1 +kind: NodeClass +metadata: + name: local-zone +spec: + subnetSelectorTerms: + - id: +``` + +== Step 3: Create NodePool with NodeClass and Taint + +With your NodeClass configured, you now need to create a NodePool that uses this NodeClass. A NodePool defines the compute characteristics of your nodes, including instance types. The NodePool uses the NodeClass as a reference to determine where to launch instances. + +In the example below, we create a NodePool that references our "local-zone" NodeClass. We also add a taint to the nodes to ensure that only pods with a matching toleration can be scheduled on these Local Zone nodes. This is particularly important for Local Zone nodes, which typically have higher costs and should only be used by workloads that specifically benefit from the reduced latency. + +For more information, see <>. + + +```yaml +apiVersion: karpenter.sh/v1 +kind: NodePool +metadata: + name: my-node-pool +spec: + template: + metadata: + labels: + node-type: local-zone + spec: + nodeClassRef: + group: eks.amazonaws.com + kind: NodeClass + name: local-zone + taints: + - key: "aws.amazon.com/local-zone" + value: "true" + effect: NoSchedule + + requirements: + - key: "eks.amazonaws.com/instance-category" + operator: In + values: ["c", "m", "r"] + - key: "eks.amazonaws.com/instance-cpu" + operator: In + values: ["4", "8", "16", "32"] +``` + +The taint with key `aws.amazon.com/local-zone` and effect `NoSchedule` ensures that pods without a matching toleration won't be scheduled on these nodes. This prevents regular workloads from accidentally running in the Local Zone, which could lead to unexpected costs. + +== Step 4: Deploy Workloads with Toleration and Node Affinity + +For optimal control over workload placement on Local Zone nodes, use both taints/tolerations and node affinity together. This combined approach provides the following benefits: + +. **Cost Control**: The taint ensures that only pods with explicit tolerations can use potentially expensive Local Zone resources. +. **Guaranteed Placement**: Node affinity ensures that your latency-sensitive applications run exclusively in the Local Zone, not on regular cluster nodes. + +Here's an example of a Deployment configured to run specifically on Local Zone nodes: + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: low-latency-app + namespace: default +spec: + replicas: 2 + selector: + matchLabels: + app: low-latency-app + template: + metadata: + labels: + app: low-latency-app + spec: + tolerations: + - key: "aws.amazon.com/local-zone" + operator: "Equal" + value: "true" + effect: "NoSchedule" + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "node-type" + operator: "In" + values: ["local-zone"] + containers: + - name: application + image: my-low-latency-app:latest + resources: + limits: + cpu: "1" + memory: "1Gi" + requests: + cpu: "500m" + memory: "512Mi" +``` + +This Deployment has two key scheduling configurations: + +. The **toleration** allows the pods to be scheduled on nodes with the `aws.amazon.com/local-zone` taint. +. The **node affinity** requirement ensures these pods will only run on nodes with the label `node-type: local-zone`. + +Together, these ensure that your latency-sensitive application runs only on Local Zone nodes, and regular applications don't consume the Local Zone resources unless explicitly configured to do so. + +== Step 5: Verify with {aws} Console + +After setting up your NodeClass, NodePool, and Deployments, you should verify that nodes are being provisioned in your Local Zone as expected and that your workloads are running on them. You can use the {aws} Management Console to verify that EC2 instances are being launched in the correct Local Zone subnet. + +Additionally, you can check the Kubernetes node list using `kubectl get nodes -o wide` to confirm that the nodes are joining your cluster with the correct labels and taints: + +```bash +kubectl get nodes -o wide +kubectl describe node | grep -A 5 Taints +``` + +You can also verify that your workload pods are scheduled on the Local Zone nodes: + +```bash +kubectl get pods -o wide +``` + +This approach ensures that only workloads that specifically tolerate the Local Zone taint will be scheduled on these nodes, helping you control costs and make the most efficient use of your Local Zone resources. diff --git a/latest/ug/automode/auto-migrate-fargate.adoc b/latest/ug/automode/auto-migrate-fargate.adoc new file mode 100644 index 000000000..fdc1e0d21 --- /dev/null +++ b/latest/ug/automode/auto-migrate-fargate.adoc @@ -0,0 +1,241 @@ +include::../attributes.txt[] + +[.topic] +[#auto-migrate-fargate] += Migrate from EKS Fargate to EKS Auto Mode +:info_titleabbrev: Migrate from Fargate + +This topic walks you through the process of migrating workloads from EKS Fargate to Amazon EKS Auto Mode using `kubectl`. +The migration can be performed gradually, allowing you to move workloads at your own pace while maintaining cluster stability and application availability throughout the transition. + +The step-by-step approach outlined below enables you to run EKS Fargate and EKS Auto Mode side by side during the migration period. +This dual-operation strategy helps ensure a smooth transition by allowing you to validate workload behavior on EKS Auto Mode before completely decommissioning EKS Fargate. +You can migrate applications individually or in groups, providing flexibility to accommodate your specific operational requirements and risk tolerance. + +## Comparing Amazon EKS Auto Mode and EKS with {aws} Fargate + +Amazon EKS with {aws} Fargate remains an option for customers who want to run EKS, but Amazon EKS Auto Mode is the recommended approach moving forward. +EKS Auto Mode is fully Kubernetes conformant, supporting all upstream Kubernetes primitives and platform tools like Istio, which Fargate is unable to support. +EKS Auto Mode also fully supports all EC2 runtime purchase options, including GPU and Spot instances, enabling customers to leverage negotiated EC2 discounts and other savings mechanisms +These capabilities are not available when using EKS with Fargate. + +Furthermore, EKS Auto Mode allows customers to achieve the same isolation model as Fargate, using standard Kubernetes scheduling capabilities to ensure each EC2 instance runs a single application container. +By adopting Amazon EKS Auto Mode, customers can unlock the full benefits of running Kubernetes on {aws} — a fully Kubernetes-conformant platform that provides the flexibility to leverage the entire breadth of EC2 and purchasing options while retaining the ease of use and abstraction from infrastructure management that Fargate provides. + +## Prerequisites + +Before beginning the migration, ensure you have + +* Set up a cluster with Fargate. For more information, see <>. + +* Installed and connected `kubectl` to your cluster. For more information, see <>. + +## Step 1: Check the Fargate cluster + +. Check if the EKS cluster with Fargate is running: ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get node +---- ++ +[source,subs="verbatim,attributes"] +---- +NAME STATUS ROLES AGE VERSION +fargate-ip-192-168-92-52.ec2.internal Ready 25m v1.30.8-eks-2d5f260 +fargate-ip-192-168-98-196.ec2.internal Ready 24m v1.30.8-eks-2d5f260 +---- +. Check running pods: ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get pod -A +---- ++ +[source,subs="verbatim,attributes"] +---- +NAMESPACE NAME READY STATUS RESTARTS AGE +kube-system coredns-6659cb98f6-gxpjz 1/1 Running 0 26m +kube-system coredns-6659cb98f6-gzzsx 1/1 Running 0 26m +---- +. Create a deployment in a file called `deployment_fargate.yaml`: ++ +[source,bash,subs="verbatim,attributes"] +---- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment + labels: + app: nginx +spec: + replicas: 3 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + annotations: + eks.amazonaws.com/compute-type: fargate + spec: + containers: + - name: nginx + image: nginx + ports: + - containerPort: 80 +---- +. Apply the deployment: ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl apply -f deployment_fargate.yaml +---- ++ +[source,subs="verbatim,attributes"] +---- +deployment.apps/nginx-deployment created +---- +. Check the pods and deployments: ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get pod,deploy +---- ++ +[source,subs="verbatim,attributes"] +---- +NAME READY STATUS RESTARTS AGE +pod/nginx-deployment-5c7479459b-6trtm 1/1 Running 0 61s +pod/nginx-deployment-5c7479459b-g8ssb 1/1 Running 0 61s +pod/nginx-deployment-5c7479459b-mq4mf 1/1 Running 0 61s + +NAME READY UP-TO-DATE AVAILABLE AGE +deployment.apps/nginx-deployment 3/3 3 3 61s +---- +. Check the node: ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get node -owide +---- ++ +[source,subs="verbatim,attributes"] +---- +NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME +fargate-ip-192-168-111-43.ec2.internal Ready 31s v1.30.8-eks-2d5f260 192.168.111.43 Amazon Linux 2 5.10.234-225.910.amzn2.x86_64 containerd://1.7.25 +fargate-ip-192-168-117-130.ec2.internal Ready 36s v1.30.8-eks-2d5f260 192.168.117.130 Amazon Linux 2 5.10.234-225.910.amzn2.x86_64 containerd://1.7.25 +fargate-ip-192-168-74-140.ec2.internal Ready 36s v1.30.8-eks-2d5f260 192.168.74.140 Amazon Linux 2 5.10.234-225.910.amzn2.x86_64 containerd://1.7.25 +---- + +## Step 2: Enable EKS Auto Mode on the cluster + +. Enable EKS Auto Mode on your existing cluster using the {aws} CLI or Management Console. For more information, see <>. +. Check the nodepool: ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get nodepool +---- ++ +[source,subs="verbatim,attributes"] +---- +NAME NODECLASS NODES READY AGE +general-purpose default 1 True 6m58s +system default 0 True 3d14h +---- + +## Step 3: Update workloads for migration + +Identify and update the workloads you want to migrate to EKS Auto Mode. + +To migrate a workload from Fargate to EKS Auto Mode, apply the annotation `eks.amazonaws.com/compute-type: ec2`. +This ensures that the workload will not be scheduled by Fargate, despite the Fargate profile, +and will be caught up by the EKS Auto Mode NodePool. +For more information, see <>. + +. Modify your deployments (for example, the `deployment_fargate.yaml` file) to change the compute type to `ec2`: ++ +[source,bash,subs="verbatim,attributes"] +---- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment + labels: + app: nginx +spec: + replicas: 3 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + annotations: + eks.amazonaws.com/compute-type: ec2 + spec: + containers: + - name: nginx + image: nginx + ports: + - containerPort: 80 +---- +. Apply the deployment. This change allows the workload to be scheduled on the new EKS Auto Mode nodes: ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl apply -f deployment_fargate.yaml +---- +. Check that the deployment is running in the EKS Auto Mode cluster: ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get pod -o wide +---- ++ +[source,subs="verbatim,attributes"] +---- +NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES +nginx-deployment-97967b68d-ffxxh 1/1 Running 0 3m31s 192.168.43.240 i-0845aafcb51630ffb +nginx-deployment-97967b68d-mbcgj 1/1 Running 0 2m37s 192.168.43.241 i-0845aafcb51630ffb +nginx-deployment-97967b68d-qpd8x 1/1 Running 0 2m35s 192.168.43.242 i-0845aafcb51630ffb +---- +. Verify there is no Fargate node running and deployment running in the EKS Auto Mode managed nodes: ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get node -owide +---- ++ +[source,subs="verbatim,attributes"] +---- +NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME +i-0845aafcb51630ffb Ready 3m30s v1.30.8-eks-3c20087 192.168.41.125 3.81.118.95 Bottlerocket (EKS Auto) 2025.3.14 (aws-k8s-1.30) 6.1.129 containerd://1.7.25+bottlerocket +---- + +## Step 4: Gradually migrate workloads + +Repeat Step 3 for each workload you want to migrate. +This allows you to move workloads individually or in groups, based on your requirements and risk tolerance. + +## Step 5: Remove the original fargate profile + +Once all workloads have been migrated, you can remove the original `fargate` profile. +Replace [.replaceable]`` with the name of your Fargate profile: + +[source,bash,subs="verbatim,attributes"] +---- +aws eks delete-fargate-profile --cluster-name eks-fargate-demo-cluster --fargate-profile-name +---- + +## Step 6: Scale down CoreDNS + +Because EKS Auto mode handles CoreDNS, you scale the `coredns` deployment down to 0: + +[source,bash,subs="verbatim,attributes"] +---- +kubectl scale deployment coredns -n kube-system —-replicas=0 +---- diff --git a/latest/ug/automode/auto-migrate-karpenter.adoc b/latest/ug/automode/auto-migrate-karpenter.adoc index 081d52538..bb3bdc530 100644 --- a/latest/ug/automode/auto-migrate-karpenter.adoc +++ b/latest/ug/automode/auto-migrate-karpenter.adoc @@ -1,19 +1,15 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[auto-migrate-karpenter,auto-migrate-karpenter.title]] +[#auto-migrate-karpenter] = Migrate from Karpenter to EKS Auto Mode using kubectl -:info_doctype: section -:info_title: Migrate from Karpenter to EKS Auto Mode using kubectl :info_titleabbrev: Migrate from Karpenter -:info_abstract: Migrate from Karpenter to EKS Auto Mode using kubectl - -include::../attributes.txt[] This topic walks you through the process of migrating workloads from Karpenter to Amazon EKS Auto Mode using kubectl. The migration can be performed gradually, allowing you to move workloads at your own pace while maintaining cluster stability and application availability throughout the transition. The step-by-step approach outlined below enables you to run Karpenter and EKS Auto Mode side by side during the migration period. This dual-operation strategy helps ensure a smooth transition by allowing you to validate workload behavior on EKS Auto Mode before completely decommissioning Karpenter. You can migrate applications individually or in groups, providing flexibility to accommodate your specific operational requirements and risk tolerance. -## Prerequisites +== Prerequisites Before beginning the migration, ensure you have: @@ -22,7 +18,7 @@ Before beginning the migration, ensure you have: This topic assumes you are familiar with Karpenter and NodePools. For more information, see the https://karpenter.sh/[Karpenter Documentation.] -## Step 1: Enable EKS Auto Mode on the cluster +== Step 1: Enable EKS Auto Mode on the cluster Enable EKS Auto Mode on your existing cluster using the {aws} CLI or Management Console. For more information, see <>. @@ -34,7 +30,7 @@ For more information, see <>. ==== -## Step 2: Create a tainted EKS Auto Mode NodePool +== Step 2: Create a tainted EKS Auto Mode NodePool Create a new NodePool for EKS Auto Mode with a taint. This ensures that existing pods won't automatically schedule on the new EKS Auto Mode nodes. This node pool uses the `default` `NodeClass` built into EKS Auto Mode. For more information, see <>. @@ -61,9 +57,9 @@ spec: effect: "NoSchedule" ``` -Update the requirements for the node pool to match the Karpenter configuration you are migrating form. You need at least one requirement. +Update the requirements for the node pool to match the Karpenter configuration you are migrating from. You need at least one requirement. -## Step 3: Update workloads for migration +== Step 3: Update workloads for migration Identify and update the workloads you want to migrate to EKS Auto Mode. Add both tolerations and node selectors to these workloads: @@ -84,11 +80,11 @@ This change allows the workload to be scheduled on the new EKS Auto Mode nodes. EKS Auto Mode uses different labels than Karpenter. Labels related to EC2 managed instances start with `eks.amazonaws.com`. For more information, see <>. -## Step 4: Gradually migrate workloads +== Step 4: Gradually migrate workloads Repeat Step 3 for each workload you want to migrate. This allows you to move workloads individually or in groups, based on your requirements and risk tolerance. -## Step 5: Remove the original Karpenter NodePool +== Step 5: Remove the original Karpenter NodePool Once all workloads have been migrated, you can remove the original Karpenter NodePool: @@ -96,7 +92,7 @@ Once all workloads have been migrated, you can remove the original Karpenter Nod kubectl delete nodepool ``` -## Step 6: Remove taint from EKS Auto Mode NodePool (Optional) +== Step 6: Remove taint from EKS Auto Mode NodePool (Optional) If you want EKS Auto Mode to become the default for new workloads, you can remove the taint from the EKS Auto Mode NodePool: @@ -115,7 +111,7 @@ spec: # Remove the taints section ``` -## Step 7: Remove node selectors from workloads (Optional) +== Step 7: Remove node selectors from workloads (Optional) If you've removed the taint from the EKS Auto Mode NodePool, you can optionally remove the node selectors from your workloads, as EKS Auto Mode is now the default: @@ -131,7 +127,6 @@ spec: effect: "NoSchedule" ``` -## Step 8: Uninstall Karpenter from your cluster - -The steps to remove Karpenter depend on how you installed it. For more information, see the https://karpenter.sh/docs/getting-started/getting-started-with-karpenter/#create-a-cluster-and-add-karpenter[Karpenter install instructions] and the https://helm.sh/docs/helm/helm_uninstall/[Helm Uninstall command]. +== Step 8: Uninstall Karpenter from your cluster +The steps to remove Karpenter depend on how you installed it. For more information, see the https://karpenter.sh/docs/getting-started/getting-started-with-karpenter/#create-a-cluster-and-add-karpenter[Karpenter install instructions]. \ No newline at end of file diff --git a/latest/ug/automode/auto-migrate-mng.adoc b/latest/ug/automode/auto-migrate-mng.adoc index 10fe6d8be..2ac24c0d3 100644 --- a/latest/ug/automode/auto-migrate-mng.adoc +++ b/latest/ug/automode/auto-migrate-mng.adoc @@ -1,30 +1,29 @@ -//!!NODE_ROOT
+include::../attributes.txt[] [.topic] -[[auto-migrate-mng,auto-migrate-mng.title]] +[#auto-migrate-mng] = Migrate from EKS Managed Node Groups to EKS Auto Mode -:info_doctype: section -:info_title: Migrate from EKS Managed Node Groups to EKS Auto Mode -:info_titleabbrev: Migrate from Managed Node Groups -:info_abstract: Migrate from EKS Managed Node Groups - -include::../attributes.txt[] +:info_titleabbrev: Migrate from MNGs -When transitioning your Amazon EKS cluster to use EKS auto mode, you can smoothly migrate your existing workloads from managed node groups using the eksctl CLI tool. This process ensures continuous application availability while EKS auto mode optimizes your compute resources. The migration can be performed with minimal disruption to your running applications. +When transitioning your Amazon EKS cluster to use EKS auto mode, you can smoothly migrate your existing workloads from managed node groups (MNGs) using the eksctl CLI tool. This process ensures continuous application availability while EKS auto mode optimizes your compute resources. The migration can be performed with minimal disruption to your running applications. This topic walks you through the steps to safely drain pods from your existing managed node groups and allow EKS auto mode to reschedule them on newly provisioned instances. By following this procedure, you can take advantage of EKS auto mode's intelligent workload consolidation while maintaining your application's availability throughout the migration. == Prerequisites -* Cluster with {eam} enabled +* Cluster with EKS Auto Mode enabled * `eksctl` CLI installed and connected to your cluster. For more information, see <>. * Karpenter is not installed on the cluster. == Procedure -Use the following `eksctl` CLI command to initiate draining pods from the existing {mng} instances. {eam} will create new nodes to back the displaced pods. +Use the following `eksctl` CLI command to initiate draining pods from the existing managed node group instances. EKS Auto Mode will create new nodes to back the displaced pods. [source,cli] ---- -eksctl update auto-mode-config --drain-all-nodegroups +eksctl delete nodegroup --cluster= --name= ---- + +You will need to run this command for each managed node group in your cluster. + +For more information on this command, see https://eksctl.io/usage/nodegroups/#deleting-and-draining-nodegroups[Deleting and draining nodegroups] in the eksctl docs. \ No newline at end of file diff --git a/latest/ug/automode/auto-mng.adoc b/latest/ug/automode/auto-mng.adoc index 34af935c3..4ccb66259 100644 --- a/latest/ug/automode/auto-mng.adoc +++ b/latest/ug/automode/auto-mng.adoc @@ -1,26 +1,21 @@ +include::../attributes.txt[] -//!!NODE_ROOT
[.topic] -[[auto-mng,auto-mng.title]] +[#auto-mng] = Compare EKS Auto Mode with EKS managed node groups -:info_doctype: section -:info_title: Compare EKS Auto Mode with EKS managed node groups -:info_titleabbrev: Compare with Managed Node Groups -:info_abstract: Compare EKS Auto Mode with EKS managed node groups - - -include::../attributes.txt[] +:info_titleabbrev: Compare with managed node groups [IMPORTANT] ==== *{aws} Internal:* The table below will be used to update the larger table at <> ==== - - -[cols="3*", options="header"] +[%header,cols="3"] |=== -|Criteria |EKS managed node groups |EKS Auto Mode +|Criteria +|EKS managed node groups +|EKS Auto Mode + |Can be deployed to {aws} Outposts |No @@ -150,4 +145,4 @@ include::../attributes.txt[] |Cost of Amazon EC2 instance that runs multiple Pods. For more information, see Amazon EC2 pricing. | When EKS Auto Mode is enabled in your cluster, you pay a separate fee, in addition to the standard EC2 instance charges, for the instances launched using Auto Mode's compute capability. The amount varies with the instance type launched and the {aws} region where your cluster is located. For more information, see link:eks/pricing/["Amazon EKS pricing",type="marketing"]. -|=== +|=== \ No newline at end of file diff --git a/latest/ug/automode/auto-net-pol.adoc b/latest/ug/automode/auto-net-pol.adoc index 4194fc26e..e9611415c 100644 --- a/latest/ug/automode/auto-net-pol.adoc +++ b/latest/ug/automode/auto-net-pol.adoc @@ -1,8 +1,6 @@ -//!!NODE_ROOT
[.topic] -[[auto-net-pol,auto-net-pol.title]] +[#auto-net-pol] = Use Network Policies with EKS Auto Mode -:info_doctype: section :info_titleabbrev: Use network policies include::../attributes.txt[] @@ -11,12 +9,12 @@ include::../attributes.txt[] Network policies allow you to control traffic flow at the IP address or port level within your Amazon EKS cluster. This topic explains how to enable and use network policies with EKS Auto Mode. -## Prerequisites +== Prerequisites * An Amazon EKS cluster with EKS Auto Mode enabled * kubectl configured to connect to your cluster -## Step 1: Enable Network Policy Controller +== Step 1: Enable Network Policy Controller To use network policies with EKS Auto Mode, you first need to enable the Network Policy Controller by applying a ConfigMap to your cluster. @@ -37,7 +35,7 @@ data: kubectl apply -f enable-network-policy.yaml ``` -## Step 2: Enable Network Policies in Node Class +== Step 2: Enable Network Policies in Node Class Before you can use network policies, you need to ensure that your Node Class is configured to support them. Follow these steps: @@ -69,6 +67,6 @@ kubectl get nodeclass network-policy-enabled Once your nodes are using this Node Class, they will be able to enforce network policies. You can now proceed to create and apply network policies to control traffic within your cluster. For all the node class configuration options, see <>. -## Step 3: Create and test network policies +== Step 3: Create and test network policies -Your EKS Auto Mode cluster is now configured to support Kubernetes network policies. You can test this with the <>. +Your EKS Auto Mode cluster is now configured to support Kubernetes network policies. You can test this with the <>. \ No newline at end of file diff --git a/latest/ug/automode/auto-networking.adoc b/latest/ug/automode/auto-networking.adoc index 711f7bd30..4ee5f2915 100644 --- a/latest/ug/automode/auto-networking.adoc +++ b/latest/ug/automode/auto-networking.adoc @@ -1,70 +1,95 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[auto-networking,auto-networking.title]] +[#auto-networking] = Learn about VPC Networking and Load Balancing in EKS Auto Mode -:info_doctype: section -:info_title: Learn about VPC networking and load balancing in EKS Auto Mode :info_titleabbrev: Networking -:info_abstract: Learn about VPC networking and load balancing in EKS Auto Mode. - - -include::../attributes.txt[] +This topic explains how to configure Virtual Private Cloud (VPC) networking and load balancing features in EKS Auto Mode. While EKS Auto Mode manages most networking components automatically, you can still customize certain aspects of your cluster's networking configuration through `NodeClass` resources and load balancer annotations. -This topic explains how to configure Virtual Private Cloud (VPC) networking and load balancing features in EKS Auto Mode. While EKS Auto Mode manages most networking components automatically, you can still customize certain aspects of your cluster's networking configuration through NodeClass resources and load balancer annotations. +When you use EKS Auto Mode, {aws} manages the VPC Container Network Interface (CNI) configuration and load balancer provisioning for your cluster. You can influence networking behaviors by defining `NodeClass` objects and applying specific annotations to your Service and Ingress resources, while maintaining the automated operational model that EKS Auto Mode provides. -When you use EKS Auto Mode, {aws} manages the VPC Container Network Interface (CNI) configuration and load balancer provisioning for your cluster. You can influence networking behaviors by defining NodeClass objects and applying specific annotations to your Service and Ingress resources, while maintaining the automated operational model that EKS Auto Mode provides. +== Networking capability -== VPC CNI networking +EKS Auto Mode has a new networking capability that handles node and pod networking. You can configure it by creating a `NodeClass` Kubernetes object. -With {eam}, you do not directly configure the {aws} VPC CNI. {aws} manages node and pod networking. Instead, you create a `NodeClass` Kubernetes object. +Configuration options for the previous {aws} VPC CNI will not apply to EKS Auto Mode. -=== Configure VPC CNI with NodeClass +=== Configure networking with a `NodeClass` -The NodeClass resource in EKS Auto Mode allows you to customize certain aspects of the VPC Container Network Interface (CNI) configuration without directly managing the CNI plugin. Through NodeClass, you can specify security group selections, control node placement across VPC subnets, set SNAT policies, configure network policies, and enable network event logging. This approach maintains the automated operational model of EKS Auto Mode while providing flexibility for network customization. +The `NodeClass` resource in EKS Auto Mode allows you to customize certain aspects of the networking capability. Through `NodeClass`, you can specify security group selections, control node placement across VPC subnets, set SNAT policies, configure network policies, and enable network event logging. This approach maintains the automated operational model of EKS Auto Mode while providing flexibility for network customization. -You can use a NodeClass to: +You can use a `NodeClass` to: * Select a Security Group for Nodes * Control how nodes are placed on VPC Subnets * Set the Node SNAT Policy to `random` or `disabled` -* Set the Network Policy to Default Deny or Default Allow -* Enable Network Event Logging to a file. +* Enable Kubernetes __network policies__ including: +** Set the Network Policy to Default Deny or Default Allow +** Enable Network Event Logging to a file. +* Isolate pod traffic from the node traffic by attaching pods to different subnets. -Learn how to xref:create-node-class[Create an Amazon EKS NodeClass]. +Learn how to <>. === Considerations -{eam} supports: +EKS Auto Mode supports: * EKS Network Policies. * The `HostPort` and `HostNetwork` options for Kubernetes Pods. -* Pods in public or private subnets. +* Nodes and Pods in public or private subnets. +* Caching DNS queries on the node. -{eam} does *not* support: +EKS Auto Mode does *not* support: * Security Groups per Pod (SGPP). -* Custom Networking. The IP Addresses of Pods and Nodes must be from the same CIDR Block. +* Custom Networking in the `ENIConfig`. You can put pods in multiple subnets or exclusively isolate them from the node traffic with <>. * Warm IP, warm prefix, and warm ENI configurations. * Minimum IP targets configuration. -* Enabling or disabling prefix delegation. -* Other configurations supported by the open-source {aws} CNI. +* Other configurations supported by the open source {aws} VPC CNI. * Network Policy configurations such as conntrack timer customization (default is 300s). * Exporting network event logs to CloudWatch. +=== Network Resource Management + +EKS Auto Mode handles prefix, IP addressing, and network interface management by monitoring NodeClass resources for networking configurations. The service performs several key operations automatically: + +*Prefix Delegation* + +EKS Auto Mode defaults to using prefix delegation (/28 prefixes) for pod networking and maintains a predefined warm pool of IP resources that scales based on the number of scheduled pods. When pod subnet fragmentation is detected, Auto Mode provisions secondary IP addresses (/32). Due to this default pod networking algorithm, Auto Mode calculates max pods per node based on the number of ENIs and IPs supported per instance type (assuming the worst case of fragmentation). For more information about Max ENIs and IPs per instance type, see link:AWSEC2/latest/UserGuide/AvailableIpPerENI.html[Maximum IP addresses per network interface,type="documentation"] in the EC2 User Guide. Newer generation (Nitro v6 and above) instance families generally have increased ENIs and IPs per instance type, and Auto Mode adjusts the max pods calculation accordingly. + +For IPv6 clusters, only prefix delegation is used, and Auto Mode always uses a max pods limit of 110 pods per node. + +*Cooldown Management* -[[auto-lb-consider,auto-lb-consider.title]] +The service implements a cooldown pool for prefixes or secondary IPv4 addresses that are no longer in use. After the cooldown period expires, these resources are released back to the VPC. However, if pods reuse these resources during the cooldown period, they are restored from the cooldown pool. + +*IPv6 Support* + +For IPv6 clusters, EKS Auto Mode provisions a `/80` IPv6 prefix per node on the primary network interface. + +The service also ensures proper management and garbage collection of all network interfaces. + + + +[#auto-lb-consider] == Load balancing -You configure {aws} Elastic Load Balancers provisioned by {eam} using annotations on Service and Ingress resources. +You configure {aws} Elastic Load Balancers provisioned by EKS Auto Mode using annotations on Service and Ingress resources. For more information, see <> or <>. -=== Considerations for load balancing with {eam} +=== Considerations for load balancing with EKS Auto Mode * The default targeting mode is IP Mode, not Instance Mode. -* {eam} only supports Security Group Mode for Network Load Balancers. -* {aws} does not support migrating load balancers from the self managed {aws} load balancer controller to management by {eam}. +* EKS Auto Mode only supports Security Group Mode for Network Load Balancers. +* {aws} does not support migrating load balancers from the self managed {aws} load balancer controller to management by EKS Auto Mode. * The `networking.ingress.ipBlock` field in `TargetGroupBinding` spec is not supported. -* If your worker nodes use custom security groups (not `+eks-cluster-sg-*+` naming pattern), your cluster role needs additional IAM permissions. The default EKS-managed policy only allows EKS to modify security groups named `+eks-cluster-sg-*+`. Without permission to modify your custom security groups, EKS cannot add the required ingress rules that allow ALB/NLB traffic to reach your pods. -* You cannot bring your own target groups. +* If your worker nodes use custom security groups (not `eks-cluster-sg-*` naming pattern), your cluster role needs additional IAM permissions. The default EKS-managed policy only allows EKS to modify security groups named `eks-cluster-sg-*`. Without permission to modify your custom security groups, EKS cannot add the required ingress rules that allow ALB/NLB traffic to reach your pods. + +[#dns-consider] +==== CoreDNS considerations + +EKS Auto Mode does not use the traditional CoreDNS deployment to provide DNS resolution within the cluster. Instead, Auto Mode nodes utilize CoreDNS running as a system service directly on each node. If transitioning a traditional cluster to Auto Mode, you can remove the CoreDNS deployment from your cluster once your workloads have been moved to the Auto Mode nodes. + +IMPORTANT: If you plan to maintain a cluster with both Auto Mode and non-Auto Mode nodes, you must retain the CoreDNS deployment. Non-Auto Mode nodes rely on the traditional CoreDNS pods for DNS resolution, as they cannot access the node-level DNS service that Auto Mode provides. diff --git a/latest/ug/automode/auto-odcr.adoc b/latest/ug/automode/auto-odcr.adoc new file mode 100644 index 000000000..949a806be --- /dev/null +++ b/latest/ug/automode/auto-odcr.adoc @@ -0,0 +1,170 @@ +include::../attributes.txt[] + +[.topic] +[#auto-odcr] += Control deployment of workloads into Capacity Reservations with EKS Auto Mode +:info_titleabbrev: Control Capacity Reservations + +You can control the deployment of workloads onto link:AWSEC2/latest/UserGuide/capacity-reservation-overview.html[Capacity Reservations,type="documentation"]. EKS Auto Mode supports EC2 On-Demand Capacity Reservations (ODCRs), and EC2 Capacity Blocks for ML. + +TIP: By default, EKS Auto Mode automatically launches into open ODCRs and ML Capacity Blocks. When using `capacityReservationSelectorTerms` in the NodeClass definition, EKS Auto Mode will no longer automatically use any open Capacity Reservations. + +== EC2 On-Demand Capacity Reservations (ODCRs) + +EC2 On-Demand Capacity Reservations (ODCRs) allow you to reserve compute capacity for your Amazon EC2 instances in a specific Availability Zone for any duration. When using EKS Auto Mode, you may want to control whether your Kubernetes workloads are deployed onto these reserved instances to maximize utilization of pre-purchased capacity or to ensure critical workloads have access to guaranteed resources. + +By default, EKS Auto Mode automatically launches into open ODCRs. However, by configuring `capacityReservationSelectorTerms` on a NodeClass, you can explicitly control which ODCRs your workloads use. Nodes provisioned using configured ODCRs will have `karpenter.sh/capacity-type: reserved` and will be prioritized over on-demand and spot. Once this feature is enabled, EKS Auto Mode will no longer automatically use open ODCRs—they must be explicitly selected by a NodeClass, giving you precise control over capacity reservation usage across your cluster. + +[WARNING] +==== +If you configure `capacityReservationSelectorTerms` on a NodeClass in a cluster, EKS Auto Mode will no longer automatically use open ODCRs for _any_ NodeClass in the cluster. +==== + +=== Example NodeClass + +```yaml +apiVersion: eks.amazonaws.com/v1 +kind: NodeClass +spec: + # Optional: Selects upon on-demand capacity reservations and capacity blocks + # for EKS Auto Mode to prioritize. + capacityReservationSelectorTerms: + - id: cr-56fac701cc1951b03 + # Alternative Approaches + - tags: + app: "my-app" + # Optional owning account ID filter + owner: "012345678901" +``` + +This example NodeClass demonstrates two approaches for selecting ODCRs. The first method directly references a specific ODCR by its ID (`cr-56fac701cc1951b03`). The second method uses tag-based selection, targeting ODCRs with the tag `Name: "targeted-odcr"`. You can also optionally filter by the {aws} account that owns the reservation, which is particularly useful in cross-account scenarios or when working with shared capacity reservations. + +== EC2 Capacity Blocks for ML + +Capacity Blocks for ML reserve GPU-based accelerated computing instances on a future date to support your short duration machine learning (ML) workloads. Instances that run inside a Capacity Block are automatically placed close together inside Amazon EC2 UltraClusters, for low-latency, petabit-scale, non-blocking networking. + +For more information about the supported platforms and instance types, see link:AWSEC2/latest/UserGuide/ec2-capacity-blocks.html[Capacity Blocks for ML,type="documentation"] in the EC2 User Guide. + +You can create an EKS Auto Mode NodeClass that uses a Capacity Block for ML, similar to an ODCR (described earlier). + +The following sample definitions create three resources: + +. A NodeClass that references your Capacity Block reservation +. A NodePool that uses the NodeClass and applies a taint +. A Pod specification that tolerates the taint and requests GPU resources + +=== Example NodeClass + +This NodeClass references a specific Capacity Block for ML by its reservation ID. You can obtain this ID from the EC2 console. + +```yaml +apiVersion: eks.amazonaws.com/v1 +kind: NodeClass +metadata: + name: gpu +spec: + # Specify your Capacity Block reservation ID + capacityReservationSelectorTerms: + - id: cr-56fac701cc1951b03 +``` + +For more information, see <>. + +=== Example NodePool + +This NodePool references the `gpu` NodeClass and specifies important configuration: + +* It **only** uses reserved capacity by setting `karpenter.sh/capacity-type: reserved` +* It requests specific GPU instance families appropriate for ML workloads +* It applies a `nvidia.com/gpu` taint to ensure only GPU workloads are scheduled on these nodes + +```yaml +apiVersion: karpenter.sh/v1 +kind: NodePool +metadata: + name: gpu +spec: + template: + spec: + nodeClassRef: + group: eks.amazonaws.com + kind: NodeClass + name: gpu + requirements: + - key: eks.amazonaws.com/instance-family + operator: In + values: + - g6 + - p4d + - p4de + - p5 + - p5e + - p5en + - p6 + - p6-b200 + - key: karpenter.sh/capacity-type + operator: In + values: + - reserved + # Enable other capacity types + # - on-demand + # - spot + taints: + - effect: NoSchedule + key: nvidia.com/gpu +``` + +For more information, see <>. + +=== Example Pod + +This example pod demonstrates how to configure a workload to run on your Capacity Block nodes: + +* It uses a **nodeSelector** to target specific GPU types (in this case, H200 GPUs) +* It includes a **toleration** for the `nvidia.com/gpu` taint applied by the NodePool +* It explicitly **requests GPU resources** using the `nvidia.com/gpu` resource type + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: nvidia-smi +spec: + nodeSelector: + # Select specific GPU type - uncomment as needed + # eks.amazonaws.com/instance-gpu-name: l4 + # eks.amazonaws.com/instance-gpu-name: a100 + eks.amazonaws.com/instance-gpu-name: h200 + # eks.amazonaws.com/instance-gpu-name: b200 + eks.amazonaws.com/compute-type: auto + restartPolicy: OnFailure + containers: + - name: nvidia-smi + image: public.ecr.aws/amazonlinux/amazonlinux:2023-minimal + args: + - "nvidia-smi" + resources: + requests: + # Uncomment if needed + # memory: "30Gi" + # cpu: "3500m" + nvidia.com/gpu: 1 + limits: + # Uncomment if needed + # memory: "30Gi" + nvidia.com/gpu: 1 + tolerations: + - key: nvidia.com/gpu + effect: NoSchedule + operator: Exists +``` + +For more information, see https://kubernetes.io/docs/concepts/workloads/pods/[Pods] in the Kubernetes documentation. + +=== Related Resources + +* link:AWSEC2/latest/UserGuide/ec2-capacity-blocks.html[Capacity Blocks for ML,type="documentation"] in the Amazon EC2 User Guide +* link:AWSEC2/latest/UserGuide/capacity-blocks-purchase.html[Find and purchase Capacity Blocks,type="documentation"] in the Amazon EC2 User Guide +* link:eks/latest/userguide/ml-compute-management.html[Manage compute resources for AI/ML workloads on Amazon EKS,type="documentation"] +* link:eks/latest/best-practices/aiml-compute.html#_gpu_resource_optimization_and_cost_management[GPU Resource Optimization and Cost Management,type="documentation"] in the EKS Best Practices Guide + diff --git a/latest/ug/automode/auto-reference.adoc b/latest/ug/automode/auto-reference.adoc index 25be16060..b0e57b9ad 100644 --- a/latest/ug/automode/auto-reference.adoc +++ b/latest/ug/automode/auto-reference.adoc @@ -1,17 +1,9 @@ -//!!NODE_ROOT -[.topic] include::../attributes.txt[] -[[auto-reference,auto-reference.title]] + +[.topic] +[#auto-reference] = Learn how EKS Auto Mode works -:info_doctype: section -:icons: font -:experimental: -:idprefix: -:idseparator: - -:sourcedir: . -:info_title: Learn how EKS Auto Mode works :info_titleabbrev: How it works -:info_abstract: Learn how EKS Auto Mode works [abstract] -- @@ -35,4 +27,4 @@ include::auto-networking.adoc[leveloffset=+1] //include::term-reference.adoc[leveloffset=+1] -//include::auto-glossary.adoc[leveloffset=+1] +//include::auto-glossary.adoc[leveloffset=+1] \ No newline at end of file diff --git a/latest/ug/automode/auto-troubleshoot.adoc b/latest/ug/automode/auto-troubleshoot.adoc index ee9897bfc..dfe2431e8 100644 --- a/latest/ug/automode/auto-troubleshoot.adoc +++ b/latest/ug/automode/auto-troubleshoot.adoc @@ -1,43 +1,49 @@ -//!!NODE_ROOT
+include::../attributes.txt[] [.topic] -[[auto-troubleshoot,auto-troubleshoot.title]] +[#auto-troubleshoot] = Troubleshoot EKS Auto Mode -:info_doctype: section -:info_title: Troubleshoot EKS Auto Mode :info_titleabbrev: Troubleshoot -:info_abstract: Troubleshoot EKS Auto Mode - -include::../attributes.txt[] - -With {eam}, {aws} assumes more {resp} for {e2i}s in {yaa}. EKS assumes {resp} for the container runtime on nodes, the operating system on the nodes, and certain controllers. This includes a block storage controller, a load balancing controller, and a compute controller. +With EKS Auto Mode, {aws} assumes more responsibility for EC2 Instances in your {aws} account. EKS assumes responsibility for the container runtime on nodes, the operating system on the nodes, and certain controllers. This includes a block storage controller, a load balancing controller, and a compute controller. -You must use {aws} and {k8s} APIs to troubleshoot nodes. You can: +You must use {aws} and Kubernetes APIs to troubleshoot nodes. You can: -* Use a Kubernetes `NodeDiagnostic` resource to {ret} node logs. -* Use the {aws} EC2 CLI command `get-console-output` to {ret} console output from nodes. +* Use a Kubernetes `NodeDiagnostic` resource to retrieve node logs by using the <>. For more steps, see <>. +* Use the {aws} EC2 CLI command `get-console-output` to retrieve console output from nodes. For more steps, see <>. +* Use Kubernetes _debugging containers_ to retrieve node logs. For more steps, see <>. [NOTE] ==== -{eam} uses {emi}s. You cannot directly access {emi}s, including by SSH. +EKS Auto Mode uses EC2 managed instances. You cannot directly access EC2 managed instances, including by SSH. ==== -If you have a problem with a controller, you should research: +You might have the following problems that have solutions specific to EKS Auto Mode components: -* If the resources associated with that controller are properly formatted and valid. -* If the {aws} IAM and Kubernetes RBAC resources are properly configured for your cluster. For more information, see <>. +* Pods stuck in the `Pending` state, that aren't being scheduled onto Auto Mode nodes. For solutions see <>. +* EC2 managed instances that don't join the cluster as Kubernetes nodes. For solutions see <>. +* Errors and issues with the `NodePools`, `PersistentVolumes`, and `Services` that use the controllers that are included in EKS Auto Mode. For solutions see <>. +* Enhanced Pod security prevents sharing volumes across Pods. For solutions see <>. -[[auto-node-monitoring-agent,auto-node-monitoring-agent.title]] +You can use the following methods to troubleshoot EKS Auto Mode components: + +* <> +* <> +* <> +* <> +* <> + +[#auto-node-monitoring-agent] == Node monitoring agent -{eam} includes the Amazon EKS node monitoring agent. You can use this agent to view troubleshooting and debugging information about nodes. The node monitoring agent publishes Kubernetes `events` and node `conditions`. For more information, see <>. +EKS Auto Mode includes the Amazon EKS node monitoring agent. You can use this agent to view troubleshooting and debugging information about nodes. The node monitoring agent publishes Kubernetes `events` and node `conditions`. For more information, see <>. -== Get console output from an {emi} by using the {aws} EC2 CLI +[#auto-node-console] +== Get console output from an EC2 managed instance by using the {aws} EC2 CLI This procedure helps with troubleshooting boot-time or kernel-level issues. -First, you need to {det} the EC2 Instance ID of the instance associated with your workload. Second, use the {aws} CLI to {ret} the console output. +First, you need to determine the EC2 Instance ID of the instance associated with your workload. Second, use the {aws} CLI to retrieve the console output. . Confirm you have `kubectl` installed and connected to your cluster . (Optional) Use the name of a Kubernetes Deployment to list the associated pods. @@ -52,20 +58,71 @@ kubectl get pods -l app= ---- kubectl get pod -o wide ---- -. Use the EC2 instance ID to {ret} the console output. +. Use the EC2 instance ID to retrieve the console output. + [source,cli] ---- aws ec2 get-console-output --instance-id --latest --output text ---- -== Get node logs by using the kubectl CLI +[#auto-node-debug-logs] +== Get node logs by using __debug containers__ and the `kubectl` CLI + +The recommended way of retrieving logs from an EKS Auto Mode node is to use `NodeDiagnostic` resource. For these steps, see <>. + +However, you can stream logs live from an instance by using the `kubectl debug node` command. This command launches a new Pod on the node that you want to debug which you can then interactively use. + +. Launch a debug container. The following command uses `i-01234567890123456` for the instance ID of the node, `-it` allocates a `tty` and attach `stdin` for interactive usage, and uses the `sysadmin` profile from the kubeconfig file. ++ +[source,cli] +---- +kubectl debug node/i-01234567890123456 -it --profile=sysadmin --image=public.ecr.aws/amazonlinux/amazonlinux:2023 +---- ++ +An example output is as follows. ++ +[source,none] +---- +Creating debugging pod node-debugger-i-01234567890123456-nxb9c with container debugger on node i-01234567890123456. +If you don't see a command prompt, try pressing enter. +bash-5.2# +---- + +. From the shell, you can now install `util-linux-core` which provides the `nsenter` command. Use `nsenter` to enter the mount namespace of PID 1 (`init`) on the host, and run the `journalctl` command to stream logs from the `kubelet`: ++ +[source,bash] +---- +yum install -y util-linux-core +nsenter -t 1 -m journalctl -f -u kubelet +---- + +For security, the Amazon Linux container image doesn't install many binaries by default. You can use the `yum whatprovides` command to identify the package that must be installed to provide a given binary. + +[source,cli] +---- +yum whatprovides ps +---- + +[source,none] +---- +Last metadata expiration check: 0:03:36 ago on Thu Jan 16 14:49:17 2025. +procps-ng-3.3.17-1.amzn2023.0.2.x86_64 : System and process monitoring utilities +Repo : @System +Matched from: +Filename : /usr/bin/ps +Provide : /bin/ps -For information about getting node logs, see <>. +procps-ng-3.3.17-1.amzn2023.0.2.x86_64 : System and process monitoring utilities +Repo : amazonlinux +Matched from: +Filename : /usr/bin/ps +Provide : /bin/ps +---- -== View resources associated with {eam} in the {aws} Console +[#auto-node-ec2-web] +== View resources associated with EKS Auto Mode in the {aws} Console -You can use the {aws} console to view the status of resources associated with {yec}. +You can use the {aws} console to view the status of resources associated with your EKS Auto Mode cluster. * link:ec2/home#Volumes["EBS Volumes",type="console"] ** View EKS Auto Mode volumes by searching for the tag key `eks:eks-cluster-name` @@ -74,7 +131,8 @@ You can use the {aws} console to view the status of resources associated with {y * link:ec2/home#Instances["EC2 Instances",type="console"] ** View EKS Auto Mode instances by searching for the tag key `eks:eks-cluster-name` -== View IAM Errors in {yaa} +[#auto-node-iam] +== View IAM Errors in your {aws} account . Navigate to CloudTrail console . Select "Event History" from the left navigation pane @@ -83,6 +141,197 @@ You can use the {aws} console to view the status of resources associated with {y ** UnauthorizedOperation ** InvalidClientTokenId -Look for errors related to your EKS cluster. Use the error messages to update your EKS access entries, Cluster IAM Role, or Node IAM Role. You may need to attach a new policy these roles with permissions for {eam}. +Look for errors related to your EKS cluster. Use the error messages to update your EKS access entries, cluster IAM role, or node IAM role. You might need to attach a new policy to these roles with permissions for EKS Auto Mode. //Ensure you are running the latest version of the {aws} CLI, eksctl, etc. + +[#auto-troubleshoot-schedule] +== Troubleshoot Pod failing to schedule onto Auto Mode node + +If pods staying in the `Pending` state and aren't being scheduled onto an auto mode node, verify if your pod or deployment manifest has a `nodeSelector`. If a `nodeSelector` is present, ensure that it is using `eks.amazonaws.com/compute-type: auto` to be scheduled on nodes that are made by EKS Auto Mode. For more information about the node labels that are used by EKS Auto Mode, see <>. + +[#auto-troubleshoot-join] +== Troubleshoot node not joining the cluster + +EKS Auto Mode automatically configures new EC2 instances with the correct information to join the cluster, including the cluster endpoint and cluster certificate authority (CA). However, these instances can still fail to join the EKS cluster as a node. Run the following commands to identify instances that didn't join the cluster: + +. Run `kubectl get nodeclaim` to check for `NodeClaims` that are `Ready = False`. ++ +[source,cli] +---- +kubectl get nodeclaim +---- + +. Run `kubectl describe nodeclaim ` and look under *Status* to find any issues preventing the node from joining the cluster. ++ +[source,cli] +---- +kubectl describe nodeclaim +---- + +*Common error messages:* + +`Error getting launch template configs`:: +You might receive this error if you are setting custom tags in the `NodeClass` with the default cluster IAM role permissions. See <>. + +`Error creating fleet`:: +There might be some authorization issue with calling the `RunInstances` call from the EC2 API. Check {aws} CloudTrail for errors and see <> for the required IAM permissions. + + +[#auto-node-reachability] +=== Detect node connectivity issues with the `VPC Reachability Analyzer` + +[NOTE] +==== +You are charged for each analysis that is run the VPC Reachability Analyzer. For pricing details, see link:vpc/pricing/[Amazon VPC Pricing,type="marketing"]. +==== + +One reason that an instance didn't join the cluster is a network connectivity issue that prevents them from reaching the API server. To diagnose this issue, you can use the link:vpc/latest/reachability/what-is-reachability-analyzer.html[VPC Reachability Analyzer,type="documentation"] to perform an analysis of the connectivity between a node that is failing to join the cluster and the API server. You will need two pieces of information: + +* *instance ID* of a node that can't join the cluster +* IP address of the *Kubernetes API server endpoint* + +To get the *instance ID*, you will need to create a workload on the cluster to cause EKS Auto Mode to launch an EC2 instance. This also creates a `NodeClaim` object in your cluster that will have the instance ID. Run `kubectl get nodeclaim -o yaml` to print all of the `NodeClaims` in your cluster. Each `NodeClaim` contains the instance ID as a field and again in the providerID: + +[source,cli] +---- +kubectl get nodeclaim -o yaml +---- + +An example output is as follows. + +[source,bash,subs="verbatim,attributes"] +---- + nodeName: i-01234567890123456 + providerID: aws:///us-west-2a/i-01234567890123456 +---- + +You can determine your *Kubernetes API server endpoint* by running `kubectl get endpoint kubernetes -o yaml`. The addresses are in the addresses field: + +[source,cli] +---- +kubectl get endpoints kubernetes -o yaml +---- + +An example output is as follows. + +[source,bash,subs="verbatim,attributes"] +---- +apiVersion: v1 +kind: Endpoints +metadata: + name: kubernetes + namespace: default +subsets: +- addresses: + - ip: 10.0.143.233 + - ip: 10.0.152.17 + ports: + - name: https + port: 443 + protocol: TCP +---- + +With these two pieces of information, you can perform the s analysis. First navigate to the VPC Reachability Analyzer in the {aws-management-console}. + +. Click "Create and Analyze Path" +. Provide a name for the analysis (e.g. "Node Join Failure") +. For the "Source Type" select "Instances" +. Enter the instance ID of the failing Node as the "Source" +. For the "Path Destination" select "IP Address" +. Enter one of the IP addresses for the API server as the "Destination Address" +. Expand the "Additional Packet Header Configuration Section" +. Enter a "Destination Port" of 443 +. Select "Protocol" as TCP if it is not already selected +. Click "Create and Analyze Path" +. The analysis might take a few minutes to complete. If the analysis results indicates failed reachability, it will indicate where the failure was in the network path so you can resolve the issue. + +[#auto-troubleshoot-share-pod-volumes] +== Sharing Volumes Across Pods + +EKS Auto Mode Nodes are configured with SELinux in enforcing mode which provides more isolation between Pods that are running on the same Node. When SELinux is enabled, most non-privileged pods will automatically have their own multi-category security (MCS) label applied to them. This MCS label is unique per Pod, and is designed to ensure that a process in one Pod cannot manipulate a process in any other Pod or on the host. Even if a labeled Pod runs as root and has access to the host filesystem, it will be unable to manipulate files, make sensitive system calls on the host, access the container runtime, or obtain kubelet’s secret key material. + +Due to this, you may experience issues when trying to share data between Pods. For example, a `PersistentVolumeClaim` with an access mode of `ReadWriteOnce` will still not allow multiple Pods to access the volume concurrently. + +To enable this sharing between Pods, you can use the Pod's `seLinuxOptions` to configure the same MCS label on those Pods. In this example, we assign the three categories `c123,c456,c789` to the Pod. This will not conflict with any categories assigned to Pods on the node automatically, as they will only be assigned two categories. + +[source,bash,subs="verbatim,attributes"] +---- +securityContext: + seLinuxOptions: + level: "s0:c123,c456,c789" +---- + +[#auto-view-karpenter-logs] +== View Karpenter events in control plane logs + +For EKS clusters with control plane logs enabled, you can gain insights into Karpenter's actions and decision-making process by querying the logs. This can be particularly useful for troubleshooting EKS Auto Mode issues related to node provisioning, scaling, and termination. To view Karpenter-related events, use the following CloudWatch Logs Insights query: + +[source] +---- +fields @timestamp, @message +| filter @logStream like /kube-apiserver-audit/ +| filter @message like 'DisruptionBlocked' +or @message like 'DisruptionLaunching' +or @message like 'DisruptionTerminating' +or @message like 'DisruptionWaitingReadiness' +or @message like 'Unconsolidatable' +or @message like 'FailedScheduling' +or @message like 'NoCompatibleInstanceTypes' +or @message like 'NodeRepairBlocked' +or @message like 'Disrupted' +or @message like 'Evicted' +or @message like 'FailedDraining' +or @message like 'TerminationGracePeriodExpiring' +or @message like 'TerminationFailed' +or @message like 'FailedConsistencyCheck' +or @message like 'InsufficientCapacityError' +or @message like 'UnregisteredTaintMissing' +or @message like 'NodeClassNotReady' +| sort @timestamp desc +---- + +This query filters for specific https://github.com/kubernetes-sigs/karpenter/blob/main/pkg/events/reason.go[Karpenter-related events] in the kube-apiserver audit logs. The events include various disruption states, scheduling failures, capacity issues, and node-related problems. By analyzing these logs, you can gain a better understanding of: + +* Why Karpenter is taking certain actions. +* Any issues preventing proper node provisioning, scaling, or termination. +* Potential capacity or compatibility problems with instance types. +* Node lifecycle events such as disruptions, evictions, or terminations. + +To use this query: + +. Navigate to the CloudWatch console +. Select "Logs Insights" from the left navigation pane +. Choose the log group for your EKS cluster's control plane logs +. Paste the query into the query editor +. Adjust the time range as needed +. Run the query + +The results will show you a timeline of Karpenter-related events, helping you troubleshoot issues, and understand the behavior of EKS Auto Mode in your cluster. To review Karpenter actions on a specific node, you can add the below line filter specifying the instance ID to the aforementioned query: + +[source] +---- +|filter @message like /[.replaceable]`i-12345678910123456`/ +---- + +[NOTE] +==== +To use this query, control plane logging must be enabled on your EKS cluster. If you haven't done this yet, please refer to <>. +==== + +[#auto-troubleshoot-controllers] +== Troubleshoot included controllers in Auto Mode + +If you have a problem with a controller, you should research: + +* If the resources associated with that controller are properly formatted and valid. +* If the {aws} IAM and Kubernetes RBAC resources are properly configured for your cluster. For more information, see <>. + +== Related resources + +Use these articles from {aws} re:Post for advanced troubleshooting steps: + +* https://repost.aws/articles/ARLpQOknr5Rb-w5iAT9sUBpQ[How to troubleshoot common scaling issues in EKS Auto-Mode?] +* https://repost.aws/articles/ARPcmFS1POTgqPCBdcZFp6BQ[How do I troubleshoot custom nodepool and nodeclass provisioning issues in Amazon EKS Auto Mode?] +* https://repost.aws/en/articles/ARLhrdl45TRASGkvViwtBG0Q[How do I troubleshoot EKS Auto Mode built-in node pools with Unknown Status?] + diff --git a/latest/ug/automode/auto-upgrade.adoc b/latest/ug/automode/auto-upgrade.adoc index 8e3d503e1..7fae8179f 100644 --- a/latest/ug/automode/auto-upgrade.adoc +++ b/latest/ug/automode/auto-upgrade.adoc @@ -1,11 +1,9 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[auto-upgrade,auto-upgrade.title]] +[#auto-upgrade] = Update the Kubernetes Version of an EKS Auto Mode cluster -:info_doctype: section -:info_titleabbrev: Update Kubernetes Version - -include::../attributes.txt[] +:info_titleabbrev: Update Kubernetes version This topic explains how to update the Kubernetes version of your Auto Mode cluster. Auto Mode simplifies the version update process by handling the coordination of control plane updates with node replacements, while maintaining workload availability through pod disruption budgets. @@ -13,17 +11,18 @@ When upgrading an Auto Mode cluster, many components that traditionally required == Learn about updates with EKS Auto Mode -After you initiate a control plane upgrade, {eam} begins replacing nodes in your cluster. The new nodes have the corresponding new Kubernetes version. {eam} observes pod disruption budgets when upgrading nodes. +After you initiate a control plane upgrade, EKS Auto Mode will upgrade nodes in your cluster. As nodes expire, EKS Auto Mode will replace them with new nodes. The new nodes have the corresponding new Kubernetes version. EKS Auto Mode observes pod disruption budgets when upgrading nodes. Additionally, you no longer need to update components like: -* CoreDNS -* KubeProxy +* Amazon VPC CNI * {aws} Load Balancer Controller +* CoreDNS +* `kube-proxy` * Karpenter -* {aws} EBS CSI Driver +* {aws} EBS CSI driver -{eam} replaces these components with service functionality. +EKS Auto Mode replaces these components with service functionality. You are still responsible for updating: @@ -36,4 +35,4 @@ Learn link:eks/latest/best-practices/cluster-upgrades.html["Best Practices for C == Start Cluster Update -To start a cluster update, see <>. +To start a cluster update, see <>. \ No newline at end of file diff --git a/latest/ug/automode/auto-workloads.adoc b/latest/ug/automode/auto-workloads.adoc index 7a99b60cc..685a5f9ef 100644 --- a/latest/ug/automode/auto-workloads.adoc +++ b/latest/ug/automode/auto-workloads.adoc @@ -1,12 +1,9 @@ -//!!NODE_ROOT -[.topic] include::../attributes.txt[] -[[auto-workloads,auto-workloads.title]] + +[.topic] +[#auto-workloads] = Run sample workloads in EKS Auto Mode clusters -:info_doctype: section -:info_title: Run workloads in EKS Auto Mode clusters :info_titleabbrev: Run workloads -:info_abstract: Run workloads in EKS Auto Mode clusters [abstract] -- @@ -22,6 +19,7 @@ You can use these use case-based samples to run workloads in EKS Auto Mode clust <>:: Shows how to deploy a sample workload to an EKS Auto Mode cluster using `kubectl` commands. <>:: Shows how to deploy a containerized version of the 2048 game on Amazon EKS. <>:: Shows how to deploy a sample stateful application to an EKS Auto Mode cluster. +<>:: Shows how to deploy hardware-accelerated workloads to nodes managed by EKS Auto Mode. <>:: Shows how to use an annotation to control if a workload is deployed to nodes managed by EKS Auto Mode. @@ -30,3 +28,5 @@ include::automode-workload.adoc[leveloffset=+1] include::auto-elb-example.adoc[leveloffset=+1] include::sample-storage-workload.adoc[leveloffset=+1] + +include::auto-accelerated.adoc[leveloffset=+1] diff --git a/latest/ug/automode/automode-get-started-cli.adoc b/latest/ug/automode/automode-get-started-cli.adoc index 99eec8c18..d58f3654d 100644 --- a/latest/ug/automode/automode-get-started-cli.adoc +++ b/latest/ug/automode/automode-get-started-cli.adoc @@ -1,27 +1,22 @@ -//!!NODE_ROOT
- include::../attributes.txt[] [.topic] -[[automode-get-started-cli,automode-get-started-cli.title]] +[#automode-get-started-cli] = Create an EKS Auto Mode Cluster with the {aws} CLI -:info_doctype: section :config: configuration -:info_title: Create an EKS Auto Mode Cluster with the {aws} CLI :info_titleabbrev: {aws} CLI -:info_abstract: Create an EKS Auto Mode cluster with the {aws} CLI EKS Auto Mode Clusters automate routine cluster management tasks for compute, storage, and networking. For example, EKS Auto Mode Clusters automatically detect when additional nodes are required and provision new EC2 instances to meet workload demands. This topic guides you through creating a new EKS Auto Mode Cluster using the {aws} CLI and optionally deploying a sample workload. -## Prerequisites +== Prerequisites * The latest version of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device. To check your current version, use `aws --version`. To install the latest version, see link:cli/latest/userguide/getting-started-install.html["Installing",type="documentation"] and link:cli/latest/userguide/cli-chap-configure.html#cli-configure-quickstart-config["Quick configuration",type="documentation"] with aws configure in the {aws} Command Line Interface User Guide. ** Login to the CLI with sufficent IAM permissions to create {aws} resources including IAM Policies, IAM Roles, and EKS Clusters. * The kubectl command line tool installed on your device. {aws} suggests you use the same kubectl version as the Kubernetes version of your EKS Cluster. To install or upgrade kubectl, see <>. -## Specify VPC subnets +== Specify VPC subnets Amazon EKS Auto Mode deploy nodes to VPC subnets. When creating an EKS cluster, you must specify the VPC subnets where the nodes will be deployed. You can use the default VPC subnets in your {aws} account or create a dedicated VPC for critical workloads. @@ -29,7 +24,7 @@ Amazon EKS Auto Mode deploy nodes to VPC subnets. When creating an EKS cluster, * The EKS Console assists with creating a new VPC. Learn how to <>. * Alternatively, you can use the default VPC of your {aws} account. Use the following instructions to find the Subnet IDs. -[[auto-find-subnet,auto-find-subnet.title]] +[#auto-find-subnet] .To find the Subnet IDs of your default VPC [%collapsible, expand-section="_collapse_all_"] ==== @@ -42,7 +37,7 @@ Amazon EKS Auto Mode deploy nodes to VPC subnets. When creating an EKS cluster, aws ec2 describe-subnets --filters "Name=vpc-id,Values=$(aws ec2 describe-vpcs --query 'Vpcs[?IsDefault==`true`].VpcId' --output text)" --query 'Subnets[*].{ID:SubnetId,AZ:AvailabilityZone}' --output table ``` + -. Save the output and note the **Subnet IDs**. +. Save the output and note the *Subnet IDs*. + Sample output: + @@ -60,21 +55,21 @@ Sample output: ==== -[[auto-mode-create-roles,auto-mode-create-roles.title]] +[#auto-mode-create-roles] == IAM Roles for EKS Auto Mode Clusters -[[auto-roles-cluster-iam-role,auto-roles-cluster-iam-role.title]] +[#auto-roles-cluster-iam-role] === Cluster IAM Role EKS Auto Mode requires a Cluster IAM Role to perform actions in your {aws} account, such as provisioning new EC2 instances. You must create this role to grant EKS the necessary permissions. {aws} recommends attaching the following {aws} managed policies to the Cluster IAM Role: -* xref:security-iam-awsmanpol-AmazonEKSComputePolicy[AmazonEKSComputePolicy] -* xref:security-iam-awsmanpol-AmazonEKSBlockStoragePolicy[AmazonEKSBlockStoragePolicy] -* xref:security-iam-awsmanpol-AmazonEKSLoadBalancingPolicy[AmazonEKSLoadBalancingPolicy] -* xref:security-iam-awsmanpol-AmazonEKSNetworkingPolicy[AmazonEKSNetworkingPolicy] -* xref:security-iam-awsmanpol-amazoneksclusterpolicy[AmazonEKSClusterPolicy] +* <> +* <> +* <> +* <> +* <> -[[auto-roles-node-iam-role,auto-roles-node-iam-role.title]] +[#auto-roles-node-iam-role] === Node IAM Role When you create an EKS Auto Mode cluster, you specify a Node IAM Role. When EKS Auto Mode creates nodes to process pending workloads, each new EC2 instance node is assigned the Node IAM Role. This role allows the node to communicate with EKS but is generally not accessed by workloads running on the node. @@ -83,39 +78,26 @@ If you want to grant permissions to workloads running on a node, use EKS Pod Ide You must create this role and attach the following {aws} managed policy: -* xref:security-iam-awsmanpol-AmazonEKSWorkerNodeMinimalPolicy[AmazonEKSWorkerNodeMinimalPolicy] +* <> * link:AmazonECR/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-AmazonEC2ContainerRegistryPullOnly["AmazonEC2ContainerRegistryPullOnly",type="documentation"] [discrete] -#### **Service-Linked Role** +==== Service-Linked Role -EKS Auto Mode also requires a Service-Linked Role, which is automatically created and configured by {aws}. For more information, see xref:using-service-linked-roles-eks[AWSServiceRoleForAmazonEKS]. +EKS Auto Mode also requires a Service-Linked Role, which is automatically created and configured by {aws}. For more information, see <>. -## **Create an EKS Auto Mode Cluster IAM Role** +== Create an EKS Auto Mode Cluster IAM Role -### Step 1: Create the Trust Policy +=== Step 1: Create the Trust Policy Create a trust policy that allows the Amazon EKS service to assume the role. Save the policy as `trust-policy.json`: -``` -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Service": "eks.amazonaws.com" - }, - "Action": [ - "sts:AssumeRole", - "sts:TagSession" - ] - } - ] -} -``` +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:3c455c2f-72a7-48be-b85c-d2431fccbb3b[] +---- -### Step 2: Create the IAM Role +=== Step 2: Create the IAM Role Use the trust policy to create the Cluster IAM Role: @@ -125,7 +107,7 @@ aws iam create-role \ --assume-role-policy-document file://trust-policy.json ``` -### Step 3: Note the Role ARN +=== Step 3: Note the Role ARN Retrieve and save the ARN of the new role for use in subsequent steps: @@ -133,74 +115,69 @@ Retrieve and save the ARN of the new role for use in subsequent steps: aws iam get-role --role-name AmazonEKSAutoClusterRole --query "Role.Arn" --output text ``` -### Step 4: Attach Required Policies +=== Step 4: Attach Required Policies Attach the following {aws} managed policies to the Cluster IAM Role to grant the necessary permissions: -**AmazonEKSClusterPolicy**: +*AmazonEKSClusterPolicy*: -``` +[source,bash,subs="verbatim,attributes"] +---- aws iam attach-role-policy \ --role-name AmazonEKSAutoClusterRole \ - --policy-arn arn:aws:iam::aws:policy/AmazonEKSClusterPolicy -``` + --policy-arn {arn-aws}iam::aws:policy/AmazonEKSClusterPolicy +---- -**AmazonEKSComputePolicy**: +*AmazonEKSComputePolicy*: -``` +[source,bash,subs="verbatim,attributes"] +---- aws iam attach-role-policy \ --role-name AmazonEKSAutoClusterRole \ - --policy-arn arn:aws:iam::aws:policy/AmazonEKSComputePolicy -``` + --policy-arn {arn-aws}iam::aws:policy/AmazonEKSComputePolicy +---- -**AmazonEKSBlockStoragePolicy**: +*AmazonEKSBlockStoragePolicy*: -``` +[source,bash,subs="verbatim,attributes"] +---- aws iam attach-role-policy \ --role-name AmazonEKSAutoClusterRole \ - --policy-arn arn:aws:iam::aws:policy/AmazonEKSBlockStoragePolicy -``` + --policy-arn {arn-aws}iam::aws:policy/AmazonEKSBlockStoragePolicy +---- -**AmazonEKSLoadBalancingPolicy**: +*AmazonEKSLoadBalancingPolicy*: -``` +[source,bash,subs="verbatim,attributes"] +---- aws iam attach-role-policy \ --role-name AmazonEKSAutoClusterRole \ - --policy-arn arn:aws:iam::aws:policy/AmazonEKSLoadBalancingPolicy -``` + --policy-arn {arn-aws}iam::aws:policy/AmazonEKSLoadBalancingPolicy +---- -**AmazonEKSNetworkingPolicy**: +*AmazonEKSNetworkingPolicy*: -``` +[source,bash,subs="verbatim,attributes"] +---- aws iam attach-role-policy \ --role-name AmazonEKSAutoClusterRole \ - --policy-arn arn:aws:iam::aws:policy/AmazonEKSNetworkingPolicy -``` + --policy-arn {arn-aws}iam::aws:policy/AmazonEKSNetworkingPolicy +---- -## **Create an EKS Auto Mode Node IAM Role** +== Create an EKS Auto Mode Node IAM Role -### Step 1: Create the Trust Policy +=== Step 1: Create the Trust Policy Create a trust policy that allows the Amazon EKS service to assume the role. Save the policy as `node-trust-policy.json`: -``` -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Service": "ec2.amazonaws.com" - }, - "Action": "sts:AssumeRole" - } - ] -} -``` +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:8648fc4c-e230-41b8-af3e-daabdcf02888[] +---- -#### Step 2: Create the Node IAM Role +==== Step 2: Create the Node IAM Role -Use the **node-trust-policy.json** file from the previous step to define which entities can assume the role. Run the following command to create the Node IAM Role: +Use the *node-trust-policy.json* file from the previous step to define which entities can assume the role. Run the following command to create the Node IAM Role: ``` aws iam create-role \ @@ -208,7 +185,7 @@ aws iam create-role \ --assume-role-policy-document file://node-trust-policy.json ``` -#### Step 3: Note the Role ARN +==== Step 3: Note the Role ARN After creating the role, retrieve and save the ARN of the Node IAM Role. You will need this ARN in subsequent steps. Use the following command to get the ARN: @@ -216,29 +193,31 @@ After creating the role, retrieve and save the ARN of the Node IAM Role. You wil aws iam get-role --role-name AmazonEKSAutoNodeRole --query "Role.Arn" --output text ``` -#### Step 4: Attach Required Policies +==== Step 4: Attach Required Policies Attach the following {aws} managed policies to the Node IAM Role to provide the necessary permissions: -**AmazonEKSWorkerNodeMinimalPolicy**: +*AmazonEKSWorkerNodeMinimalPolicy*: -``` +[source,bash,subs="verbatim,attributes"] +---- aws iam attach-role-policy \ --role-name AmazonEKSAutoNodeRole \ - --policy-arn arn:aws:iam::aws:policy/AmazonEKSWorkerNodeMinimalPolicy -``` + --policy-arn {arn-aws}iam::aws:policy/AmazonEKSWorkerNodeMinimalPolicy +---- -**AmazonEC2ContainerRegistryPullOnly**: +*AmazonEC2ContainerRegistryPullOnly*: -``` +[source,bash,subs="verbatim,attributes"] +---- aws iam attach-role-policy \ --role-name AmazonEKSAutoNodeRole \ - --policy-arn arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPullOnly -``` + --policy-arn {arn-aws}iam::aws:policy/AmazonEC2ContainerRegistryPullOnly +---- -## **Create an EKS Auto Mode Cluster** +== Create an EKS Auto Mode Cluster -### Overview +=== Overview To create an EKS Auto Mode Cluster using the {aws} CLI, you will need the following parameters: @@ -248,20 +227,20 @@ To create an EKS Auto Mode Cluster using the {aws} CLI, you will need the follow * `cluster-role-arn`: ARN of the Cluster IAM Role. * `node-role-arn`: ARN of the Node IAM Role. -#### Default Cluster Configurations +==== Default Cluster Configurations Review these default values and features before creating the cluster: -* `nodePools`: EKS Auto Mode includes general-purpose and system default Node Pools. Learn more about xref:create-node-pool[Node Pools]. +* `nodePools`: EKS Auto Mode includes general-purpose and system default Node Pools. Learn more about <>. -**Note:** Node Pools in EKS Auto Mode differ from Amazon EKS Managed Node Groups but can coexist in the same cluster. +*Note:* Node Pools in EKS Auto Mode differ from Amazon EKS Managed Node Groups but can coexist in the same cluster. * `computeConfig.enabled`: Automates routine compute tasks, such as creating and deleting EC2 instances. * `kubernetesNetworkConfig.elasticLoadBalancing.enabled`: Automates load balancing tasks, including creating and deleting Elastic Load Balancers. * `storageConfig.blockStorage.enabled`: Automates storage tasks, such as creating and deleting Amazon EBS volumes. -* `accessConfig.authenticationMode`: Requires EKS access entries. Learn more about xref:grant-k8s-access[EKS authentication modes]. +* `accessConfig.authenticationMode`: Requires EKS access entries. Learn more about <>. -#### Run the Command +==== Run the Command Use the following command to create the cluster: @@ -299,9 +278,9 @@ aws eks create-cluster \ } ``` -### **Check Cluster Status** +=== Check Cluster Status -#### Step 1: Verify Cluster Creation +==== Step 1: Verify Cluster Creation Run the following command to check the status of your cluster. Cluster creation typically takes about 15 minutes: @@ -309,7 +288,7 @@ Run the following command to check the status of your cluster. Cluster creation aws eks describe-cluster --name "${CLUSTER_NAME}" --output json ``` -#### Step 2: Update kubeconfig +==== Step 2: Update kubeconfig Once the cluster is ready, update your local kubeconfig file to enable `kubectl` to communicate with the cluster. This configuration uses the {aws} CLI for authentication. @@ -317,7 +296,7 @@ Once the cluster is ready, update your local kubeconfig file to enable `kubectl` aws eks update-kubeconfig --name "${CLUSTER_NAME}" ``` -#### Step 3: Verify Node Pools +==== Step 3: Verify Node Pools List the Node Pools in your cluster using the following command: @@ -327,4 +306,4 @@ kubectl get nodepools == Next Steps -* Learn how to xref:automode-workload[deploy a sample workload] to your new EKS Auto Mode cluster. +* Learn how to <> to your new EKS Auto Mode cluster. \ No newline at end of file diff --git a/latest/ug/automode/automode-get-started-console.adoc b/latest/ug/automode/automode-get-started-console.adoc index 714f2b2ae..b38c76d64 100644 --- a/latest/ug/automode/automode-get-started-console.adoc +++ b/latest/ug/automode/automode-get-started-console.adoc @@ -1,54 +1,49 @@ -//!!NODE_ROOT
- include::../attributes.txt[] [.topic] -[[automode-get-started-console,automode-get-started-console.title]] -= Create an EKS Auto Mode Cluster with the {aws} Management Console -:info_doctype: section -:info_title: Create an EKS Auto Mode Cluster with the {aws} Management Console +[#automode-get-started-console] += Create an EKS Auto Mode Cluster with the {aws-management-console} :info_titleabbrev: Management console -:info_abstract: Create an EKS Auto Mode cluster with the {aws} Management Console -Creating an {eam} cluster in the {aws} Management Console requires less {config} than other options. EKS integrates with {aws} IAM and VPC Networking to help you create the resources associated with an EKS cluster. +Creating an EKS Auto Mode cluster in the {aws-management-console} requires less configuration than other options. EKS integrates with {aws} IAM and VPC Networking to help you create the resources associated with an EKS cluster. You have two options to create a cluster in the console: -* Quick {config} (with EKS Auto Mode) -* Custom {config} +* Quick configuration (with EKS Auto Mode) +* Custom configuration -In this topic, you will learn how to create an {eam} cluster using the Quick {config} option. +In this topic, you will learn how to create an EKS Auto Mode cluster using the Quick configuration option. -== Create an EKS Auto Mode using the quick {config} option +== Create an EKS Auto Mode using the quick configuration option -You must be logged into the {aws} management console with sufficent permissions to manage {aws} resources including: EC2 instances, EC2 networking, EKS clusters, and IAM roles. +You must be logged into the {aws-management-console} with sufficent permissions to manage {aws} resources including: EC2 instances, EC2 networking, EKS clusters, and IAM roles. . Navigate to the EKS Console . Click *Create cluster* -. Confirm the *Quick {config}* option is selected +. Confirm the *Quick configuration* option is selected . Determine the following values, or use the defaults for a test cluster. ** Cluster *Name* ** Kubernetes Version -. Select the Cluster IAM Role. If this is your first time creating an {eam} cluster, use the *Create {recd} role* option. -** Optionally, you can reuse a single Cluster IAM Role in {yaa} for all {eam} clusters. -** The Cluster IAM Role includes required permissions for {eam} to manage resources including EC2 instances, EBS volumes, and EC2 load balancers. -** The *Create {recd} role* option pre-fills all fields with {recd} values. Select *Next* and then *Create*. The role will use the suggested `AmazonEKSAutoClusterRole` name. +. Select the Cluster IAM Role. If this is your first time creating an EKS Auto Mode cluster, use the *Create recommended role* option. +** Optionally, you can reuse a single Cluster IAM Role in your {aws} account for all EKS Auto Mode clusters. +** The Cluster IAM Role includes required permissions for EKS Auto Mode to manage resources including EC2 instances, EBS volumes, and EC2 load balancers. +** The *Create recommended role* option pre-fills all fields with recommended values. Select *Next* and then *Create*. The role will use the suggested `AmazonEKSAutoClusterRole` name. ** If you recently created a new role, use the *Refresh* icon to reload the role selection dropdown. -. Select the Node IAM Role. If this is your first time creating an {eam} cluster, use the *Create {recd} role* option. -** Optionally, you can reuse a single Node IAM Role in {yaa} for all {eam} clusters. -** The Node IAM Role includes required permissions for Auto Mode nodes to connect to the cluster. The Node IAM Role must include permissions to {ret} ECR images for your containers. -** The *Create {recd} role* option pre-fills all fields with {recd} values. Select *Next* and then *Create*. The role will use the suggested `AmazonEKSAutoNodeRole` name. +. Select the Node IAM Role. If this is your first time creating an EKS Auto Mode cluster, use the *Create recommended role* option. +** Optionally, you can reuse a single Node IAM Role in your {aws} account for all EKS Auto Mode clusters. +** The Node IAM Role includes required permissions for Auto Mode nodes to connect to the cluster. The Node IAM Role must include permissions to retrieve ECR images for your containers. +** The *Create recommended role* option pre-fills all fields with recommended values. Select *Next* and then *Create*. The role will use the suggested `AmazonEKSAutoNodeRole` name. ** If you recently created a new role, use the *Refresh* icon to reload the role selection dropdown. -. Select the VPC for {yec}. Choose the *Create VPC* to create a new VPC for EKS, or choose a VPC you previously created for EKS. +. Select the VPC for your EKS Auto Mode cluster. Choose the *Create VPC* to create a new VPC for EKS, or choose a VPC you previously created for EKS. ** If you use the VPC Console to create a new VPC, {aws} suggests you create at least one NAT Gateway per Availability Zone. Otherwise, you can use all other defaults. ** For more information and details of IPv6 cluster requirements, see <>. -. (optional) {eam} automatically populates the private subnets for your selected VPC. You can remove unwanted subnets. +. (optional) EKS Auto Mode automatically populates the private subnets for your selected VPC. You can remove unwanted subnets. ** EKS automatically selects private subnets from the VPC following best practices. You can optionally select additional subnets from the VPC, such as public subnets. -. (optional) Select *View quick configuration defaults* to review all {config} values for the new cluster. The table indicates some values are not editable after the cluster is created. +. (optional) Select *View quick configuration defaults* to review all configuration values for the new cluster. The table indicates some values are not editable after the cluster is created. . Select *Create cluster* . Note it may take fifteen minutes for cluster creation to complete. == Next Steps -* Learn how to xref:sample-storage-workload[Deploy a Sample Workload to {yec}] +* Learn how to <> -//call out refactored IAM +//call out refactored IAM \ No newline at end of file diff --git a/latest/ug/automode/automode-get-started-eksctl.adoc b/latest/ug/automode/automode-get-started-eksctl.adoc index 8677b35bf..9a6bcf5b2 100644 --- a/latest/ug/automode/automode-get-started-eksctl.adoc +++ b/latest/ug/automode/automode-get-started-eksctl.adoc @@ -1,16 +1,10 @@ -//!!NODE_ROOT
+include::../attributes.txt[] [.topic] -[[automode-get-started-eksctl,automode-get-started-eksctl.title]] +[#automode-get-started-eksctl] = Create an EKS Auto Mode Cluster with the eksctl CLI -:info_doctype: section :config: configuration -:info_title: Create an EKS Auto Mode Cluster with the eksctl CLI :info_titleabbrev: eksctl CLI -:info_abstract: Create an EKS Auto Mode cluster with the eksctl CLI - - -include::../attributes.txt[] This topic shows you how to create an Amazon EKS Auto Mode cluster using the eksctl command line interface (CLI). You can create an Auto Mode cluster either by running a single CLI command or by applying a YAML configuration file. Both methods provide the same functionality, with the YAML approach offering more granular control over cluster settings. @@ -22,11 +16,11 @@ You must install version `0.195.0` or greater of eksctl. For more information, s ==== -== Create an {eam} cluster with a CLI command +== Create an EKS Auto Mode cluster with a CLI command You must have the `aws` and `eksctl` tools installed. You must be logged into the {aws} CLI with sufficent permissions to manage {aws} resources including: EC2 instances, EC2 networking, EKS clusters, and IAM roles. For more information, see <>. -Run the following command to create a new {eam} cluster with +Run the following command to create a new EKS Auto Mode cluster with [source,cli] ---- @@ -36,16 +30,16 @@ eksctl create cluster --name= --enable-auto-mode //Cluster IAM Role? //Update kubeconfig? -== Create an {eam} cluster with a YAML file +== Create an EKS Auto Mode cluster with a YAML file :enai: enabling You must have the `aws` and `eksctl` tools installed. You must be logged into the {aws} CLI with sufficent permissions to manage {aws} resources including: EC2 instances, EC2 networking, EKS clusters, and IAM roles. For more information, see <>. -Review the {eam} configuration options in the sample ClusterConfig resource below. For the full ClusterConfig specification, see the https://eksctl.io/usage/creating-and-managing-clusters/[eksctl documentation]. +Review the EKS Auto Mode configuration options in the sample ClusterConfig resource below. For the full ClusterConfig specification, see the https://eksctl.io/usage/creating-and-managing-clusters/[eksctl documentation]. -{aws} suggests {enai} {eam}. If this is your first time creating an {eam} cluster, leave the `nodeRoleARN` unspecified to create a Node IAM Role for {eam}. If you already have a Node IAM Role in {yaa}, {aws} suggests reusing it. +{aws} suggests {enai} EKS Auto Mode. If this is your first time creating an EKS Auto Mode cluster, leave the `nodeRoleARN` unspecified to create a Node IAM Role for EKS Auto Mode. If you already have a Node IAM Role in your {aws} account, {aws} suggests reusing it. -{aws} suggests not specifying any value for `nodePools`. {eam} will create default node pools. You can use the Kubernetes API to create additional node pools. +{aws} suggests not specifying any value for `nodePools`. EKS Auto Mode will create default node pools. You can use the Kubernetes API to create additional node pools. [source,yaml] ---- @@ -80,4 +74,4 @@ Save the `ClusterConfig` file as `cluster.yaml`, and use the following command t [source,cli] ---- eksctl create cluster -f cluster.yaml ----- +---- \ No newline at end of file diff --git a/latest/ug/automode/automode-learn-instances.adoc b/latest/ug/automode/automode-learn-instances.adoc index 57220f30f..b2ca96e97 100644 --- a/latest/ug/automode/automode-learn-instances.adoc +++ b/latest/ug/automode/automode-learn-instances.adoc @@ -1,22 +1,13 @@ -//!!NODE_ROOT
+include::../attributes.txt[] [.topic] -[[automode-learn-instances,automode-learn-instances.title]] +[#automode-learn-instances] = Learn about Amazon EKS Auto Mode Managed instances -:info_doctype: section :am: EKS Auto Mode :aam: Amazon {am} :ec2i: EC2 Instance :mi: managed instance -:emi: EC2 managed instance -:emi: EC2 {mi} -:emng: EKS Managed Node Group -:info_title: Learn about Amazon EKS Auto Mode managed instances :info_titleabbrev: Managed instances -:info_abstract: Learn about Amazon EKS Auto Mode managed instances - -include::../attributes.txt[] - This topic explains how {aam} manages Amazon EC2 instances in your EKS cluster. When you enable {am}, your cluster's compute resources are automatically provisioned and managed by EKS, changing how you interact with the EC2 instances that serve as nodes in your cluster. @@ -28,38 +19,38 @@ Understanding how {aam} manages instances is essential for planning your workloa {ec2i}s created by {am} are different from other {ec2i}s, they are {mi}s. These {mi}s are owned by EKS and are more restricted. You can't directly access or install software on instances managed by {am}. -{aws} suggests running either {eam} or self-managed Karpenter. You can install both during a migration or in an advanced configuration. If you have both installed, configure your node pools so that workloads are associated with either Karpenter or {eam}. +{aws} suggests running either EKS Auto Mode or self-managed Karpenter. You can install both during a migration or in an advanced configuration. If you have both installed, configure your node pools so that workloads are associated with either Karpenter or EKS Auto Mode. For more information, see link:AWSEC2/latest/UserGuide/amazon-ec2-managed-instances.html["Amazon EC2 managed instances",type="documentation"] in the Amazon EC2 user guide. == Comparison table -[cols="1,1", options="header"] +[%header,cols="2"] |=== +|Standard {ec2i} +|{am} {mi} -| Standard {ec2i} -| {am} {mi} -| You are responsible for patching and updating the instance. -| {aws} automatically patches and updates the instance. +|You are responsible for patching and updating the instance. +|{aws} automatically patches and updates the instance. -| EKS is not responsible for the software on the instance. -| EKS is responsible for certain software on the instance, such as `kubelet`, the container runtime, and the operating system. +|EKS is not responsible for the software on the instance. +|EKS is responsible for certain software on the instance, such as `kubelet`, the container runtime, and the operating system. -| You can delete the {ec2i} using the EC2 API. -| EKS determines the number of instances deployed in your account. If you delete a workload, EKS will reduce the number of instances in your account. +|You can delete the {ec2i} using the EC2 API. +|EKS determines the number of instances deployed in your account. If you delete a workload, EKS will reduce the number of instances in your account. -| You can use SSH to access the {ec2i}. -| You can deploy pods and containers to the {mi}. +|You can use SSH to access the {ec2i}. +|You can deploy pods and containers to the {mi}. -| You determine the operating system and image (AMI). -| {aws} determines the operating system and image. +|You determine the operating system and image (AMI). +|{aws} determines the operating system and image. -| You can deploy workloads that rely on Windows or Ubuntu functionality. -| You can deploy containers based on Linux, but without specific OS dependencies. +|You can deploy workloads that rely on Windows or Ubuntu functionality. +|You can deploy containers based on Linux, but without specific OS dependencies. -| You determine what instance type and family to launch. -| {aws} determines what instance type and family to launch. You can use a Node Pool to limit the instance types {eam} selects from. +|You determine what instance type and family to launch. +|{aws} determines what instance type and family to launch. You can use a Node Pool to limit the instance types EKS Auto Mode selects from. |=== @@ -69,13 +60,20 @@ The following functionality works for both Managed instances and Standard EC2 in * You can view the instance in the {aws} console. * You can use instance storage as ephemeral storage for workloads. -== Supported instance reference +=== AMI Support -// Source: https://code.amazon.com/packages/EKSKarpenterController/blobs/a56aeb0ddc3e8a54406421e8f3a091e8e13abea1/--/pkg/providers/instancetype/instancetype.go#L43-L49 +With EKS Auto Mode, {aws} determines the image (AMI) used for your compute nodes. {aws} monitors the rollout of new EKS Auto Mode AMI versions. If you experience workload issues related to an AMI version, create a support case. For more information, see link:awssupport/latest/user/case-management.html["Creating support cases and case management",type="documentation"] in the {aws} Support User Guide. + +Generally, EKS releases a new AMI each week containing CVE and security fixes. + +[#auto-supported-instances] +== EKS Auto Mode supported instance reference + +EKS Auto Mode only creates instances of supported types, and that meet a minimum size requirement. EKS Auto Mode supports the following instance types: -[cols="1,4",options="header"] +[%header,cols="1,4"] |=== |Family |Instance Types @@ -95,18 +93,50 @@ EKS Auto Mode supports the following instance types: |z1d, x8g, x2gd |Storage Optimized (I/D) -|i4g, i4i, i3, i3en, is4gen, d3, d3en, im4gn +|i8g, i7ie, i4g, i4i, i3, i3en, is4gen, d3, d3en, im4gn |Accelerated Computing (P/G/Inf/Trn) -|p5, p4d, p3, p3dn, gr6, g6, g6e, g5g, g5, g4dn, inf2, inf1, trn1, trn1n +|p5, p4d, p4de, p3, p3dn, gr6, g6, g6e, g5g, g5, g4dn, inf2, inf1, trn1, trn1n |High Performance Computing (X2) |x2iezn, x2iedn, x2idn |=== +Additionally, EKS Auto Mode will only create EC2 instances that meet the following requirements: + +* More than 1 CPU +* Instance size is not nano, micro or small + +For more information, see link:ec2/latest/instancetypes/instance-type-names.html["Amazon EC2 instance type naming conventions",type="documentation"]. + +== Instance Metadata Service + +* EKS Auto Mode enforces IMDSv2 with a hop limit of 1 by default, adhering to {aws} security best practices. +* This default configuration cannot be modified in Auto Mode. +* For add-ons that typically require IMDS access, supply parameters (such as {aws} region) during installation to avoid IMDS lookups. For more information, see <>. +* If a Pod absolutely requires IMDS access when running in Auto Mode, the Pod must be configured to run with `hostNetwork: true`. This allows the Pod to access the instance metadata service directly. +* Consider the security implications when granting Pods access to instance metadata. + +For more information about the Amazon EC2 Instance Metadata Service (IMDS), see link:AWSEC2/latest/UserGuide/configuring-instance-metadata-options.html[Configure the Instance Metadata Service options,type="documentation"] in the _Amazon EC2 User Guide_. + == Considerations -* EKS Auto Mode automatically formats and configures NVMe local storage on supported instance types. For nodes with multiple NVMe drives, EKS sets up a RAID 0 array. This automation eliminates the need for manual formatting and RAID configuration of local NVMe storage in EKS clusters. -* Amazon EKS Auto Mode does not support {aws} Fault Injection Service. For more information, see link:resilience-hub/latest/userguide/testing.html["Managing Fault Injection Service experiments",type="documentation"] in the {aws} Resilience Hub User Guide. -* You do not need to install the `Neuron Device Plugin` on EKS Auto Mode nodes. -** If you have other types of nodes in your cluster, you need to configure the Neuron Device plugin to not run on auto mode nodes. For more information, see <>. +* If the configured ephemeral storage in the NodeClass is smaller than the NVMe local storage for the instance, EKS Auto Mode eliminates the need for manual configuration by automatically taking the following actions: +** Uses a smaller (20 GiB) Amazon EBS data volume to reduce costs. +** Formats and configures the NVMe local storage for ephemeral data use. This includes setting up a RAID 0 array if there are multiple NVMe drives. +* When `ephemeralStorage.size` equals or exceeds the local NVMe capacity, the following actions occur: +** Auto Mode skips the small EBS volume. +** The NVMe drive(s) are exposed directly for your workload. +* Amazon EKS Auto Mode does not support the following {aws} Fault Injection Service actions: +** `ec2:RebootInstances` +** `ec2:SendSpotInstanceInterruptions` +** `ec2:StartInstances` +** `ec2:StopInstances` +** `ec2:TerminateInstances` +** `ec2:PauseVolumeIO` +* Amazon EKS Auto Mode supports {aws} Fault Injection Service EKS Pod actions. For more information, see link:resilience-hub/latest/userguide/testing.html[Managing Fault Injection Service experiments,type="documentation"] and link:fis/latest/userguide/eks-pod-actions.html#configure-service-account[Use the {aws} FIS aws:eks:pod actions,type="documentation"] in the {aws} Resilience Hub User Guide. +* You do not need to install the `Neuron Device Plugin` on EKS Auto Mode nodes. ++ +If you have other types of nodes in your cluster, you need to configure the Neuron Device plugin to not run on Auto Mode nodes. For more information, see <>. + + diff --git a/latest/ug/automode/automode-workload.adoc b/latest/ug/automode/automode-workload.adoc index 015a1856e..0198e5c97 100644 --- a/latest/ug/automode/automode-workload.adoc +++ b/latest/ug/automode/automode-workload.adoc @@ -1,20 +1,15 @@ -//!!NODE_ROOT
+include::../attributes.txt[] [.topic] -[[automode-workload,automode-workload.title]] +[#automode-workload] = Deploy a sample inflate workload to an Amazon EKS Auto Mode cluster -:info_doctype: section -:info_title: Deploy a sample inflate workload to an Amazon EKS Auto Mode cluster :info_titleabbrev: Deploy inflate workload -:info_abstract: Deploy a sample inflate workload to an Amazon EKS Auto Mode cluster - -include::../attributes.txt[] In this tutorial, you'll learn how to deploy a sample workload to an EKS Auto Mode cluster and observe how it automatically provisions the required compute resources. You'll use `kubectl` commands to watch the cluster's behavior and see firsthand how Auto Mode simplifies Kubernetes operations on {aws}. By the end of this tutorial, you'll understand how EKS Auto Mode responds to workload deployments by automatically managing the underlying compute resources, without requiring manual node group configuration. == Prerequisites -* An Amazon EKS Auto Mode cluster with the compute capability enabled. Note the name and {aws} region of the cluster. +* An Amazon EKS Auto Mode cluster. Note the name and {aws} region of the cluster. * An IAM principal, such as a user or role, with sufficent permissions to manage networking, compute, and EKS resources. ** For more information, see link:IAM/latest/UserGuide/access_policies_job-functions_create-policies.html["Creating roles and attaching policies in the IAM User Guide",type="documentation"] in the IAM User Guide. * `aws` CLI installed and configured with an IAM identity. @@ -131,4 +126,4 @@ If you have no other workloads deployed to your cluster, the node created by EKS In the default configration, EKS Auto Mode detects nodes that have been empty for thirty seconds, and terminates them. -Use `kubectl` or the EC2 console to confirm the associated instance has been deleted. +Use `kubectl` or the EC2 console to confirm the associated instance has been deleted. \ No newline at end of file diff --git a/latest/ug/automode/automode.adoc b/latest/ug/automode/automode.adoc index a9a6353a3..2d41f3088 100644 --- a/latest/ug/automode/automode.adoc +++ b/latest/ug/automode/automode.adoc @@ -1,51 +1,45 @@ -//!!NODE_ROOT +include::../attributes.txt[] -[[automode,automode.title]] +[.topic] +[#automode] = Automate cluster infrastructure with EKS Auto Mode -:info_doctype: chapter -:toclevels: 2 -:toc: -:info_title: Automate cluster infrastructure with EKS Auto Mode :info_titleabbrev: EKS Auto Mode -:info_abstract: Automate cluster infrastructure with EKS Auto Mode [abstract] -- Automate cluster infrastructure with EKS Auto Mode -- -include::../attributes.txt[] - EKS Auto Mode extends {aws} management of Kubernetes clusters beyond the cluster itself, to allow {aws} to also set up and manage the infrastructure that enables the smooth operation of your workloads. You can delegate key infrastructure decisions and leverage the expertise of {aws} for day-to-day operations. Cluster infrastructure managed by {aws} includes many Kubernetes capabilities as core components, as opposed to add-ons, such as compute autoscaling, pod and service networking, application load balancing, cluster DNS, block storage, and GPU support. To get started, you can deploy a new EKS Auto Mode cluster or enable EKS Auto Mode on an existing cluster. -You can deploy, upgrade, or modify your EKS Auto Mode clusters using eksctl, the {aws} CLI, the {aws} Management Console, EKS APIs, or your preferred infrastructure-as-code tools. +You can deploy, upgrade, or modify your EKS Auto Mode clusters using eksctl, the {aws} CLI, the {aws-management-console}, EKS APIs, or your preferred infrastructure-as-code tools. With EKS Auto Mode, you can continue using your preferred Kubernetes-compatible tools. EKS Auto Mode integrates with {aws} services like Amazon EC2, Amazon EBS, and ELB, leveraging {aws} cloud resources that follow best practices. These resources are automatically scaled, cost-optimized, and regularly updated to help minimize operational costs and overhead. -## Features +== Features EKS Auto Mode provides the following high-level features: -**Streamline Kubernetes Cluster Management**: EKS Auto Mode streamlines EKS management by providing production-ready clusters with minimal operational overhead. With EKS Auto Mode, you can run demanding, dynamic workloads confidently, without requiring deep EKS expertise. +*Streamline Kubernetes Cluster Management*: EKS Auto Mode streamlines EKS management by providing production-ready clusters with minimal operational overhead. With EKS Auto Mode, you can run demanding, dynamic workloads confidently, without requiring deep EKS expertise. -**Application Availability**: EKS Auto Mode dynamically adds or removes nodes in your EKS cluster based on the demands of your Kubernetes applications. This minimizes the need for manual capacity planning and ensures application availability. +*Application Availability*: EKS Auto Mode dynamically adds or removes nodes in your EKS cluster based on the demands of your Kubernetes applications. This minimizes the need for manual capacity planning and ensures application availability. //what? -**Efficiency**: EKS Auto Mode is designed to compute costs while adhering to the flexibility defined by your NodePool and workload requirements. It also terminates unused instances and consolidates workloads onto other nodes to improve cost efficiency. +*Efficiency*: EKS Auto Mode is designed to optimize compute costs while adhering to the flexibility defined by your NodePool and workload requirements. It also terminates unused instances and consolidates workloads onto other nodes to improve cost efficiency. -**Security**: EKS Auto Mode uses AMIs that are treated as immutable for your nodes. These AMIs enforce locked-down software, enable SELinux mandatory access controls, and provide read-only root file systems. Additionally, nodes launched by EKS Auto Mode have a maximum lifetime of 21 days (which you can reduce), after which they are automatically replaced with new nodes. This approach enhances your security posture by regularly cycling nodes, aligning with best practices already adopted by many customers. +*Security*: EKS Auto Mode uses AMIs that are treated as immutable, for your nodes. These AMIs enforce locked-down software, enable SELinux mandatory access controls, and provide read-only root file systems. Additionally, nodes launched by EKS Auto Mode have a maximum lifetime of 21 days (which you can reduce), after which they are automatically replaced with new nodes. This approach enhances your security posture by regularly cycling nodes, aligning with best practices already adopted by many customers. -**Automated Upgrades**: EKS Auto Mode keeps your Kubernetes cluster, nodes, and related components up to date with the latest patches, while respecting your configured Pod Disruption Budgets (PDBs) and NodePool Disruption Budgets (NDBs). Up to the 21-day maximum lifetime, intervention might be required if blocking PDBs or other configurations prevent updates. +*Automated Upgrades*: EKS Auto Mode keeps your Kubernetes cluster, nodes, and related components up to date with the latest patches, while respecting your configured Pod Disruption Budgets (PDBs) and NodePool Disruption Budgets (NDBs). Up to the 21-day maximum lifetime, intervention might be required if blocking PDBs or other configurations prevent updates. -**Managed Components**: EKS Auto Mode includes Kubernetes and {aws} cloud features as core components that would otherwise have to be managed as add-ons. This includes built-in support for Pod IP address assignments, Pod network policies, local DNS services, GPU plug-ins, health checkers, and EBS CSI storage. +*Managed Components*: EKS Auto Mode includes Kubernetes and {aws} cloud features as core components that would otherwise have to be managed as add-ons. This includes built-in support for Pod IP address assignments, Pod network policies, local DNS services, GPU plug-ins, health checkers, and EBS CSI storage. -**Customizable NodePools and NodeClasses**: If your workload requires changes to storage, compute, or networking configurations, you can create custom NodePools and NodeClasses using EKS Auto Mode. While default NodePools and NodeClasses can't be edited, you can add new custom NodePools or NodeClasses alongside the default configurations to meet your specific requirements. +*Customizable NodePools and NodeClasses*: If your workload requires changes to storage, compute, or networking configurations, you can create custom NodePools and NodeClasses using EKS Auto Mode. While you should not edit default NodePools and NodeClasses, you can add new custom NodePools or NodeClasses alongside the default configurations to meet your specific requirements. -## Automated Components +== Automated Components EKS Auto Mode streamlines the operation of your Amazon EKS clusters by automating key infrastructure components. Enabling EKS Auto Mode further reduces the tasks to manage your EKS clusters. @@ -54,9 +48,10 @@ The following is a list of data plane components that are automated: * *Compute*: For many workloads, with EKS Auto Mode you can forget about many aspects of compute for your EKS clusters. These include: ** *Nodes*: EKS Auto Mode nodes are designed to be treated like appliances. EKS Auto Mode does the following: *** Chooses an appropriate AMI that's configured with many services needed to run your workloads without intervention. -*** Locks down those features using SELinux enforcing mode and a read-only root file system. +*** Locks down access to files on the AMI using SELinux enforcing mode and a read-only root file system. *** Prevents direct access to the nodes by disallowing SSH or SSM access. *** Includes GPU support, with separate kernel drivers and plugins for NVIDIA and Neuron GPUs, enabling high-performance workloads. +*** Automatically handles link:AWSEC2/latest/UserGuide/spot-instance-termination-notices.html[EC2 Spot Instance interruption notices,type="documentation"] and EC2 Instance health events ** *Auto scaling*: Relying on https://karpenter.sh/docs/[Karpenter] auto scaling, EKS Auto Mode monitors for unschedulable Pods and makes it possible for new nodes to be deployed to run those pods. As workloads are terminated, EKS Auto Mode dynamically disrupts and terminates nodes when they are no longer needed, optimizing resource usage. ** *Upgrades*: Taking control of your nodes streamlines EKS Auto Mode's ability to provide security patches and operating system and component upgrades as needed. Those upgrades are designed to provide minimal disruption of your workloads. EKS Auto Mode enforces a 21-day maximum node lifetime to ensure up-to-date software and APIs. * *Load balancing*: EKS Auto Mode streamlines load balancing by integrating with Amazon's Elastic Load Balancing service, automating the provisioning and configuration of load balancers for Kubernetes Services and Ingress resources. It supports advanced features for both Application and Network Load Balancers, manages their lifecycle, and scales them to match cluster demands. This integration provides a production-ready load balancing solution adhering to {aws} best practices, allowing you to focus on applications rather than infrastructure management. @@ -66,13 +61,13 @@ The following is a list of data plane components that are automated: For more information about these components, see <>. -## Configuration +== Configuration While EKS Auto Mode will effectively manage most of your data plane services without your intervention, there might be times when you want to change the behavior of some of those services. You can modify the configuration of your EKS Auto Mode clusters in the following ways: * *Kubernetes DaemonSets*: Rather than modify services installed on your nodes, you can instead use Kubernetes daemonsets. Daemonsets are designed to be managed by Kubernetes, but run on every node in the cluster. In this way, you can add special services for monitoring or otherwise watching over your nodes. -* *Custom NodePools and NodeClasses*: Default NodePools and NodeClasses are configured by EKS Auto Mode and can't be edited. To customize node behavior, you can create additional NodePools or NodeClasses for use cases such as: +* *Custom NodePools and NodeClasses*: Default NodePools and NodeClasses are configured by EKS Auto Mode and you should not edit them. To customize node behavior, you can create additional NodePools or NodeClasses for use cases such as: ** Selecting specific instance types (for example, accelerated processors or EC2 Spot instances). ** Isolating workloads for security or cost-tracking purposes. @@ -82,7 +77,28 @@ While EKS Auto Mode will effectively manage most of your data plane services wit For more information about options for configuring EKS Auto Mode, see <>. +== Shared responsibility model + +The {aws} Shared Responsibility Model defines security and compliance responsibilities between {aws} and customers. +The images and text below compare and contrast how customer and {aws} responsibilities differ between EKS Auto Mode and EKS standard mode. + +image::images/eksautosrm.png[Shared responsibility model with EKS Auto Mode and standard mode] + +EKS Auto Mode shifts much of the shared responsibility for Kubernetes infrastructure from customers to {aws}. With EKS Auto Mode, {aws} takes on more responsibility for cloud security, which was once the customer’s responsibility and is now shared. Customers can now focus more on their applications while {aws} manages the underlying infrastructure. +**Customer responsibility** + +Under EKS Auto Mode, customers continue to maintain responsibility for the application containers, including availability, security, and monitoring. They also maintain control over VPC infrastructure and EKS cluster configuration. This model lets customers concentrate on application-specific concerns while delegating cluster infrastructure management to {aws}. Optional per-node features can be included in clusters through {aws} add-ons. + +**{aws} responsibility** + +With EKS Auto Mode, {aws} expands its responsibility to include the management of several additional critical components compared to those already managed in EKS clusters not using Auto Mode. In particular, EKS Auto Mode takes over the configuration, management, security, and scaling of the EC2 instances launched as well as cluster capabilities for load balancing, IP address management, networking policy, and block storage. The following components are managed by {aws} in EKS Auto Mode: + +* *Auto Mode-launched EC2 Instances*: {aws} handles the complete lifecycle of nodes by leveraging Amazon EC2 managed instances. EC2 managed instances take responsibility for operating system configuration, patching, monitoring, and health maintenance. In this model, both the instance itself and the guest operating system running on it are the responsibility of {aws}. The nodes use variants of https://aws.amazon.com/bottlerocket[Bottlerocket] AMIs that are optimized to run containers. The Bottlerocket AMIs have locked-down software, immutable root file systems, and secure network access (to prevent direct communications through SSH or SSM). + +* *Cluster Capabilities*: {aws} manages compute autoscaling, Pod networking with network policy enforcement, Elastic Load Balancing integration, and storage drivers configuration. +* *Cluster Control Plane*: {aws} continues to manage the Kubernetes API server, cross-account ENIs, and the etcd database, as with standard EKS. +* *Foundation Services and Global Infrastructure*: {aws} maintains responsibility for the underlying compute, storage, networking, and monitoring services, as well as the global infrastructure of regions, local zones, and edge locations. include::create-auto.adoc[leveloffset=+1] @@ -96,4 +112,6 @@ include::auto-reference.adoc[leveloffset=+1] include::auto-troubleshoot.adoc[leveloffset=+1] +include::auto-change.adoc[leveloffset=+1] + //include::wip.adoc[leveloffset=+1] diff --git a/latest/ug/automode/create-auto.adoc b/latest/ug/automode/create-auto.adoc index 10221587f..49275f99f 100644 --- a/latest/ug/automode/create-auto.adoc +++ b/latest/ug/automode/create-auto.adoc @@ -1,27 +1,16 @@ -//!!NODE_ROOT -[.topic] include::../attributes.txt[] -[[create-auto,create-auto.title]] + +[.topic] +[#create-auto] = Create a cluster with Amazon EKS Auto Mode -:info_doctype: section -:icons: font -:experimental: -:idprefix: -:idseparator: - -:sourcedir: . -:info_title: Create cluster with EKS Auto Mode :info_titleabbrev: Create cluster -:info_abstract: Learn about the tools needed for creating and working with an Amazon EKS cluster in EKS Auto Mode. -:keywords: getting, started, tutorials, quick, start [abstract] -- Learn about the tools needed for creating and working with an Amazon EKS cluster in EKS Auto Mode. -- - - -This chapter explains how to create an Amazon EKS cluster with Auto Mode enabled using various tools and interfaces. Auto Mode simplifies cluster creation by automatically configuring and managing the cluster's compute, networking, and storage infrastructure. You'll learn how to create an Auto Mode cluster using the {aws} CLI, {aws} Management Console, or the eksctl command line tool. +This chapter explains how to create an Amazon EKS cluster with Auto Mode enabled using various tools and interfaces. Auto Mode simplifies cluster creation by automatically configuring and managing the cluster's compute, networking, and storage infrastructure. You'll learn how to create an Auto Mode cluster using the {aws} CLI, {aws-management-console}, or the eksctl command line tool. [NOTE] ==== @@ -29,11 +18,11 @@ EKS Auto Mode requires Kubernetes version 1.29 or greater. ==== -Choose your preferred tool based on your needs: The {aws} Management Console provides a visual interface ideal for learning about EKS Auto Mode features and creating individual clusters. The {aws} CLI is best suited for scripting and automation tasks, particularly when integrating cluster creation into existing workflows or CI/CD pipelines. The eksctl CLI offers a Kubernetes-native experience and is recommended for users familiar with Kubernetes tooling who want simplified command line operations with sensible defaults. +Choose your preferred tool based on your needs: The {aws-management-console} provides a visual interface ideal for learning about EKS Auto Mode features and creating individual clusters. The {aws} CLI is best suited for scripting and automation tasks, particularly when integrating cluster creation into existing workflows or CI/CD pipelines. The eksctl CLI offers a Kubernetes-native experience and is recommended for users familiar with Kubernetes tooling who want simplified command line operations with sensible defaults. Before you begin, ensure you have the necessary prerequisites installed and configured, including appropriate IAM permissions to create EKS clusters. To learn how to install CLI tools such as `kubectl`, `aws`, and `eksctl`, see <>. -You can use the {aws} CLI, {aws} Management Console, or eksctl CLI to create a cluster with Amazon EKS Auto Mode. +You can use the {aws} CLI, {aws-management-console}, or eksctl CLI to create a cluster with Amazon EKS Auto Mode. [.topiclist] [[Topic List]] diff --git a/latest/ug/automode/create-node-class.adoc b/latest/ug/automode/create-node-class.adoc index cf1bb913d..e76177b37 100644 --- a/latest/ug/automode/create-node-class.adoc +++ b/latest/ug/automode/create-node-class.adoc @@ -1,20 +1,17 @@ -//!!NODE_ROOT
+include::../attributes.txt[] [.topic] -[[create-node-class,create-node-class.title]] +[#create-node-class] = Create a Node Class for Amazon EKS -:info_doctype: section :info_titleabbrev: Create node class -include::../attributes.txt[] - -Amazon EKS Node Classes provide granular control over the configuration of your {eam} managed nodes. A Node Class defines infrastructure-level settings that apply to groups of nodes in your EKS cluster, including network configuration, storage settings, and resource tagging. This topic explains how to create and configure a Node Class to meet your specific operational requirements. +Amazon EKS Node Classes are templates that offer granular control over the configuration of your EKS Auto Mode managed nodes. A Node Class defines infrastructure-level settings that apply to groups of nodes in your EKS cluster, including network configuration, storage settings, and resource tagging. This topic explains how to create and configure a Node Class to meet your specific operational requirements. -When you need to customize how {eam} provisions and configures EC2 instances beyond the default settings, creating a Node Class gives you precise control over critical infrastructure parameters. For example, you can specify private subnet placement for enhanced security, configure instance ephemeral storage for performance-sensitive workloads, or apply custom tagging for cost allocation. +When you need to customize how EKS Auto Mode provisions and configures EC2 instances beyond the default settings, creating a Node Class gives you precise control over critical infrastructure parameters. For example, you can specify private subnet placement for enhanced security, configure instance ephemeral storage for performance-sensitive workloads, or apply custom tagging for cost allocation. -## Create a Node Class +== Create a Node Class -To create a Node Class, follow these steps: +To create a `NodeClass`, follow these steps: . Create a YAML file (for example, `nodeclass.yaml`) with your Node Class configuration . Apply the configuration to your cluster using `kubectl` @@ -22,7 +19,7 @@ To create a Node Class, follow these steps: You need `kubectl` installed and configured. For more information, see <>. -### Basic Node Class Example +=== Basic Node Class Example Here's an example Node Class: @@ -34,13 +31,20 @@ kind: NodeClass metadata: name: private-compute spec: + subnetSelectorTerms: + - tags: + Name: "private-subnet" + kubernetes.io/role/internal-elb: "1" + securityGroupSelectorTerms: + - tags: + Name: "eks-cluster-sg" ephemeralStorage: size: "160Gi" ``` This NodeClass increases the amount of ephemeral storage on the node. -Apply this configuration using: +Apply this configuration by using: ```bash kubectl apply -f nodeclass.yaml @@ -48,58 +52,242 @@ kubectl apply -f nodeclass.yaml Next, reference the Node Class in your Node Pool configuration. For more information, see <>. -== Node Class Specification +[#auto-node-access-entry] +== Create node class access entry -[source,yaml] +If you create a custom node class, you need to create an EKS Access Entry to permit the nodes to join the cluster. EKS automatically creates access entries when you use the built-in node class and node pools. + +For information about how Access Entries work, see <>. + +When creating access entries for EKS Auto Mode node classes, you need to use the `EC2` access entry type. + +=== Create access entry with CLI + +*To create an access entry for EC2 nodes and associate the EKS Auto Node Policy:* + +Update the following CLI commands with your cluster name, and node role ARN. The node role ARN is specified in the node class YAML. + +[source,bash,subs="verbatim,attributes"] +---- +# Create the access entry for EC2 nodes +aws eks create-access-entry \ + --cluster-name \ + --principal-arn \ + --type EC2 + +# Associate the auto node policy +aws eks associate-access-policy \ + --cluster-name \ + --principal-arn \ + --policy-arn {arn-aws}eks::aws:cluster-access-policy/AmazonEKSAutoNodePolicy \ + --access-scope type=cluster +---- + +=== Create access entry with CloudFormation + +*To create an access entry for EC2 nodes and associate the EKS Auto Node Policy:* + +Update the following CloudFormation with your cluster name, and node role ARN. The node role ARN is specified in the node class YAML. + +[source,yaml,subs="verbatim,attributes"] +---- +EKSAutoNodeRoleAccessEntry: + Type: AWS::EKS::AccessEntry + Properties: + ClusterName: + PrincipalArn: + Type: "EC2" + AccessPolicies: + - AccessScope: + Type: cluster + PolicyArn: {arn-aws}eks::aws:cluster-access-policy/AmazonEKSAutoNodePolicy + DependsOn: [ ] # previously defined in CloudFormation +---- + +For information about deploying CloudFormation stacks, see link:AWSCloudFormation/latest/UserGuide/GettingStarted.html["Getting started with CloudFormation", type="documentation"] + +[#auto-node-class-spec] +== Node Class Specification + +[source,yaml,subs="verbatim,attributes"] ---- apiVersion: eks.amazonaws.com/v1 kind: NodeClass metadata: - name: default + name: my-node-class spec: + # Required fields - # Required: Name of IAM Role for Nodes - role: "MyNodeRole" - - # Required: Subnet selection for node placement + # role and instanceProfile are mutually exclusive fields. + role: MyNodeRole # IAM role for EC2 instances + # NOTE: instance profile names must start with the 'eks' prefix. this is a + # temporary limitation to be lifted soon. + # instanceProfile: eks-MyNodeInstanceProfile # IAM instance-profile for EC2 instances + subnetSelectorTerms: - tags: - Name: "" + Name: "private-subnet" kubernetes.io/role/internal-elb: "1" # Alternative using direct subnet ID # - id: "subnet-0123456789abcdef0" - # Required: Security group selection for nodes securityGroupSelectorTerms: - tags: - Name: "eks-cluster-node-sg" + Name: "eks-cluster-sg" # Alternative approaches: # - id: "sg-0123456789abcdef0" - # - name: "eks-cluster-node-security-group" - - # Optional: Configure SNAT policy (defaults to Random) + # - name: "eks-cluster-security-group" + + # Optional: Pod subnet selector for advanced networking + podSubnetSelectorTerms: + - tags: + Name: "pod-subnet" + kubernetes.io/role/pod: "1" + # Alternative using direct subnet ID + # - id: "subnet-0987654321fedcba0" +# must include Pod security group selector also + podSecurityGroupSelectorTerms: + - tags: + Name: "eks-pod-sg" + # Alternative using direct security group ID + # - id: "sg-0123456789abcdef0" + + # Optional: Selects on-demand capacity reservations and capacity blocks + # for EKS Auto Mode to prioritize. + capacityReservationSelectorTerms: + - id: cr-56fac701cc1951b03 + # Alternative Approaches + - tags: + Name: "targeted-odcr" + # Optional owning account ID filter + owner: "012345678901" + + # Optional fields snatPolicy: Random # or Disabled - # Optional: Network policy configuration (defaults to DefaultAllow) networkPolicy: DefaultAllow # or DefaultDeny - - # Optional: Network policy event logging (defaults to Disabled) networkPolicyEventLogs: Disabled # or Enabled - # Optional: Configure ephemeral storage (shown with default values) ephemeralStorage: size: "80Gi" # Range: 1-59000Gi or 1-64000G or 1-58Ti or 1-64T iops: 3000 # Range: 3000-16000 throughput: 125 # Range: 125-1000 + # Optional KMS key for encryption + kmsKeyID: "{arn-aws}kms:region:account:key/key-id" + # Accepted formats: + # KMS Key ID + # KMS Key ARN + # Key Alias Name + # Key Alias ARN + + advancedNetworking: + # Optional: Controls whether public IP addresses are assigned to instances that are launched with the nodeclass. + # If not set, defaults to the MapPublicIpOnLaunch setting on the subnet. + associatePublicIPAddress: false + + # Optional: Forward proxy, commonly requires certificateBundles as well + # for EC2, see https://repost.aws/knowledge-center/eks-http-proxy-containerd-automation + httpsProxy: http://192.0.2.4:3128 #commonly port 3128 (Squid) or 8080 (NGINX) #Max 255 characters + #httpsProxy: http://[2001:db8::4]:3128 # IPv6 address with port, use [] + noProxy: #Max 50 entries + - localhost #Max 255 characters each + - 127.0.0.1 + #- ::1 # IPv6 localhost + #- 0:0:0:0:0:0:0:1 # IPv6 localhost + - 169.254.169.254 # EC2 Instance Metadata Service + #- [fd00:ec2::254] # IPv6 EC2 Instance Metadata Service + # Domains to exclude, put all VPC endpoints here + - .internal + - .eks.amazonaws.com + + advancedSecurity: + # Optional, US regions only: Specifying `fips: true` will cause nodes in the nodeclass to run FIPS compatible AMIs. + fips: false + + # Optional: Custom certificate bundles. + certificateBundles: + - name: "custom-cert" + data: "base64-encoded-cert-data" - # Optional: Additional EC2 tags + # Optional: Additional EC2 tags (with restrictions) tags: Environment: "production" Team: "platform" + # Note: Cannot use restricted tags like: + # - kubernetes.io/cluster/* + # - karpenter.sh/provisioner-name + # - karpenter.sh/nodepool + # - karpenter.sh/nodeclaim + # - karpenter.sh/managed-by + # - eks.amazonaws.com/nodeclass +---- + + +== Considerations + +* If you want to verify how much local storage an instance has, you can describe the node to see the ephemeral storage resource. +* *Volume Encryption* - EKS uses the configured custom KMS key to encrypt the read-only root volume of the instance and the read/write data volume. +* *Replace the node IAM role* - If you change the node IAM role associated with a `NodeClass`, you will need to create a new Access Entry. EKS automatically creates an Access Entry for the node IAM role during cluster creation. The node IAM role requires the `AmazonEKSAutoNodePolicy` EKS Access Policy. For more information, see <>. +* *maximum Pod density* - EKS limits the maximum number of Pods on a node to 110. This limit is applied after the existing max Pods calculation. For more information, see <>. +* *Tags* - If you want to propagate tags from Kubernetes to EC2, you need to configure additional IAM permissions. For more information, see <>. +* *Default node class* - Do not name your custom node class `default`. This is because EKS Auto Mode includes a `NodeClass` called `default` that is automatically provisioned when you enable at least one built-in `NodePool`. For information about enabling built-in `NodePools`, see <>. +* *`subnetSelectorTerms` behavior with multiple subnets* - If there are multiple subnets that match the `subnetSelectorTerms` conditions or that you provide by ID, EKS Auto Mode creates nodes distributed across the subnets. + +** If the subnets are in different Availability Zones (AZs), you can use Kubernetes features like https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#pod-topology-spread-constraints[Pod topology spread constraints] and https://kubernetes.io/docs/concepts/services-networking/topology-aware-routing/[Topology Aware Routing] to spread Pods and traffic across the zones, respectively. + +** If there are multiple subnets __in the same AZ__ that match the `subnetSelectorTerms`, EKS Auto Mode creates Pods on each node distributed across the subnets in that AZ. EKS Auto Mode creates secondary network interfaces on each node in the other subnets in the same AZ. It chooses based on the number of available IP addresses in each subnet, to use the subnets more efficiently. However, you can't specify which subnet EKS Auto Mode uses for each Pod; if you need Pods to run in specific subnets, use <> instead. + + +[#pod-subnet-selector] +== Subnet selection for Pods + +The `podSubnetSelectorTerms` and `podSecurityGroupSelectorTerms` fields enables advanced networking configurations by allowing Pods to run in different subnets than their nodes. This separation provides enhanced control over network traffic routing and security policies. Note that `podSecurityGroupSelectorTerms` are required with the `podSubnetSelectorTerms`. + +=== Use cases + +Use `podSubnetSelectorTerms` when you need to: + +* Separate infrastructure traffic (node-to-node communication) from application traffic (Pod-to-Pod communication) +* Apply different network configurations to node subnets than Pod subnets. +* Implement different security policies or routing rules for nodes and Pods. +* Configure reverse proxies or network filtering specifically for node traffic without affecting Pod traffic. Use `advancedNetworking` and `certificateBundles` to define your reverse proxy and any self-signed or private certificates for the proxy. + +=== Example configuration + +[source,yaml] +---- +apiVersion: eks.amazonaws.com/v1 +kind: NodeClass +metadata: + name: advanced-networking +spec: + role: MyNodeRole + + # Subnets for EC2 instances (nodes) + subnetSelectorTerms: + - tags: + Name: "node-subnet" + kubernetes.io/role/internal-elb: "1" + + securityGroupSelectorTerms: + - tags: + Name: "eks-cluster-sg" + + # Separate subnets for Pods + podSubnetSelectorTerms: + - tags: + Name: "pod-subnet" + kubernetes.io/role/pod: "1" + + podSecurityGroupSelectorTerms: + - tags: + Name: "eks-pod-sg" ---- -*Considerations:* +=== Considerations for subnet selectors for Pods -* If you change the Node IAM Role associated with a NodeClass, you will need to create a new Access Entry. EKS automatically creates an Access Entry for the Node IAM Role during cluster creation. The Node IAM Role requires the `AmazonEKSAutoNodePolicy` EKS Access Policy. For more information, see <>. -* EKS limits the maximum number of pods on a node to 110. This limit is applied after the existing max pods calculation. For more information, see <>. -* If you want to propagate tags from Kubernetes to EC2, you need to configure additional IAM permissions. For more information, see <>. +* *Reduced Pod density*: Fewer Pods can run on each node when using `podSubnetSelectorTerms`, because the primary network interface of the node is in the node subnet, and can't be used for Pods in the Pod subnet. +* *Subnet selector limitations*: The standard `subnetSelectorTerms` and `securityGroupSelectorTerms` configurations don't apply to Pod subnet selection. +* *Network planning*: Ensure adequate IP address space in both node and Pod subnets to support your workload requirements. +* *Routing configuration*: Verify that route table and network Access Control List (ACL) of the Pod subnets are properly configured for communication between node and Pod subnets. +* *Availability Zones*: Verify that you've created Pod subnets across multiple AZs. If you are using specific Pod subnet, it must be in the same AZ as the node subnet AZ. diff --git a/latest/ug/automode/create-node-pool.adoc b/latest/ug/automode/create-node-pool.adoc index 74ec9097d..942814c20 100644 --- a/latest/ug/automode/create-node-pool.adoc +++ b/latest/ug/automode/create-node-pool.adoc @@ -1,23 +1,23 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[create-node-pool,create-node-pool.title]] +[#create-node-pool] = Create a Node Pool for EKS Auto Mode -:info_doctype: section :info_titleabbrev: Create node pool -include::../attributes.txt[] +Amazon EKS node pools offer a flexible way to manage compute resources in your Kubernetes cluster. This topic demonstrates how to create and configure node pools by using Karpenter, a node provisioning tool that helps optimize cluster scaling and resource utilization. With Karpenter's NodePool resource, you can define specific requirements for your compute resources, including instance types, availability zones, architectures, and capacity types. -Amazon EKS node pools provide a flexible way to manage compute resources in your Kubernetes cluster. This topic demonstrates how to create and configure node pools using Karpenter, a node provisioning tool that helps optimize cluster scaling and resource utilization. With Karpenter's NodePool resource, you can define specific requirements for your compute resources, including instance types, availability zones, architectures, and capacity types. +You cannot modify the built-in `system` and `general-purpose` node pools. You can only enable or disable them. For more information, see <>. -The NodePool specification allows for fine-grained control over your EKS cluster's compute resources through various supported labels and requirements. These include options for specifying EC2 instance categories, CPU configurations, availability zones, architectures (ARM64/AMD64), and capacity types (spot/on-demand). You can also set resource limits for CPU and memory usage, ensuring your cluster stays within desired operational boundaries. +The NodePool specification allows for fine-grained control over your EKS cluster's compute resources through various supported labels and requirements. These include options for specifying EC2 instance categories, CPU configurations, availability zones, architectures (ARM64/AMD64), and capacity types (spot or on-demand). You can also set resource limits for CPU and memory usage, ensuring your cluster stays within required operational boundaries. -EKS Auto Mode leverages well-known Kubernetes labels to provide consistent and standardized ways of identifying node characteristics. These labels, such as `topology.kubernetes.io/zone` for availability zones and `kubernetes.io/arch` for CPU architecture, follow established Kubernetes conventions. Additionally, EKS-specific labels (prefixed with `eks.amazonaws.com/`) extend this functionality with {aws}-specific attributes like instance types, CPU manufacturers, GPU capabilities, and networking specifications. This standardized labeling system enables seamless integration with existing Kubernetes tooling while providing deep {aws} infrastructure integration. +EKS Auto Mode leverages well-known Kubernetes labels to provide consistent and standardized ways of identifying node characteristics. These labels, such as `topology.kubernetes.io/zone` for availability zones and `kubernetes.io/arch` for CPU architecture, follow established Kubernetes conventions. Additionally, EKS-specific labels (prefixed with `eks.amazonaws.com/`) extend this functionality with {aws}-specific attributes such as instance types, CPU manufacturers, GPU capabilities, and networking specifications. This standardized labeling system enables seamless integration with existing Kubernetes tools while providing deep {aws} infrastructure integration. -## Create a NodePool +== Create a NodePool Follow these steps to create a NodePool for your Amazon EKS cluster: -. Create a YAML file named `nodepool.yaml` with your desired NodePool configuration. You can use the sample configuration below. +. Create a YAML file named `nodepool.yaml` with your required NodePool configuration. You can use the sample configuration below. . Apply the NodePool to your cluster: + ```bash @@ -43,7 +43,7 @@ Ensure that your NodePool references a valid NodeClass that exists in your clust apiVersion: karpenter.sh/v1 kind: NodePool metadata: - name: default + name: my-node-pool spec: template: metadata: @@ -74,15 +74,21 @@ spec: memory: 1000Gi ---- +[#auto-supported-labels] +== EKS Auto Mode Supported Labels -== {eam} Supported +EKS Auto Mode supports the following well known labels. -{eam} supports the following well known labels. +NOTE: EKS Auto Mode uses different labels than Karpenter. Labels related to EC2 managed instances start with `eks.amazonaws.com`. [role="no-scroll"] -[cols="3,1,4",options="header"] +[%header,cols="3,1,4"] |=== -|Label |Example |Description + +|Label +|Example +|Description + |topology.kubernetes.io/zone |us-east-2a @@ -92,10 +98,9 @@ spec: |g4dn.8xlarge |{aws} instance type - |kubernetes.io/arch |amd64 -|Architectures are defined by link:https://github.com/golang/go/blob/master/src/go/build/syslist.go#L50[GOARCH values] on the instance +|Architectures are defined by link:https://github.com/golang/go/blob/master/src/internal/syslist/syslist.go#L58[GOARCH values] on the instance |karpenter.sh/capacity-type |spot @@ -134,7 +139,7 @@ spec: |Number of CPUs on the instance |eks.amazonaws.com/instance-cpu-manufacturer -|aws +|`aws` |Name of the CPU manufacturer |eks.amazonaws.com/instance-memory @@ -143,11 +148,11 @@ spec: |eks.amazonaws.com/instance-ebs-bandwidth |9500 -|Number of link:https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-optimized.html#ebs-optimization-performance[maximum megabits] of EBS available on the instance +|Number of link:AWSEC2/latest/UserGuide/ebs-optimized.html#ebs-optimization-performance[maximum megabits,type="documentation"] of EBS available on the instance |eks.amazonaws.com/instance-network-bandwidth |131072 -|Number of link:https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html[baseline megabits] available on the instance +|Number of link:AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html[baseline megabits,type="documentation"] available on the instance |eks.amazonaws.com/instance-gpu-name |t4 @@ -168,12 +173,65 @@ spec: |eks.amazonaws.com/instance-local-nvme |900 |Number of gibibytes of local nvme storage on the instance + |=== -== {eam} Not Supported +NOTE: EKS Auto Mode only supports certain instances, and has minimum size requirements. For more information, see <>. -{eam} does not support the following labels. +== EKS Auto Mode Not Supported Labels -* {eam} only supports Linux +EKS Auto Mode does not support the following labels. + +* EKS Auto Mode only supports Linux ** `node.kubernetes.io/windows-build` ** `kubernetes.io/os` + +== Disable built-in node pools + +If you create custom node pools, you can disable the built-in node pools. For more information, see <>. + +== Cluster without built-in node pools + +You can create a cluster without the built-in node pools. This is helpful when your organization has created customized node pools. + +NOTE: When you create a cluster without built-in node pools, the `default` NodeClass is not automatically provisioned. You'll need to create a custom NodeClass. For more information, see <>. + +*Overview:* + +. Create an EKS cluster with the both `nodePools` and `nodeRoleArn` values empty. +** Sample eksctl `autoModeConfig`: ++ +[source,yaml] +---- +autoModeConfig: + enabled: true + nodePools: [] + # Do not set a nodeRoleARN +---- ++ +For more information, see <> +. Create a custom node class with a node role ARN +** For more information, see <> +. Create an access entry for the custom node class +** For more information, see <> +. Create a custom node pool, as described above. + +== Disruption + +You can configure EKS Auto Mode to disrupt Nodes through your NodePool in multiple ways. You can use `spec.disruption.consolidationPolicy`, `spec.disruption.consolidateAfter`, or `spec.template.spec.expireAfter`. You can also rate limit EKS Auto Mode's disruption through the NodePool’s `spec.disruption.budgets`. You can also control the time windows and number of simultaneous Nodes disrupted. For instructions on configuring this behavior, see https://karpenter.sh/docs/concepts/disruption/[Disruption] in the Karpenter Documentation. + +You can configure disruption for node pools to: + +- Identify when instances are underutilized, and consolidate workloads. +- Create a node pool disruption budget to rate limit node terminations due to drift, emptiness, and consolidation. + +By default, EKS Auto Mode: + +- Consolidates underutilized instances. +- Terminates instances after 336 hours. +- Sets a single disruption budget of 10% of nodes. +- Allows Nodes to be replaced due to drift when a new Auto Mode AMI is released, which occurs roughly once per week. + +== Termination Grace Period + +When a `terminationGracePeriod` is not explicitly defined on an EKS Auto NodePool, the system automatically applies a default 24-hour termination grace period to the associated NodeClaim. While EKS Auto customers will not see a `terminationGracePeriod` defaulted in their custom NodePool configurations, they will observe this default value on the NodeClaim. The functionality remains consistent whether the grace period is explicitly set on the NodePool or defaulted on the NodeClaim, ensuring predictable node termination behavior across the cluster. diff --git a/latest/ug/automode/create-storage-class.adoc b/latest/ug/automode/create-storage-class.adoc index 8e28c3a80..701a1b4bc 100644 --- a/latest/ug/automode/create-storage-class.adoc +++ b/latest/ug/automode/create-storage-class.adoc @@ -1,18 +1,15 @@ -//!!NODE_ROOT
-[.topic] -[[create-storage-class,create-storage-class.title]] -= Create a Storage Class -:info_doctype: section -:info_titleabbrev: Create storage class - - include::../attributes.txt[] -A StorageClass in Amazon EKS Auto Mode defines how Amazon EBS volumes are automatically provisioned when applications request persistent storage. This page explains how to create and configure a StorageClass that works with the Amazon EKS Auto Mode to provision EBS volumes. +[.topic] +[#create-storage-class] += Create a storage class +:info_titleabbrev: Create StorageClass -By configuring a StorageClass, you can specify default settings for your EBS volumes including volume type, encryption, IOPS, and other storage parameters. You can also configure the StorageClass to use {aws} KMS keys for encryption management. +A `StorageClass` in Amazon EKS Auto Mode defines how Amazon EBS volumes are automatically provisioned when applications request persistent storage. This page explains how to create and configure a `StorageClass` that works with the Amazon EKS Auto Mode to provision EBS volumes. -{eam} does not create a StorageClass for you. You must create a StorageClass referencing `ebs.csi.eks.amazonaws.com` to use the storage capability of {eam}. +By configuring a `StorageClass`, you can specify default settings for your EBS volumes including volume type, encryption, IOPS, and other storage parameters. You can also configure the `StorageClass` to use {aws} KMS keys for encryption management. + +EKS Auto Mode does not create a `StorageClass` for you. You must create a `StorageClass` referencing `ebs.csi.eks.amazonaws.com` to use the storage capability of EKS Auto Mode. First, create a file named `storage-class.yaml`: @@ -24,6 +21,11 @@ metadata: name: auto-ebs-sc annotations: storageclass.kubernetes.io/is-default-class: "true" +allowedTopologies: +- matchLabelExpressions: + - key: eks.amazonaws.com/compute-type + values: + - auto provisioner: ebs.csi.eks.amazonaws.com volumeBindingMode: WaitForFirstConsumer parameters: @@ -40,10 +42,11 @@ kubectl apply -f storage-class.yaml *Key components:* -- `provisioner: ebs.csi.eks.amazonaws.com` - Uses {eam} +- `provisioner: ebs.csi.eks.amazonaws.com` - Uses EKS Auto Mode +- `allowedTopologies` - Specifying `matchLabelExpressions` to match on `eks.amazonaws.com/compute-type:auto` will ensure that if your pods need a volume to be automatically provisioned using Auto Mode then the pods will not be scheduled on non-Auto nodes. - `volumeBindingMode: WaitForFirstConsumer` - Delays volume creation until a pod needs it - `type: gp3` - Specifies the EBS volume type -- `encrypted: "true"` - EBS will encrypt any volumes created using the StorageClass. EBS will use the default `aws/ebs` key alias. For more information, see link:ebs/latest/userguide/how-ebs-encryption-works.html["How Amazon EBS encryption works",type="documentation"] in the Amazon EBS User Guide. This value is optional but suggested. +- `encrypted: "true"` - EBS will encrypt any volumes created using the `StorageClass`. EBS will use the default `aws/ebs` key alias. For more information, see link:ebs/latest/userguide/how-ebs-encryption-works.html["How Amazon EBS encryption works",type="documentation"] in the Amazon EBS User Guide. This value is optional but suggested. - `storageclass.kubernetes.io/is-default-class: "true"` - Kubernetes will use this storage class by default, unless you specify a different volume class on a persistent volume claim. This value is optional. Use caution when setting this value if you are migrating from a different storage controller. == Use self-managed KMS key to encrypt EBS volumes @@ -63,51 +66,15 @@ link:IAM/latest/UserGuide/access_policies_job-functions_create-policies.html["Cr Update the following values in the policy below: -* `` -- Your {aws} account ID, such as `111122223333` -* `` -- The {aws} region of your cluster, such as `us-west-2` +* `` – Your {aws} account ID, such as `111122223333` +* `` – The {aws} region of your cluster, such as `us-west-2` -[source,json] +[source,json,subs="verbatim,attributes"] ---- -{ - "Version": "2012-10-17", - "Id": "key-auto-policy-3", - "Statement": [ - { - "Sid": "Enable IAM User Permissions", - "Effect": "Allow", - "Principal": { - "AWS": "arn:aws:iam:::root" - }, - "Action": "kms:*", - "Resource": "*" - }, - { - "Sid": "Allow access through EBS for all principals in the account that are authorized to use EBS", - "Effect": "Allow", - "Principal": { - "AWS": "*" - }, - "Action": [ - "kms:Encrypt", - "kms:Decrypt", - "kms:ReEncrypt*", - "kms:GenerateDataKey*", - "kms:CreateGrant", - "kms:DescribeKey" - ], - "Resource": "*", - "Condition": { - "StringEquals": { - "kms:CallerAccount": "", - "kms:ViaService": "ec2..amazonaws.com" - } - } - } - ] -} +include::iam_snippet:cb3ec475-7e7f-44d9-8e3f-54e5f092b59e[] ---- -=== Sample self-managed KMS StorageClass +=== Sample self-managed KMS `StorageClass` [source,yaml] ---- @@ -118,82 +85,107 @@ parameters: ---- -== StorageClass Parameters Reference +== `StorageClass` Parameters Reference For general information on the Kubernetes `StorageClass` resources, see https://kubernetes.io/docs/concepts/storage/storage-classes/[Storage Classes] in the Kubernetes Documentation. THe `parameters` section of the `StorageClass` resource is specific to {aws}. Use the following table to review available options. [role="no-scroll"] -[cols="4*", options="header"] +[%header,cols="4"] |=== -|Parameters |Values |Default |Description + +|Parameters +|Values +|Default +|Description + |"csi.storage.k8s.io/fstype" |xfs, ext2, ext3, ext4 |ext4 |File system type that will be formatted during volume creation. This parameter is case sensitive! + |"type" |io1, io2, gp2, gp3, sc1, st1, standard, sbp1, sbg1 |gp3 |EBS volume type. + |"iopsPerGB" | | |I/O operations per second per GiB. Can be specified for IO1, IO2, and GP3 volumes. + |"allowAutoIOPSPerGBIncrease" |true, false |false |When `"true"`, the CSI driver increases IOPS for a volume when `iopsPerGB * ` is too low to fit into IOPS range supported by {aws}. This allows dynamic provisioning to always succeed, even when user specifies too small PVC capacity or `iopsPerGB` value. On the other hand, it may introduce additional costs, as such volumes have higher IOPS than requested in `iopsPerGB`. + |"iops" | | |I/O operations per second. Can be specified for IO1, IO2, and GP3 volumes. + |"throughput" | |125 |Throughput in MiB/s. Only effective when gp3 volume type is specified. + |"encrypted" |true, false |false |Whether the volume should be encrypted or not. Valid values are "true" or "false". + |"blockExpress" |true, false |false |Enables the creation of io2 Block Express volumes. + |"kmsKeyId" | | |The full ARN of the key to use when encrypting the volume. If not specified, {aws} will use the default KMS key for the region the volume is in. This will be an auto-generated key called `/aws/ebs` if not changed. + |"blockSize" | | |The block size to use when formatting the underlying filesystem. Only supported on linux nodes and with fstype `ext2`, `ext3`, `ext4`, or `xfs`. + |"inodeSize" | | |The inode size to use when formatting the underlying filesystem. Only supported on linux nodes and with fstype `ext2`, `ext3`, `ext4`, or `xfs`. + |"bytesPerInode" | | |The `bytes-per-inode` to use when formatting the underlying filesystem. Only supported on linux nodes and with fstype `ext2`, `ext3`, `ext4`. + |"numberOfInodes" | | |The `number-of-inodes` to use when formatting the underlying filesystem. Only supported on linux nodes and with fstype `ext2`, `ext3`, `ext4`. + |"ext4BigAlloc" |true, false |false |Changes the `ext4` filesystem to use clustered block allocation by enabling the `bigalloc` formatting option. Warning: `bigalloc` may not be fully supported with your node's Linux kernel. + |"ext4ClusterSize" | | |The cluster size to use when formatting an `ext4` filesystem when the `bigalloc` feature is enabled. Note: The `ext4BigAlloc` parameter must be set to true. + |=== For more information, see the https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/docs/parameters.md[{aws} EBS CSI Driver] on GitHub. == Considerations +[NOTE] +==== +You can only deploy workloads depending on EKS Auto Mode StorageClasses on EKS Auto Mode nodes. If you have a cluster with mixed types of nodes, you need to configure your workloads to run only on EKS Auto Mode nodes. For more information, see <>. +==== + The block storage capability of EKS Auto Mode is different from the EBS CSI Driver. * Static Provisioning @@ -202,7 +194,7 @@ The block storage capability of EKS Auto Mode is different from the EBS CSI Driv ** You cannot use the node startup taint feature to prevent pod scheduling before storage capability readiness * Custom Tags on Dynamically Provisioned Volumes ** You cannot use the extra-tag CLI flag to configure custom tags on dynamically provisioned EBS volumes -** You can use StorageClass Tagging to add custom tags. EKS Auto Mode will add tags to the associated {aws} resources. You will need to update the Cluster IAM Role for custom tags. For more information, see <>. +** You can use `StorageClass` tagging to add custom tags. EKS Auto Mode will add tags to the associated {aws} resources. You will need to update the Cluster IAM Role for custom tags. For more information, see <>. * EBS Detailed Performance Metrics ** You cannot access Prometheus metrics for EBS detailed performance @@ -219,7 +211,7 @@ For more information, see: * <> -[[auto-install-snapshot-controller,auto-install-snapshot-controller.title]] +[#auto-install-snapshot-controller] === To install snapshot controller in system node pool . Open your EKS cluster in the {aws} console diff --git a/latest/ug/automode/critical-workload.adoc b/latest/ug/automode/critical-workload.adoc index 75c3668ea..012231e0d 100644 --- a/latest/ug/automode/critical-workload.adoc +++ b/latest/ug/automode/critical-workload.adoc @@ -1,12 +1,10 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[critical-workload,critical-workload.title]] +[#critical-workload] = Run critical add-ons on dedicated instances -:info_doctype: section :info_titleabbrev: Run critical add-ons -include::../attributes.txt[] - In this topic, you will learn how to deploy a workload with a `CriticalAddonsOnly` toleration so EKS Auto Mode will schedule it onto the `system` node pool. EKS Auto Mode's built-in `system` node pool is designed for running critical add-ons on dedicated instances. This segregation ensures essential components have dedicated resources and are isolated from general workloads, enhancing overall cluster stability and performance. @@ -67,4 +65,4 @@ To update a workload to run on the `system` node pool, you need to: ** `tolerations` . Deploy the updated workload to your cluster with `kubectl apply` -After updating the workload, it will run on dedicated nodes. +After updating the workload, it will run on dedicated nodes. \ No newline at end of file diff --git a/latest/ug/automode/images b/latest/ug/automode/images new file mode 120000 index 000000000..5e6757319 --- /dev/null +++ b/latest/ug/automode/images @@ -0,0 +1 @@ +../images \ No newline at end of file diff --git a/latest/ug/automode/migrate-auto.adoc b/latest/ug/automode/migrate-auto.adoc index fe53704b7..eee0d0cbf 100644 --- a/latest/ug/automode/migrate-auto.adoc +++ b/latest/ug/automode/migrate-auto.adoc @@ -1,17 +1,9 @@ -//!!NODE_ROOT
-[.topic] include::../attributes.txt[] -[[migrate-auto,migrate-auto.title]] + +[.topic] +[#migrate-auto] = Enable EKS Auto Mode on existing EKS clusters -:info_doctype: section -:icons: font -:experimental: -:idprefix: -:idseparator: - -:sourcedir: . -:info_title: Enable EKS Auto Mode on existing EKS clusters :info_titleabbrev: Enable existing clusters -:info_abstract: Learn about the tools needed to migrate an Amazon EKS cluster to EKS Auto Mode. [abstract] -- @@ -20,60 +12,69 @@ Learn about the tools needed for creating and working with an Amazon EKS cluster You can enable EKS Auto Mode on existing EKS Clusters. -[NOTE] -==== -EKS Auto Mode requires Kubernetes version 1.29 or greater. -==== - *{aws} supports the following migrations:* -* Migrating from Karpenter to EKS Auto Mode Nodes -** Learn how to <> -* Migrating from EKS Managed Node Groups to EKS Auto Mode Nodes -** Learn how to <> -* Migrating from EKS Fargate to EKS Auto Mode Nodes +* Migrating from Karpenter to EKS Auto Mode nodes. For more information, see <>. +* Migrating from EKS Managed Node Groups to EKS Auto Mode nodes. For more information, see <>. +* Migrating from EKS Fargate to EKS Auto Mode. For more information, see <>. *{aws} does not support the following migrations:* -* Migrating volumes from the EBS CSI Controller to EKS Auto Mode Block Storage -** You can install the EBS CSI controller on an Amazon EKS Auto Mode cluster. Use a `StorageClass` to associate volumes with either the EBS CSI Controller or EKS Auto Mode. +* Migrating volumes from the EBS CSI controller (using the Amazon EKS add-on) to EKS Auto Mode EBS CSI controller (managed by EKS Auto Mode). PVCs made with one can't be mounted by the other, because they use two different Kubernetes volume provisioners. +** The https://github.com/awslabs/eks-auto-mode-ebs-migration-tool[`eks-auto-mode-ebs-migration-tool`] ({aws} Labs project) enables migration between standard EBS CSI StorageClass (`ebs.csi.aws.com`) and EKS Auto EBS CSI StorageClass (`ebs.csi.eks.amazonaws.com`). Note that migration requires deleting and re-creating existing PersistentVolumeClaim/PersistentVolume resources, so validation in a non-production environment is essential before implementation. * Migrating load balancers from the {aws} Load Balancer Controller to EKS Auto Mode -** You can install the {aws} Load Balancer Controller on an Amazon EKS Auto Mode cluster. Use the `IngressClass` or `loadBalancerClass` options to associate Service and Ingress resources with either the Load Balancer Controller or EKS Auto Mode. -* Migrating EKS Clusters with alternative CNIs or other unsupported networking configurations ++ +You can install the {aws} Load Balancer Controller on an Amazon EKS Auto Mode cluster. Use the `IngressClass` or `loadBalancerClass` options to associate Service and Ingress resources with either the Load Balancer Controller or EKS Auto Mode. +* Migrating EKS clusters with alternative CNIs or other unsupported networking configurations -== Migration Reference +[#migration-reference] +== Migration reference -Use the following migration reference to configure Kubernetes Resources to be owned by either self-managed controllers or EKS Auto Mode. +Use the following migration reference to configure Kubernetes resources to be owned by either self-managed controllers or EKS Auto Mode. -[%header,cols="1,1,1,1,1"] +[%header,cols="5"] |=== - | Capability | Resource | Field | Self Managed | EKS Auto Mode -| Block Storage | StorageClass | provisioner | kubernetes.io/aws-ebs | ebs.csi.eks.amazonaws.com - -| Load Balancing | Service | loadBalancerClass | service.k8s.aws/nlb | eks.amazonaws.com/nlb +| Block storage | `StorageClass` | `provisioner` | `ebs.csi.aws.com` | `ebs.csi.eks.amazonaws.com` +| Load balancing | `Service` | `loadBalancerClass` | `service.k8s.aws/nlb` | `eks.amazonaws.com/nlb` +| Load balancing | `IngressClass` | `controller` | `ingress.k8s.aws/alb` | `eks.amazonaws.com/alb` +| Load balancing | `IngressClassParams` | `apiversion` | `elbv2.k8s.aws/v1beta1` | `eks.amazonaws.com/v1` +| Load balancing | `TargetGroupBinding` | `apiversion` | `elbv2.k8s.aws/v1beta1` | `eks.amazonaws.com/v1` +| Compute | `NodeClass` | `apiVersion` | `karpenter.sh/v1` | `eks.amazonaws.com/v1` +|=== -| Load Balancing | IngressClass | controller | ingress.k8s.aws/alb | eks.amazonaws.com/alb +== Migrating EBS volumes +When migrating workloads to EKS Auto Mode, you need to handle EBS volume migration due to different CSI driver provisioners: +* EKS Auto Mode provisioner: `ebs.csi.eks.amazonaws.com` +* Open source EBS CSI provisioner: `ebs.csi.aws.com` -|Load Balancing | IngressClassParams |apiversion |elbv2.k8s.aws/v1beta1 |eks.amazonaws.com/v1 +Follow these steps to migrate your persistent volumes: -|Load Balancing | TargetGroupBinding |apiversion |elbv2.k8s.aws/v1beta1 |eks.amazonaws.com/v1 +. **Modify volume retention policy**: Change the existing platform version's (PV's) `persistentVolumeReclaimPolicy` to `Retain` to ensure the underlying EBS volume is not deleted. +. **Remove PV from Kubernetes**: Delete the old PV resource while keeping the actual EBS volume intact. +. **Create a new PV with static provisioning**: Create a new PV that references the same EBS volume but works with the target CSI driver. +. **Bind to a new PVC**: Create a new PVC that specifically references your PV using the `volumeName` field. -| Compute | NodeClass | apiVersion | karpenter.sh/v1alpha5 | eks.amazonaws.com/v1 +=== Considerations -|=== +* Ensure your applications are stopped before beginning this migration. +* Back up your data before starting the migration process. +* This process needs to be performed for each persistent volume. +* The workload must be updated to use the new PVC. -== Load Balancer Migration +== Migrating load balancers You cannot directly transfer existing load balancers from the self-managed {aws} load balancer controller to EKS Auto Mode. Instead, you must implement a blue-green deployment strategy. This involves maintaining your existing load balancer configuration while creating new load balancers under the managed controller. -To minimize service disruption, we recommend a DNS-based traffic shifting approach. First, create new load balancers using EKS Auto Mode while keeping your existing configuration operational. Then, use DNS routing (such as Route 53) to gradually shift traffic from the old load balancers to the new ones. Once traffic has been successfully migrated and you've verified the new configuration, you can decommission the old load balancers and self-managed controller. +To minimize service disruption, we recommend a DNS-based traffic shifting approach. First, create new load balancers by using EKS Auto Mode while keeping your existing configuration operational. Then, use DNS routing (such as Route 53) to gradually shift traffic from the old load balancers to the new ones. Once traffic has been successfully migrated and you've verified the new configuration, you can decommission the old load balancers and self-managed controller. include::auto-enable-existing.adoc[leveloffset=+1] include::auto-migrate-karpenter.adoc[leveloffset=+1] include::auto-migrate-mng.adoc[leveloffset=+1] + +include::auto-migrate-fargate.adoc[leveloffset=+1] diff --git a/latest/ug/automode/old/hpa_scaling.adoc b/latest/ug/automode/old/hpa_scaling.adoc index 5e0fe70c0..6565868b9 100644 --- a/latest/ug/automode/old/hpa_scaling.adoc +++ b/latest/ug/automode/old/hpa_scaling.adoc @@ -1,12 +1,8 @@ -//!!NODE_ROOT
+include::../../attributes.txt[] [.topic] -[[auto-hpa-scaling,auto-hpa-scaling.title]] +[#auto-hpa-scaling] = End-to-end Compute Autoscaling with HPA -:info_doctype: section - -include::../attributes.txt[] - This guide shows you how Karpenter autoscales nodes in conjunction with HPA scaling your applications. @@ -21,8 +17,8 @@ This guide shows you how Karpenter autoscales nodes in conjunction with HPA scal == Prerequisites * watch (https://formulae.brew.sh/formula/watch[Mac], https://www.powershellgallery.com/packages/Watch-Command/0.1.3[Windows]) -* https://kubernetes.io/docs/tasks/tools/#kubectl[kubectl] -* https://helm.sh/docs/intro/install/[Helm] +* <> +* <> == 1. Deploy Metrics Server diff --git a/latest/ug/automode/sample-storage-workload.adoc b/latest/ug/automode/sample-storage-workload.adoc index 4230d0e1c..7234aff64 100644 --- a/latest/ug/automode/sample-storage-workload.adoc +++ b/latest/ug/automode/sample-storage-workload.adoc @@ -1,25 +1,20 @@ -//!!NODE_ROOT
+include::../attributes.txt[] [.topic] -[[sample-storage-workload,sample-storage-workload.title]] +[#sample-storage-workload] = Deploy a sample stateful workload to EKS Auto Mode -:info_doctype: section -:info_title: Deploy a sample stateful workload to EKS Auto Mode :info_titleabbrev: Deploy stateful workload -:info_abstract: Deploy a sample stateful workload to EKS Auto Mode - -include::../attributes.txt[] This tutorial will guide you through deploying a sample stateful application to your EKS Auto Mode cluster. The application writes timestamps to a persistent volume, demonstrating EKS Auto Mode's automatic EBS volume provisioning and persistence capabilities. -## Prerequisites +== Prerequisites * An EKS Auto Mode cluster * The {aws} CLI configured with appropriate permissions * `kubectl` installed and configured ** For more information, see <>. -## Step 1: Configure your environment +== Step 1: Configure your environment . Set your environment variables: + @@ -35,11 +30,11 @@ export AWS_REGION="us-west-2" aws eks update-kubeconfig --name "${CLUSTER_NAME}" ---- -## Step 2: Create the storage class +== Step 2: Create the storage class -The StorageClass defines how EKS Auto Mode will provision EBS volumes. +The `StorageClass` defines how EKS Auto Mode will provision EBS volumes. -{eam} does not create a StorageClass for you. You must create a StorageClass referencing `ebs.csi.eks.amazonaws.com` to use the storage capability of {eam}. +EKS Auto Mode does not create a `StorageClass` for you. You must create a `StorageClass` referencing `ebs.csi.eks.amazonaws.com` to use the storage capability of EKS Auto Mode. . Create a file named `storage-class.yaml`: + @@ -57,7 +52,7 @@ parameters: type: gp3 encrypted: "true" ---- -. Apply the StorageClass: +. Apply the `StorageClass`: + [source,bash] ---- @@ -66,15 +61,15 @@ kubectl apply -f storage-class.yaml *Key components:* -- `provisioner: ebs.csi.eks.amazonaws.com` - Uses {eam} +- `provisioner: ebs.csi.eks.amazonaws.com` - Uses EKS Auto Mode - `volumeBindingMode: WaitForFirstConsumer` - Delays volume creation until a pod needs it - `type: gp3` - Specifies the EBS volume type -- `encrypted: "true"` - EBS will use the default `aws/ebs` key to encrypt volumes created with this class. This is optional, but reccomended. +- `encrypted: "true"` - EBS will use the default `aws/ebs` key to encrypt volumes created with this class. This is optional, but recommended. - `storageclass.kubernetes.io/is-default-class: "true"` - Kubernetes will use this storage class by default, unless you specify a different volume class on a persistent volume claim. Use caution when setting this value if you are migrating from another storage controller. (optional) -## Step 3: Create the persistent volume claim +== Step 3: Create the persistent volume claim -The PVC requests storage from the StorageClass. +The PVC requests storage from the `StorageClass`. . Create a file named `pvc.yaml`: + @@ -103,9 +98,9 @@ kubectl apply -f pvc.yaml - `accessModes: ReadWriteOnce` - Volume can be mounted by one node at a time - `storage: 8Gi` - Requests an 8 GiB volume -- `storageClassName: auto-ebs-sc` - References the StorageClass we created +- `storageClassName: auto-ebs-sc` - References the `StorageClass` we created -## Step 4: Deploy the Application +== Step 4: Deploy the Application The Deployment runs a container that writes timestamps to the persistent volume. @@ -160,7 +155,7 @@ kubectl apply -f deployment.yaml - Requests 1 CPU core - Uses node selector for EKS managed nodes -## Step 5: Verify the Setup +== Step 5: Verify the Setup . Check that the pod is running: + @@ -193,7 +188,7 @@ kubectl exec "$(kubectl get pods -l app=inflate-stateful \ cat /data/out.txt ---- -## Step 6: Cleanup +== Step 6: Cleanup Run the following command to remove all resources created in this tutorial: @@ -203,9 +198,9 @@ Run the following command to remove all resources created in this tutorial: kubectl delete deployment/inflate-stateful pvc/auto-ebs-claim storageclass/auto-ebs-sc ---- -## What's Happening Behind the Scenes +== What's Happening Behind the Scenes -. The PVC requests storage from the StorageClass +. The PVC requests storage from the `StorageClass` . When the Pod is scheduled: .. EKS Auto Mode provisions an EBS volume .. Creates a PersistentVolume @@ -214,9 +209,9 @@ kubectl delete deployment/inflate-stateful pvc/auto-ebs-claim storageclass/auto- == Snapshot Controller -{eam} is compatible with the Kubernetes CSI Snapshotter, also known as the snapshot controller. However, {eam} does not include the snapshot controller. You are responsible for installing and configuring the snapshot controller. For more information, see <>. +EKS Auto Mode is compatible with the Kubernetes CSI Snapshotter, also known as the snapshot controller. However, EKS Auto Mode does not include the snapshot controller. You are responsible for installing and configuring the snapshot controller. For more information, see <>. -Review the following `VolumeSnapshotClass` that references the storage capability of {eam}. +Review the following `VolumeSnapshotClass` that references the storage capability of EKS Auto Mode. [source,yaml] ---- diff --git a/latest/ug/automode/set-builtin-node-pools.adoc b/latest/ug/automode/set-builtin-node-pools.adoc index 0082f30ab..ddd61386a 100644 --- a/latest/ug/automode/set-builtin-node-pools.adoc +++ b/latest/ug/automode/set-builtin-node-pools.adoc @@ -1,18 +1,16 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[set-builtin-node-pools,set-builtin-node-pools.title]] +[#set-builtin-node-pools] = Enable or Disable Built-in NodePools -:info_doctype: section :info_titleabbrev: Review built-in node pools -include::../attributes.txt[] - -{eam} has two built-in NodePools. You can enable or disable these NodePools using the {aws} console, CLI, or API. +EKS Auto Mode has two built-in NodePools. You can enable or disable these NodePools using the {aws} console, CLI, or API. == Built-in NodePool Reference * `system` -** This NodePool has a `CriticalAddonsOnly` taint. Many EKS addons, such as CoreDNS, tolerate this taint. Use this system node pool to segregate cluster-critical applications. +** This NodePool has a `CriticalAddonsOnly` taint. Many EKS add-ons, such as CoreDNS, tolerate this taint. Use this system node pool to separate cluster-critical applications. ** Supports both `amd64` and `arm64` architectures. * `general-purpose` ** This NodePool provides support for launching nodes for general purpose workloads in your cluster. @@ -25,12 +23,16 @@ Both built-in NodePools: * Use the C, M, and R EC2 instance families * Require generation 5 or newer EC2 instances -## Prerequisites +NOTE: Enabling at least one built-in NodePool is required for EKS to provision the "default" NodeClass. If you disable all built-in NodePools, you'll need to create a custom NodeClass and configure a NodePool to use it. For more information about NodeClasses, see <>. + +== Procedure + +=== Prerequisites * The latest version of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device. To check your current version, use `aws --version`. To install the latest version, see link:cli/latest/userguide/getting-started-install.html["Installing",type="documentation"] and link:cli/latest/userguide/cli-chap-configure.html#cli-configure-quickstart-config["Quick configuration",type="documentation"] with aws configure in the {aws} Command Line Interface User Guide. ** Login to the CLI with sufficent IAM permissions to create {aws} resources including IAM Policies, IAM Roles, and EKS Clusters. -== Enable with {aws} CLI +=== Enable with {aws} CLI Use the following command to enable both built-in NodePools: @@ -40,14 +42,20 @@ aws eks update-cluster-config \ --name \ --compute-config '{ "nodeRoleArn": "", - "nodePools": ["general-purpose", "system"] - }' - + "nodePools": ["general-purpose", "system"], + "enabled": true + }' \ + --kubernetes-network-config '{ + "elasticLoadBalancing":{"enabled": true} + }' \ + --storage-config '{ + "blockStorage":{"enabled": true} + }' ---- You can modify the command to selectively enable the NodePools. -== Disable with {aws} CLI +=== Disable with {aws} CLI Use the following command to disable both built-in NodePools: @@ -55,5 +63,13 @@ Use the following command to disable both built-in NodePools: ---- aws eks update-cluster-config \ --name \ - --compute-config '{"nodePools": []}' + --compute-config '{ + "enabled": true, + "nodePools": [] + }' \ + --kubernetes-network-config '{ + "elasticLoadBalancing":{"enabled": true}}' \ + --storage-config '{ + "blockStorage":{"enabled": true} + }' ---- diff --git a/latest/ug/automode/settings-auto.adoc b/latest/ug/automode/settings-auto.adoc index 23f7f2bf4..ff13eb864 100644 --- a/latest/ug/automode/settings-auto.adoc +++ b/latest/ug/automode/settings-auto.adoc @@ -1,17 +1,9 @@ -//!!NODE_ROOT -[.topic] include::../attributes.txt[] -[[settings-auto,settings-auto.title]] + +[.topic] +[#settings-auto] = Configure EKS Auto Mode settings -:info_doctype: section -:icons: font -:experimental: -:idprefix: -:idseparator: - -:sourcedir: . -:info_title: Change EKS Auto cluster settings :info_titleabbrev: Configure -:info_abstract: Change EKS Auto cluster settings [abstract] -- @@ -23,9 +15,11 @@ This chapter describes how to configure specific aspects of your Amazon Elastic Using the configuration options described in this topic, you can modify networking settings, compute resources, and load balancing behaviors while maintaining the benefits of automated infrastructure management. Before making any configuration changes, review the available options in the following sections to determine which approach best suits your needs. [role="no-scroll"] -[cols="1,1"] +[%header,cols="2"] |=== -|What features do you want to configure? |Configuration option +|What features do you want to configure? +|Configuration option + a| *Node networking and storage* @@ -33,9 +27,10 @@ a| - Configure node placement across public and private subnets - Define custom security groups for node access control - Customize network address translation (SNAT) policies -- Enable detailed network policy logging and monitoring +- Enable Kubernetes __network policies__, detailed network policy logging and monitoring - Set ephemeral storage parameters (size, IOPS, throughput) - Configure encrypted ephemeral storage with custom KMS keys +- Isolate pod traffic in separate subnets from the nodes |<> a| @@ -46,7 +41,7 @@ a| - Configure capacity types (On-Demand, Spot) - Specify Availability Zones - Configure node taints and labels -- Set minimum and maximum node counts +- Set resource limits for CPU and memory usage |<> a| @@ -86,6 +81,16 @@ a| - Define custom tags for provisioned volumes |<> +a| +*Control ODCR Usage* + +- Configure workload deployment into EC2 On-Demand Capacity Reservations +- Explicitly select specific ODCRs by ID for targeted capacity usage +- Use tag-based selection to target groups of ODCRs +- Filter ODCRs by owning {aws} account for cross-account scenarios +- Control whether workloads automatically use open ODCRs +|<> + |=== @@ -111,3 +116,15 @@ include::associate-workload.adoc[leveloffset=+1] include::critical-workload.adoc[leveloffset=+1] include::auto-net-pol.adoc[leveloffset=+1] + +include::tag-subnets-auto.adoc[leveloffset=+1] + +include::auto-cis.adoc[leveloffset=+1] + +include::auto-kms.adoc[leveloffset=+1] + +include::auto-controls.adoc[leveloffset=+1] + +include::auto-odcr.adoc[leveloffset=+1] + +include::auto-local-zone.adoc[leveloffset=+1] diff --git a/latest/ug/automode/tag-subnets-auto.adoc b/latest/ug/automode/tag-subnets-auto.adoc new file mode 100644 index 000000000..63de75d02 --- /dev/null +++ b/latest/ug/automode/tag-subnets-auto.adoc @@ -0,0 +1,84 @@ +include::../attributes.txt[] + +[.topic] +[#tag-subnets-auto] += Tag subnets for EKS Auto Mode +:info_titleabbrev: Tag subnets + +If you use the load balancing capability of EKS Auto Mode, you need to add {aws} tags to your VPC subnets. + +== Background + +These tags identify subnets as associated with the cluster, and more importantly if the subnet is public or private. + +Public subnets have direct internet access via an internet gateway. They are used for resources that need to be publicly accessible such as load balancers. + +Private subnets do not have direct internet access and use NAT gateways for outbound traffic. They are used for internal resources such as EKS nodes that don't need public IPs. + +To learn more about NAT gateways and Internet gateways, see link:vpc/latest/userguide/extend-intro.html["Connect your VPC to other networks",type="documentation"] in the Amazon Virtual Private Cloud (VPC) User Guide. + +== Requirement + +At this time, subnets used for load balancing by EKS Auto Mode are required to have one of the following tags. + +=== Public subnets +Public subnets are used for internet-facing load balancers. These subnets must have the following tags: + +[%header,cols="2"] +|=== +|Key +|Value + + +|`kubernetes.io/role/elb` +|`1` or `` + +|=== + +=== Private subnets +Private subnets are used for internal load balancers. These subnets must have the following tags: + +[%header,cols="2"] +|=== +|Key +|Value + +|`kubernetes.io/role/internal-elb` +|`1` or `` +|=== + +== Procedure + +Before you begin, identify which subnets are public (with Internet Gateway access) and which are private (using NAT Gateway). You'll need permissions to modify VPC resources. + +[#auto-tag-subnets-console] +=== {aws-management-console} + +. Open the Amazon VPC console and navigate to Subnets +. Select the subnet to tag +. Choose the Tags tab and select Add tag +. Add the appropriate tag: +* For public subnets: Key=`kubernetes.io/role/elb` +* For private subnets: Key=`kubernetes.io/role/internal-elb` +. Set Value to `1` or leave empty +. Save and repeat for remaining subnets + +=== {aws} CLI + +For public subnets: +[source,bash] +---- +aws ec2 create-tags \ + --resources subnet-ID \ + --tags Key=kubernetes.io/role/elb,Value=1 +---- + +For private subnets: +[source,bash] +---- +aws ec2 create-tags \ + --resources subnet-ID \ + --tags Key=kubernetes.io/role/internal-elb,Value=1 +---- + +Replace `subnet-ID` with your actual subnet ID. diff --git a/latest/ug/automode/troubleshoot-lbc.adoc b/latest/ug/automode/troubleshoot-lbc.adoc index c9918fde8..213878707 100644 --- a/latest/ug/automode/troubleshoot-lbc.adoc +++ b/latest/ug/automode/troubleshoot-lbc.adoc @@ -1,20 +1,17 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[troubleshoot-lbc,troubleshoot-lbc.title]] -# Troubleshooting Amazon EKS Auto Mode Load Balancer Controller -:info_doctype: section +[#troubleshoot-lbc] += Troubleshooting Amazon EKS Auto Mode Load Balancer Controller [NOTE] ==== This resource is not ready for publication. ==== - -include::../attributes.txt[] - This guide helps you troubleshoot issues with the {aws} Load Balancer Controller when using Amazon EKS Auto Mode. -## Verify Ingress Resources +== Verify Ingress Resources Check the status of your Ingress resources: @@ -29,30 +26,30 @@ Look for: - ALB DNS name in the Address field - Events indicating any issues -## Check {aws} Resources +== Check {aws} Resources -Verify these resources in the {aws} Management Console or using {aws} CLI: +Verify these resources in the {aws-management-console} or using {aws} CLI: - Application Load Balancers - Target Groups - Security Groups -## Common Issues +== Common Issues -### Ingress Not Creating ALB +=== Ingress Not Creating ALB 1. Verify Ingress annotations 2. Check security group configuration 3. Validate subnet configuration 4. Review target group settings -### Target Group Health Check Failures +=== Target Group Health Check Failures 1. Ensure security group allows health check traffic 2. Verify application endpoints are responding 3. Check health check path and settings -### Networking Issues +=== Networking Issues 1. Verify subnet tagging: - Public subnets: `kubernetes.io/role/elb: 1` @@ -60,22 +57,22 @@ Verify these resources in the {aws} Management Console or using {aws} CLI: 2. Check VPC internet connectivity for public ALBs 3. Review route tables and NAT gateway configuration -### Security Group Problems +=== Security Group Problems 1. Verify inbound rules allow traffic on required ports 2. Ensure outbound rules allow health check traffic 3. Check security group associations -## Advanced Troubleshooting +== Advanced Troubleshooting -### Version Compatibility +=== Version Compatibility Ensure compatibility between: - Kubernetes version - EKS version - {aws} SDK version -### Resource Cleanup +=== Resource Cleanup For stuck resources: @@ -85,21 +82,21 @@ For stuck resources: ``` 2. Check for orphaned {aws} resources (ALBs, target groups, listener rules) -## Best Practices +== Best Practices 1. Regularly review Ingress events and {aws} resource synchronization 2. Document custom annotations and maintain troubleshooting runbooks 3. Test changes in non-production environments first -## Considerations for EKS Auto Mode +== Considerations for EKS Auto Mode - Limited customization options - {aws} manages controller updates - IAM permissions are handled automatically - Focus on monitoring {aws} resources and Ingress events -## Additional Resources +== Additional Resources -- link:https://docs.aws.amazon.com/eks/latest/userguide/troubleshooting.html[{aws} EKS Troubleshooting] -- link:https://console.aws.amazon.com/support/home[{aws} Support Center] -- link:https://status.aws.amazon.com/[{aws} Service Health Dashboard] +- <> +- link:support/home[{aws} Support Center,type="console"] +- link:https://status.aws.amazon.com/[{aws} Service Health Dashboard] \ No newline at end of file diff --git a/latest/ug/automode/wip.adoc b/latest/ug/automode/wip.adoc index 3ae07b146..b14aa485b 100644 --- a/latest/ug/automode/wip.adoc +++ b/latest/ug/automode/wip.adoc @@ -1,12 +1,8 @@ -//!!NODE_ROOT +include::../attributes.txt[] + [.topic] -[[auto-wip,auto-wip.title]] +[#auto-wip] = EKS Auto Mode: Revisions to existing pages WIP -:info_doctype: section -:toc: left - - -include::../attributes.txt[] This section contains in-progress revisions for existing pages in other chapters. @@ -18,4 +14,4 @@ This section contains in-progress revisions for existing pages in other chapters //include::wip/auto-cluster-iam-role.adoc[leveloffset=+1] -//include::wip/auto-create-node-role.adoc[leveloffset=+1] +//include::wip/auto-create-node-role.adoc[leveloffset=+1] \ No newline at end of file diff --git a/latest/ug/automode/wip/create-vpc-console.adoc b/latest/ug/automode/wip/create-vpc-console.adoc index f96c7cff8..453499fc2 100644 --- a/latest/ug/automode/wip/create-vpc-console.adoc +++ b/latest/ug/automode/wip/create-vpc-console.adoc @@ -1,12 +1,10 @@ -//!!NODE_ROOT
+include::../../attributes.txt[] + [.topic] -[[create-vpc-console,create-vpc-console.title]] +[#create-vpc-console] = Create a VPC for Amazon EKS with the web console -:info_doctype: section - -include::../../attributes.txt[] -This guide walks you through creating a Virtual Private Cloud (VPC) that's optimized for Amazon Elastic Kubernetes Service (EKS) clusters using the {aws} Management Console. +This guide walks you through creating a Virtual Private Cloud (VPC) that's optimized for Amazon Elastic Kubernetes Service (EKS) clusters using the {aws-management-console}. == Overview @@ -19,14 +17,14 @@ When creating a VPC for EKS, you'll need to configure specific networking requir == Prerequisites - An {aws} account -- Access to the {aws} Management Console +- Access to the {aws-management-console} - IAM permissions for VPC and EKS resource creation - Planned CIDR ranges for your VPC, pods, and services == Creation Steps === Step 1: Access the VPC Creation Page -1. Sign in to the {aws} Management Console +1. Sign in to the {aws-management-console} 2. Navigate to the VPC Dashboard 3. Click "Create VPC" @@ -145,4 +143,4 @@ kubernetes.io/role/internal-elb = 1 - EKS VPC Requirements Documentation - EKS Best Practices Guide - VPC Pricing Calculator -- EKS Networking Documentation +- EKS Networking Documentation \ No newline at end of file diff --git a/latest/ug/automode/wip/eksctl-docs.adoc b/latest/ug/automode/wip/eksctl-docs.adoc index f7f7c4789..858cfa892 100644 --- a/latest/ug/automode/wip/eksctl-docs.adoc +++ b/latest/ug/automode/wip/eksctl-docs.adoc @@ -1,14 +1,12 @@ -//!!NODE_ROOT
+include::../../attributes.txt[] + [.topic] -[[eksctl-docs,eksctl-docs.title]] +[#eksctl-docs] = EKS Auto Mode -:info_doctype: section - -include::../../attributes.txt[] == Introduction -eksctl supports xref:automode[EKS Auto Mode], a feature that extends {aws} management of Kubernetes clusters beyond the cluster itself, +eksctl supports <>, a feature that extends {aws} management of Kubernetes clusters beyond the cluster itself, to allow {aws} to also set up and manage the infrastructure that enables the smooth operation of your workloads. This allows you to delegate key infrastructure decisions and leverage the expertise of {aws} for day-to-day operations. Cluster infrastructure managed by {aws} includes many Kubernetes capabilities as core components, as opposed to add-ons, @@ -116,4 +114,4 @@ $ eksctl update auto-mode-config -f cluster.yaml == Further information -- xref:automode[EKS Auto Mode] +- <> \ No newline at end of file diff --git a/latest/ug/automode/wip/tag-subnets.adoc b/latest/ug/automode/wip/tag-subnets.adoc index 073651955..6f885672a 100644 --- a/latest/ug/automode/wip/tag-subnets.adoc +++ b/latest/ug/automode/wip/tag-subnets.adoc @@ -1,32 +1,39 @@ -//!!NODE_ROOT
+include::../../attributes.txt[] + [.topic] -[[tag-subnets,tag-subnets.title]] +[#tag-subnets] = Tag VPC Subnets for Load Balancer Deployment -:info_doctype: section - -include::../../attributes.txt[] -This topic explains how to tag your VPC subnets to enable load balancer deployment using the {aws} Management Console. +This topic explains how to tag your VPC subnets to enable load balancer deployment using the {aws-management-console}. == Required Tags Your subnets require specific tags based on their intended use: -[options="header",cols="1,2,1"] +[%header,cols="1,2,1"] |=== -|Subnet Type |Tag Key |Tag Value -|Private Subnet |`kubernetes.io/role/internal-elb` |`1` -|Public Subnet |`kubernetes.io/role/elb` |`1` +|Subnet Type +|Tag Key +|Tag Value + +|Private Subnet +|`kubernetes.io/role/internal-elb` +|`1` + +|Public Subnet +|`kubernetes.io/role/elb` +|`1` + |=== == Adding Tags in the Console -1. Sign in to the {aws} Management Console -2. Navigate to **VPC** +>+ **Subnets** +1. Sign in to the {aws-management-console} +2. Navigate to *VPC* +>+ *Subnets* 3. Select the subnet you want to tag -4. Select the **Tags** tab in the lower panel -5. Choose **Add/Edit Tags** -6. Click **Add Tag** and enter: +4. Select the *Tags* tab in the lower panel +5. Choose *Add/Edit Tags* +6. Click *Add Tag* and enter: - For private subnets: Key = `kubernetes.io/role/internal-elb`, Value = `1` - For public subnets: Key = `kubernetes.io/role/elb`, Value = `1` -7. Click **Save** +7. Click *Save* \ No newline at end of file diff --git a/latest/ug/book.adoc b/latest/ug/book.adoc index 8d87ae743..19abdd4c7 100644 --- a/latest/ug/book.adoc +++ b/latest/ug/book.adoc @@ -1,7 +1,3 @@ -//!!NODE_ROOT -include::attributes.txt[] -[[top]] -= Amazon EKS :doctype: book :toc: left :icons: font @@ -11,14 +7,9 @@ include::attributes.txt[] :info_doctype: book :info_title: Amazon EKS :info_subtitle: User Guide -:info_abstract: This is official Amazon Web Services ({aws}) documentation for Amazon Elastic Kubernetes Service (Amazon EKS). Amazon EKS is a managed \ - service that makes it easy for you to run Kubernetes on {aws} without needing to install and operate \ - your own Kubernetes clusters. Kubernetes is an open-source system for automating the deployment, scaling, \ - and management of containerized applications. :info_corpauthor: Amazon Web Services :info_publisher: Amazon Web Services -:info_copyright: 2024 \ -Amazon Web Services, Inc. and/or its affiliates. All rights reserved. +:info_copyright: Amazon Web Services, Inc. and/or its affiliates. All rights reserved. :info_legalnotice: Amazon's trademarks and trade dress may not be used in \ connection with any product or service that is not Amazon's, \ in any manner that is likely to cause confusion among customers, \ @@ -26,19 +17,22 @@ or in any manner that disparages or discredits Amazon. All other \ trademarks not owned by Amazon are the property of their respective \ owners, who may or may not be affiliated with, connected to, or \ sponsored by Amazon. -:keywords: EKS, Amazon EKS, Kubernetes, K8s, Cluster, Pod + +include::attributes.txt[] + +[[top]] += Amazon EKS [abstract] -- -This is official Amazon Web Services ({aws}) documentation for Amazon Elastic Kubernetes Service (Amazon EKS). Amazon EKS is a managed service that makes it easy for you to run [.noloc]`Kubernetes` on {aws} without needing to install and operate your own [.noloc]`Kubernetes` clusters. [.noloc]`Kubernetes` is an open-source system for automating the deployment, scaling, and management of containerized applications. +This is official Amazon Web Services ({aws}) documentation for Amazon Elastic Kubernetes Service (Amazon EKS). Amazon EKS is a managed service that makes it easy for you to run Kubernetes on {aws} without needing to install and operate your own Kubernetes clusters. Kubernetes is an open source system for automating the deployment, scaling, and management of containerized applications. -- -:sectnums: [.banner.info] *Help improve this page* [.banner.info] -Want to contribute to this user guide? Choose the *Edit this page on GitHub* link that is located in the right pane of every page. Your contributions will help make our user guide better for everyone. +To contribute to this user guide, choose the *Edit this page on GitHub* link that is located in the right pane of every page. include::what-is/what-is-eks.adoc[leveloffset=+1] @@ -64,7 +58,7 @@ include::networking/eks-networking.adoc[leveloffset=+1] include::workloads/eks-workloads.adoc[leveloffset=+1] -include::clusters/management/eks-managing.adoc[leveloffset=+1] +include::cluster-management/eks-managing.adoc[leveloffset=+1] include::security/security.adoc[leveloffset=+1] @@ -72,14 +66,16 @@ include::observability/eks-observe.adoc[leveloffset=+1] include::integrations/eks-integrations.adoc[leveloffset=+1] -include::troubleshooting/troubleshooting.adoc[leveloffset=+1] - include::connector/eks-connector.adoc[leveloffset=+1] include::outposts/eks-outposts.adoc[leveloffset=+1] include::ml/machine-learning-on-eks.adoc[leveloffset=+1] +include::versioning/versioning.adoc[leveloffset=+1] + +include::troubleshooting/troubleshooting.adoc[leveloffset=+1] + include::related-projects.adoc[leveloffset=+1] include::roadmap.adoc[leveloffset=+1] @@ -87,4 +83,3 @@ include::roadmap.adoc[leveloffset=+1] include::doc-history.adoc[leveloffset=+1] include::contribute/contribute.adoc[leveloffset=+1] - diff --git a/latest/ug/cluster-management/cost-monitoring-aws.adoc b/latest/ug/cluster-management/cost-monitoring-aws.adoc new file mode 100644 index 000000000..355436424 --- /dev/null +++ b/latest/ug/cluster-management/cost-monitoring-aws.adoc @@ -0,0 +1,30 @@ +include::../attributes.txt[] + +[.topic] +[#cost-monitoring-aws] += View costs by Pod in {aws} billing with split cost allocation +:info_titleabbrev: View costs by Pod + +== Cost monitoring using {aws} split cost allocation data for Amazon EKS +You can use {aws} split cost allocation data for Amazon EKS to get granular cost visibility for your Amazon EKS clusters. This enables you to analyze, optimize, and chargeback cost and usage for your Kubernetes applications. You allocate application costs to individual business units and teams based on Amazon EC2 CPU and memory resources consumed by your Kubernetes application. Split cost allocation data for Amazon EKS gives visibility into cost per Pod, and enables you to aggregate the cost data per Pod using namespace, cluster, and other Kubernetes primitives. The following are examples of Kubernetes primitives that you can use to analyze Amazon EKS cost allocation data. + +* Cluster name +* Deployment +* Namespace +* Node +* Workload Name +* Workload Type + +link:costmanagement/home#/tags[User-defined cost allocation tags,type="console"] are also supported. For more information about using split cost allocation data, see link:cur/latest/userguide/split-cost-allocation-data.html[Understanding split cost allocation data,type="documentation"] in the {aws} Billing User Guide. + + +[#task-cur-setup] +== Set up Cost and Usage Reports + +You can turn on Split Cost Allocation Data for EKS in the Cost Management Console, {aws} Command Line Interface, or the {aws} SDKs. + +Use the following for _Split Cost Allocation Data_: + +. Opt in to Split Cost Allocation Data. For more information, see link:cur/latest/userguide/enabling-split-cost-allocation-data.html[Enabling split cost allocation data,type="documentation"] in the {aws} Cost and Usage Report User Guide. +. Include the data in a new or existing report. +. View the report. You can use the Billing and Cost Management console or view the report files in Amazon Simple Storage Service. diff --git a/latest/ug/cluster-management/cost-monitoring-kubecost-bundles.adoc b/latest/ug/cluster-management/cost-monitoring-kubecost-bundles.adoc new file mode 100644 index 000000000..2ef97a63e --- /dev/null +++ b/latest/ug/cluster-management/cost-monitoring-kubecost-bundles.adoc @@ -0,0 +1,327 @@ +include::../attributes.txt[] + +[.topic] +[#cost-monitoring-kubecost-bundles] += Learn more about Kubecost + +Amazon EKS provides an {aws} optimized bundle of Kubecost for cluster cost visibility. Amazon EKS supports Kubecost, which you can use to monitor your costs broken down by Kubernetes resources including Pods, nodes, namespaces, and labels. + +This topic covers the available versions of Kubecost, and the differences between the available tiers. EKS supports Kubecost Version 1 and Version 2. Each version is available in different tiers. You can use _Amazon EKS optimized Kubecost bundle_ for your Amazon EKS clusters at no additional cost. You may be charged for use of associated {aws} services, such as Amazon Managed Service for Prometheus. Also, you can use your existing {aws} support agreements to obtain support. + +As a Kubernetes platform administrator and finance leader, you can use Kubecost to visualize a breakdown of Amazon EKS charges, allocate costs, and charge back organizational units such as application teams. You can provide your internal teams and business units with transparent and accurate cost data based on their actual {aws} bill. Moreover, you can also get customized recommendations for cost optimization based on their infrastructure environment and usage patterns within their clusters. For more information about Kubecost, see the https://www.ibm.com/docs/en/kubecost/self-hosted/2.x[Kubecost] documentation. + +*What is the difference between the custom bundle of Kubecost and the free version of Kubecost (also known as OpenCost)?* + +{aws} and Kubecost collaborated to offer a customized version of Kubecost. This version includes a subset of commercial features at no additional charge. See the tables below for features that are included with in the custom bundle of Kubecost. + +[#kubecost-v2] +== Kubecost v2 + +*What is the difference between Kubecost v1 and v2?* + +Kubecost 2.0 is a major upgrade from previous versions and includes major new features including a brand new API Backend. Note the https://www.ibm.com/docs/en/kubecost/self-hosted/2.x?topic=apis-allocation-api[Allocation] and https://www.ibm.com/docs/en/kubecost/self-hosted/2.x?topic=apis-assets-api[Assets] APIs are fully backwards compatible. https://www.ibm.com/docs/en/kubecost/self-hosted/2.x?topic=installation-kubecost-v2-installupgrade[Please review the Kubecost documentation to ensure a smooth transition.] For the full list of enhancements, https://github.com/kubecost/cost-analyzer-helm-chart/releases/tag/v2.0.0[please see the Kubecost v2.0 announcement] and https://github.com/kubecost/cost-analyzer-helm-chart/releases[the full release notes]. + +[IMPORTANT] +==== + +https://www.ibm.com/docs/en/kubecost/self-hosted/2.x[Review the Kubecost documentation before upgrading.] Upgrading may impact report availability. + +==== + +*Core features comparison:* + +[%header,cols="4"] +|=== +|Feature +|Kubecost free tier 2.0 +|Amazon EKS optimized Kubecost bundle 2.0 +|Kubecost Enterprise 2.0 + + +|Cluster cost visibility +|Unlimited clusters up to 250 cores +|Unified multi-cluster without core limits when integrated with Amazon Managed Service for Prometheus +|Unified and unlimited number of clusters across unlimited numbers of environments (i.e. multi-cloud) + +|Deployment +|User hosted +|User hosted +|User hosted, Kubecost hosted (dedicated tenant), SaaS + +|Databases supported +|Local Prometheus +|Amazon Managed Service for Prometheus or Local Prometheus +|Any prometheus flavor and custom databases + +|Database retention support (raw metrics) +|15 days +|Unlimited historical data +|Unlimited historical data + +|Kubecost API and UI retention (ETL) +|15 days +|15 days +|Unlimited + +|Hybrid cloud visibility +|- +|Amazon EKS and Amazon EKS Anywhere clusters +|Multi-cloud and hybrid cloud + +|Alerts and recurring reports +|Only supported on the primary cluster, limited to 250 cores +|Efficiency alerts, budget alerts, spend change alerts, and https://www.ibm.com/docs/en/kubecost/self-hosted/2.x?topic=navigating-kubecost-ui#ariaid-title6[more supported] across all clusters +|Efficiency alerts, budget alerts, spend change alerts, and https://www.ibm.com/docs/en/kubecost/self-hosted/2.x?topic=navigating-kubecost-ui#ariaid-title6[more supported] across all clusters + +|Saved reports +|- +|Reports using 15 days of metrics +|Reports using unlimited historical data and metrics + +|Cloud billing integration +|Only supported on the primary cluster, limited to 250 cores +|Custom pricing support for {aws} (including multiple clusters and multiple accounts) +|Custom pricing support for any cloud + +|Savings recommendations +|Only supported on the primary cluster, limited to 250 cores +|Primary cluster insights, but there is no 250 core limit +|Multi-cluster insights + +|Governance: Audits +|- +|- +|Audit historical cost events + +|Single sign-on (SSO) support +|- +|Amazon Cognito supported +|Okta, Auth0, PingID, KeyCloak, and anything else custom + +|Role-based access control (RBAC) with SAML 2.0 +|- +|- +|Okta, Auth0, PingID, KeyCloak, and anything else custom + +|Enterprise training and onboarding +|- +|- +|Full-service training and FinOps onboarding + +|Teams +|- +|- +|Yes + +|=== + +*New Features:* + +The following features have metric limits: + + + +* Kubecost Aggregator +* Network Monitoring +* Kubecost Actions +* Collections +* Anomaly detection +* Container Request Right-Sizing +* Kubecost Forecasting +* Autocomplete for filtering and aggregation + +*Metric limits:* + +[%header,cols="4"] +|=== +|Metric +|Kubecost Free Tier 2.0 +|Amazon EKS optimized Kubecost bundle 2.0 +|Kubecost Enterprise 2.0 + + +|Cluster size +|Unlimited clusters up to 250 cores +|Unlimited +|Unlimited + +|Metric retention +|15 days +|15 days +|Unlimited + +|Multi-cluster support +|Not available +|Available +|Available + +|Core limits +|250 cores per cluster +|No core limits +|No core limits + +|=== + +[#kubecost-v1] +== Kubecost v1 + +[%header,cols="4"] +|=== +|Feature +|Kubecost free tier +|Amazon EKS optimized Kubecost bundle +|Kubecost Enterprise + + +|*Deployment* +|User hosted +|User hosted +|User hosted or Kubecost hosted (SaaS) + +|*Number of clusters supported* +|Unlimited +|Unlimited +|Unlimited + +|*Databases supported* +|Local Prometheus +|Local Prometheus or Amazon Managed Service for Prometheus +|Prometheus, Amazon Managed Service for Prometheus, Cortex, or Thanos + +|*Database retention support* +|15 days +|Unlimited historical data +|Unlimited historical data + +|*Kubecost API retention (ETL)* +|15 days +|15 days +|Unlimited historical data + +|*Cluster cost visibility* +|Single clusters +|Unified multi-cluster +|Unified multi-cluster + +|*Hybrid cloud visibility* +|- +|Amazon EKS and Amazon EKS Anywhere clusters +|Multi-cloud and hybrid-cloud support + +|*Alerts and recurring reports* +|- +|Efficiency alerts, budget alerts, spend change alerts, and more supported +|Efficiency alerts, budget alerts, spend change alerts, and more supported + +|*Saved reports* +|- +|Reports using 15 days data +|Reports using unlimited historical data + +|*Cloud billing integration* +|Required for each individual cluster +|Custom pricing support for {aws} (including multiple clusters and multiple accounts) +|Custom pricing support for {aws} (including multiple clusters and multiple accounts) + +|*Savings recommendations* +|Single cluster insights +|Single cluster insights +|Multi-cluster insights + +|*Governance: Audits* +|- +|- +|Audit historical cost events + +|*Single sign-on (SSO) support* +|- +|Amazon Cognito supported +|Okta, Auth0, PingID, KeyCloak + +|*Role-based access control (RBAC) with SAML `2.0`* +|- +|- +|Okta, Auth0, PingID, Keycloak + +|*Enterprise training and onboarding* +|- +|- +|Full-service training and FinOps onboarding + +|=== + +[#cost-monitoring-faq] +== Frequently asked questions + +See the following common questions and answers about using Kubecost with Amazon EKS. + +*What is the Kubecost API retention (ETL) feature?* + +The Kubecost ETL feature aggregates and organizes metrics to surface cost visibility at various levels of granularity (such as `namespace-level`, `pod-level`, and `deployment-level`). For _Amazon EKS optimized Kubecost bundle_, customers get data and insights from metrics for the last 15 days. + +*What is the alerts and recurring reports feature? What alerts and reports does it include?* + +Kubecost alerts allow teams to receive updates on real-time Kubernetes spend as well as cloud spend. Recurring reports enable teams to receive customized views of historical Kubernetes and cloud spend. Both are configurable using the Kubecost UI or Helm values. They support email, Slack, and Microsoft Teams. + +*What do saved reports include?* + +Kubecost saved reports are predefined views of cost and efficiency metrics. They include cost by cluster, namespace, label, and more. + +*What is cloud billing integration?* + +Integration with {aws} billing APIs allows Kubecost to display out-of-cluster costs (such as Amazon S3). Additionally, it allows Kubecost to reconcile Kubecost's in-cluster predictions with actual billing data to account for spot usage, savings plans, and enterprise discounts. + +*What do savings recommendations include?* + +Kubecost provides insights and automation to help users optimize their Kubernetes infrastructure and spend. + +*Is there a charge for this functionality?* + +No. You can use _Amazon EKS optimized Kubecost bundle_ at no additional charge. If you want additional Kubecost capabilities that aren't included, you can buy an Enterprise License of Kubecost through the {aws} Marketplace, or from Kubecost directly. + +*Is support available for _Amazon EKS optimized Kubecost bundle_?* + +Yes, only if you are using the _Amazon EKS optimized Kubecost bundle_. + +*How do I get support for _Amazon EKS optimized Kubecost bundle_?* + +You can open a support case with the {aws} Support team at link:contact-us/[Contact {aws},type="marketing"]. + +*Do I need a license to use Kubecost features provided by the Amazon EKS integration?* + +No. + +*Can I integrate Kubecost with {aws} Cost and Usage Report for more accurate reporting?* + +Yes. You can configure Kubecost to ingest data from {aws} Cost and Usage Report to get accurate cost visibility, including discounts, Spot pricing, reserved instance pricing, and others. For more information, see https://www.ibm.com/docs/en/kubecost/self-hosted/2.x?topic=integrations-aws-cloud-billing-integration[{aws} Cloud Billing Integration] in the Kubecost documentation. + +*Does this version support cost management of self-managed Kubernetes clusters on Amazon EC2?* + +No. _Amazon EKS optimized Kubecost bundle_ only compatible with Amazon EKS clusters. + +*Can Kubecost track costs for Amazon EKS on {aws} Fargate?* + +Kubecost provides best effort to show cluster cost visibility for Amazon EKS on Fargate, but with lower accuracy than with Amazon EKS on Amazon EC2. This is primarily due to the difference in how you're billed for your usage. With Amazon EKS on Fargate, you're billed for consumed resources. With Amazon EKS on Amazon EC2 nodes, you're billed for provisioned resources. Kubecost calculates the cost of an Amazon EC2 node based on the node specification, which includes CPU, RAM, and ephemeral storage. With Fargate, costs are calculated based on the requested resources for the Fargate Pods. + +*How can I get updates and new versions of Kubecost?* + +You can upgrade your Kubecost version using standard Helm upgrade procedures. The latest versions are in the https://gallery.ecr.aws/kubecost/cost-analyzer[Amazon ECR Public Gallery]. + +*Is the `kubectl-cost` CLI supported? How do I install it?* + +Yes. `Kubectl-cost` is an open source tool by Kubecost (Apache 2.0 License) that provides CLI access to Kubernetes cost allocation metrics. To install `kubectl-cost`, see https://github.com/kubecost/kubectl-cost#installation[Installation] on GitHub. + +*Is the Kubecost user interface supported? How do I access it?* + +Kubecost provides a web dashboard that you can access through `kubectl` port forwarding, an ingress, or a load balancer. You can also use the {aws} Load Balancer Controller to expose Kubecost and use Amazon Cognito for authentication, authorization, and user management. For more information, see link:containers/how-to-use-application-load-balancer-and-amazon-cognito-to-authenticate-users-for-your-kubernetes-web-apps[How to use Application Load Balancer and Amazon Cognito to authenticate users for your Kubernetes web apps,type="blog"] on the {aws} blog. + +[#kubecost-additional] +== Additional Kubecost Features + +* The following features are available in both Kubecost v1 and v2. +** *Export cost metrics* – Amazon EKS optimized cost monitoring is deployed with Kubecost and Prometheus, which is an open-source monitoring system and time series database. Kubecost reads metric from Prometheus and then performs cost allocation calculations and writes the metrics back to Prometheus. The Kubecost front-end reads metrics from Prometheus and shows them on the Kubecost user interface. The architecture is illustrated in the following diagram. ++ +image::images/kubecost-architecture.png[Kubecost architecture,scaledwidth=100%] ++ +With https://prometheus.io/[Prometheus] pre-installed, you can write queries to ingest Kubecost data into your current business intelligence system for further analysis. You can also use it as a data source for your current https://grafana.com/[Grafana] dashboard to display Amazon EKS cluster costs that your internal teams are familiar with. To learn more about how to write Prometheus queries, see the https://opencost.io/docs/installation/prometheus/[Prometheus Configuration]``readme`` file on GitHub or use the example Grafana JSON models in the https://github.com/kubecost/cost-analyzer-helm-chart/tree/develop/cost-analyzer[Kubecost Github repository] as references. +** *{aws} Cost and Usage Report integration* – To perform cost allocation calculations for your Amazon EKS cluster, Kubecost retrieves the public pricing information of {aws} services and {aws} resources from the {aws} Price List API. You can also integrate Kubecost with *{aws} Cost and Usage Report* to enhance the accuracy of the pricing information specific to your {aws} account. This information includes enterprise discount programs, reserved instance usage, savings plans, and spot usage. To learn more about how the {aws} Cost and Usage Report integration works, see https://www.ibm.com/docs/en/kubecost/self-hosted/2.x?topic=integrations-aws-cloud-billing-integration[{aws} Cloud Billing Integration] in the Kubecost documentation. diff --git a/latest/ug/cluster-management/cost-monitoring-kubecost-view-dashboard.adoc b/latest/ug/cluster-management/cost-monitoring-kubecost-view-dashboard.adoc new file mode 100644 index 000000000..fe7455d7e --- /dev/null +++ b/latest/ug/cluster-management/cost-monitoring-kubecost-view-dashboard.adoc @@ -0,0 +1,41 @@ +include::../attributes.txt[] + +[.topic] +[#cost-monitoring-kubecost-dashboard] += Access Kubecost Dashboard + +[#kubecost-prereqs-dashboard] +== Prerequisites + +. Make sure the kubecost related Pods' state are "Running". + +[source,bash,subs="verbatim,attributes"] +---- +kubectl get pods --namespace kubecost +---- + +[#kubecost-dashboard] +== Access Kubecost Dashboard + +. On your device, enable port-forwarding to expose the Kubecost dashboard. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl port-forward deployment/kubecost-cost-analyzer 9090 --namespace kubecost +---- ++ +Alternatively, you can use the <> to expose Kubecost and use Amazon Cognito for authentication, authorization, and user management. For more information, see link:containers/how-to-use-application-load-balancer-and-amazon-cognito-to-authenticate-users-for-your-kubernetes-web-apps[How to use Application Load Balancer and Amazon Cognito to authenticate users for your Kubernetes web apps,type="blog"]. +. On the same device that you completed the previous step on, open a web browser and enter the following address. ++ +[source,bash,subs="verbatim,attributes"] +---- +http://localhost:9090 +---- ++ +You see the Kubecost Overview page in your browser. It might take 5–10 minutes (or more) for Kubecost to gather metrics, depends on your cluster size. You can see your Amazon EKS spend, including cumulative cluster costs, associated Kubernetes asset costs, and monthly aggregated spend. ++ +. To track costs at a cluster level, tag your Amazon EKS resources for billing. For more information, see <>. + + +* *Cost allocation* – View monthly Amazon EKS costs and cumulative costs for each of your namespaces and other dimensions over the past seven days. This is helpful for understanding which parts of your application are contributing to Amazon EKS spend. +* *Assets* – View the costs of the {aws} infrastructure assets that are associated with your Amazon EKS resources. \ No newline at end of file diff --git a/latest/ug/cluster-management/cost-monitoring-kubecost.adoc b/latest/ug/cluster-management/cost-monitoring-kubecost.adoc new file mode 100644 index 000000000..5ac023f8f --- /dev/null +++ b/latest/ug/cluster-management/cost-monitoring-kubecost.adoc @@ -0,0 +1,33 @@ +include::../attributes.txt[] + +[.topic] +[#cost-monitoring-kubecost] += Install Kubecost +:info_titleabbrev: Install Kubecost + +Amazon EKS supports Kubecost, which you can use to monitor your costs broken down by Kubernetes resources including Pods, nodes, namespaces, and labels. This topic covers installing Kubecost, and accessing the Kubecost dashboard. + +Amazon EKS provides an {aws} optimized bundle of Kubecost for cluster cost visibility. You can use your existing {aws} support agreements to obtain support. For more information about the available versions of Kubecost, see <>. + +[NOTE] +==== + +Kubecost v2 introduces several major new features. <> + +==== + +For more information about Kubecost, see the https://www.ibm.com/docs/en/kubecost/self-hosted/2.x[Kubecost] documentation and <>. + +[#kubecost-overview] +== Install Amazon EKS optimized Kubecost bundle + +You can use one of the following procedures to install the _Amazon EKS optimized Kubecost bundle_: + +* Before start, it is recommended to review https://www.ibm.com/docs/en/kubecost/self-hosted/2.x?topic=installations-amazon-eks-integration[Kubecost - Architecture Overview] to understand how Kubecost works on Amazon EKS. +* If you are new to Amazon EKS, we recommend that you use the Amazon EKS add-on for the installation because it simplifies the _Amazon EKS optimized Kubecost bundle_ installation. For more information, see https://www.ibm.com/docs/en/kubecost/self-hosted/2.x?topic=installations-amazon-eks-integration#ariaid-title3[Deploying Kubecost on an Amazon EKS cluster using Amazon EKS add-on]. +* To customize the installation, you might configure your _Amazon EKS optimized Kubecost bundle_ with Helm. For more information, see https://www.ibm.com/docs/en/kubecost/self-hosted/2.x?topic=installations-amazon-eks-integration#ariaid-title8[Deploying Kubecost on an Amazon EKS cluster using Helm] in the _Kubecost documentation_. + +[#kubecost-access-dashbaard] +== Access Kubecost dashboard + +Once the _Amazon EKS optimized Kubecost bundle_ setup done, you should have access to Kubecost dashboard. For more information, see <>. diff --git a/latest/ug/cluster-management/cost-monitoring.adoc b/latest/ug/cluster-management/cost-monitoring.adoc new file mode 100644 index 000000000..5c5600a4c --- /dev/null +++ b/latest/ug/cluster-management/cost-monitoring.adoc @@ -0,0 +1,26 @@ +include::../attributes.txt[] + +[.topic] +[#cost-monitoring] += Monitor and optimize Amazon EKS cluster costs +:info_titleabbrev: Cost monitoring + +[abstract] +-- +Learn how to monitor and optimize costs for your Amazon EKS clusters using {aws} Billing split cost allocation data or Kubecost, a Kubernetes-native cost monitoring tool integrated with {aws}. +-- + +Cost monitoring is an essential aspect of managing your Kubernetes clusters on Amazon EKS. By gaining visibility into your cluster costs, you can optimize resource utilization, set budgets, and make data-driven decisions about your deployments. Amazon EKS provides two cost monitoring solutions, each with its own unique advantages, to help you track and allocate your costs effectively: + +*{aws} Billing split cost allocation data for Amazon EKS* -- This native feature integrates seamlessly with the {aws} Billing Console, allowing you to analyze and allocate costs using the same familiar interface and workflows you use for other {aws} services. With split cost allocation, you can gain insights into your Kubernetes costs directly alongside your other {aws} spend, making it easier to optimize costs holistically across your {aws} environment. You can also leverage existing {aws} Billing features like Cost Categories and Cost Anomaly Detection to further enhance your cost management capabilities. For more information, see link:cur/latest/userguide/split-cost-allocation-data.html[Understanding split cost allocation data,type="documentation"] in the {aws} Billing User Guide. + +*Kubecost* -- Amazon EKS supports Kubecost, a Kubernetes cost monitoring tool. Kubecost offers a feature-rich, Kubernetes-native approach to cost monitoring, providing granular cost breakdowns by Kubernetes resources, cost optimization recommendations, and out-of-the-box dashboards and reports. Kubecost also retrieves accurate pricing data by integrating with the {aws} Cost and Usage Report, ensuring you get a precise view of your Amazon EKS costs. Learn how to <>. See the https://aws.amazon.com/marketplace/pp/prodview-asiz4x22pm2n2[Kubecost] {aws} Marketplace page for information on getting a free Kubecost subscription. + + +include::cost-monitoring-aws.adoc[leveloffset=+1] + +include::cost-monitoring-kubecost.adoc[leveloffset=+1] + +include::cost-monitoring-kubecost-view-dashboard.adoc[leveloffset=+1] + +include::cost-monitoring-kubecost-bundles.adoc[leveloffset=+1] diff --git a/latest/ug/cluster-management/eks-managing.adoc b/latest/ug/cluster-management/eks-managing.adoc new file mode 100644 index 000000000..10d95c265 --- /dev/null +++ b/latest/ug/cluster-management/eks-managing.adoc @@ -0,0 +1,23 @@ +include::../attributes.txt[] + +[#eks-managing] += Organize and monitor cluster resources +:info_titleabbrev: Cluster management + +This chapter includes the following topics to help you manage your cluster. You can also view information about your <> with the {aws-management-console}. + +* The Kubernetes Dashboard is a general purpose, web-based UI for Kubernetes clusters. It allows users to manage applications running in the cluster and troubleshoot them, as well as manage the cluster itself. For more information, see The https://github.com/kubernetes/dashboard[Kubernetes Dashboard] GitHub repository. +* <> – The Kubernetes Metrics Server is an aggregator of resource usage data in your cluster. It isn't deployed by default in your cluster, but is used by Kubernetes add-ons, such as the Kubernetes Dashboard and <>. In this topic you learn how to install the Metrics Server. +* <> – The Helm package manager for Kubernetes helps you install and manage applications on your Kubernetes cluster. This topic helps you install and run the Helm binaries so that you can install and manage charts using the Helm CLI on your local computer. +* <> – To help you manage your Amazon EKS resources, you can assign your own metadata to each resource in the form of _tags_. This topic describes tags and shows you how to create them. +* <> – Your {aws} account has default quotas, formerly referred to as limits, for each {aws} service. Learn about the quotas for Amazon EKS and how to increase them. + +include::cost-monitoring.adoc[leveloffset=+1] + +include::metrics-server.adoc[leveloffset=+1] + +include::helm.adoc[leveloffset=+1] + +include::eks-using-tags.adoc[leveloffset=+1] + +include::service-quotas.adoc[leveloffset=+1] \ No newline at end of file diff --git a/latest/ug/clusters/management/eks-using-tags.adoc b/latest/ug/cluster-management/eks-using-tags.adoc similarity index 78% rename from latest/ug/clusters/management/eks-using-tags.adoc rename to latest/ug/cluster-management/eks-using-tags.adoc index 447b87998..9069494ec 100644 --- a/latest/ug/clusters/management/eks-using-tags.adoc +++ b/latest/ug/cluster-management/eks-using-tags.adoc @@ -1,21 +1,16 @@ -//!!NODE_ROOT
-include::../../attributes.txt[] +include::../attributes.txt[] [.topic] -[[eks-using-tags,eks-using-tags.title]] +[#eks-using-tags] = Organize Amazon EKS resources with tags -:info_doctype: section -:info_title: Organize Amazon EKS resources with tags :info_titleabbrev: Tagging your resources -:keywords: metadata, tag, resources -:info_abstract: Learn how to use tags to categorize and manage your Amazon EKS resources like clusters, managed node groups, and Fargate profiles for billing, cost allocation, and resource identification. [abstract] -- Learn how to use tags to categorize and manage your Amazon EKS resources like clusters, managed node groups, and Fargate profiles for billing, cost allocation, and resource identification. -- -You can use _tags_ to help you manage your Amazon EKS resources. This topic provides an overview of the tags function and shows how you can create tags. +You can use _tags_ to help you manage your Amazon EKS resources. This topic provides an overview of the tags function and shows how you can create tags. [.topiclist] [[Topic List]] @@ -23,7 +18,7 @@ You can use _tags_ to help you manage your Amazon EKS resources. This topic pro [NOTE] ==== -Tags are a type of metadata that's separate from [.noloc]`Kubernetes` labels and annotations. For more information about these other metadata types, see the following sections in the [.noloc]`Kubernetes` documentation: +Tags are a type of metadata that's separate from Kubernetes labels and annotations. For more information about these other metadata types, see the following sections in the Kubernetes documentation: @@ -32,10 +27,10 @@ Tags are a type of metadata that's separate from [.noloc]`Kubernetes` labels and ==== -[[tag-basics,tag-basics.title]] +[#tag-basics] == Tag basics -A tag is a label that you assign to an {aws} resource. Each tag consists of a _key_ and an optional _value_. +A tag is a label that you assign to an {aws} resource. Each tag consists of a _key_ and an optional _value_. With tags, you can categorize your {aws} resources. For example, you can categorize resources by purpose, owner, or environment. When you have many resources of the same type, you can use the tags that you assigned to a specific resource to quickly identify that resource. For example, you can define a set of tags for your Amazon EKS clusters to help you track each cluster's owner and stack level. We recommend that you devise a consistent set of tag keys for each resource type. You can then search and filter the resources based on the tags that you add. @@ -45,7 +40,7 @@ Tags don't have any semantic meaning to Amazon EKS and are interpreted strictly If you use {aws} Identity and Access Management (IAM), you can control which users in your {aws} account have permission to manage tags. -[[tag-resources,tag-resources.title]] +[#tag-resources] == Tagging your resources The following Amazon EKS resources support tags: @@ -62,13 +57,13 @@ You can tag these resources using the following: * If you're using the Amazon EKS console, you can apply tags to new or existing resources at any time. You can do this by using the *Tags* tab on the relevant resource page. For more information, see <>. * If you're using `eksctl`, you can apply tags to resources when they're created using the `--tags` option. -* If you're using the {aws} CLI, the Amazon EKS API, or an {aws} SDK, you can apply tags to new resources using the `tags` parameter on the relevant API action. You can apply tags to existing resources using the `TagResource` API action. For more information, see link:eks/latest/APIReference/API_TagResource.html[TagResource,type="documentation"]. +* If you're using the {aws} CLI, the Amazon EKS API, or an {aws} SDK, you can apply tags to new resources using the `tags` parameter on the relevant API action. You can apply tags to existing resources using the `TagResource` API action. For more information, see link:eks/latest/APIReference/API_TagResource.html[TagResource,type="documentation"]. When you use some resource-creating actions, you can also specify tags for the resource at the same time that you create it. If tags can't be applied while the resource is being created, the resource fails to be created. This mechanism ensures that resources that you intend to tag are either created with the tags that you specify or not created at all. If you tag resources when you create them, you don't need to run custom tagging scripts after you create the resource. -Tags don't propagate to other resources that are associated with the resource that you create. For example, Fargate profile tags don't propagate to other resources that are associated with the Fargate profile, such as the [.noloc]`Pods` that are scheduled with it. +Tags don't propagate to other resources that are associated with the resource that you create. For example, Fargate profile tags don't propagate to other resources that are associated with the Fargate profile, such as the Pods that are scheduled with it. -[[tag-restrictions,tag-restrictions.title]] +[#tag-restrictions] == Tag restrictions The following restrictions apply to tags: @@ -82,12 +77,12 @@ The following restrictions apply to tags: * Don't use `aws:`, `{aws}:`, or any upper or lowercase combination of such as a prefix for either keys or values. These are reserved only for {aws} use. You can't edit or delete tag keys or values with this prefix. Tags with this prefix don't count against your tags-per-resource limit. -[[tag-resources-for-billing,tag-resources-for-billing.title]] +[#tag-resources-for-billing] == Tagging your resources for billing -When you apply tags to Amazon EKS clusters, you can use them for cost allocation in your *Cost & Usage Reports*. The metering data in your *Cost & Usage Reports* shows usage across all of your Amazon EKS clusters. For more information, see link:awsaccountbilling/latest/aboutv2/billing-reports-costusage.html[{aws} cost and usage report,type="documentation"] in the _{aws} Billing User Guide_. +When you apply tags to Amazon EKS clusters, you can use them for cost allocation in your *Cost & Usage Reports*. The metering data in your *Cost & Usage Reports* shows usage across all of your Amazon EKS clusters. For more information, see link:awsaccountbilling/latest/aboutv2/billing-reports-costusage.html[{aws} cost and usage report,type="documentation"] in the _{aws} Billing User Guide_. -The {aws} generated cost allocation tag, specifically `aws:eks:cluster-name`, lets you break down Amazon EC2 instance costs by individual Amazon EKS cluster in *Cost Explorer*. However, this tag doesn't capture the control plane expenses. The tag is automatically added to Amazon EC2 instances that participate in an Amazon EKS cluster. This behavior happens regardless of whether the instances are provisioned using Amazon EKS managed node groups, [.noloc]`Karpenter`, or directly with Amazon EC2. This specific tag doesn't count towards the 50 tags limit. To use the tag, the account owner must activate it in the {aws} Billing console or by using the API. When an {aws} Organizations management account owner activates the tag, it's also activated for all organization member accounts. +The {aws} generated cost allocation tag, specifically `aws:eks:cluster-name`, lets you break down Amazon EC2 instance costs by individual Amazon EKS cluster in *Cost Explorer*. However, this tag doesn't capture the control plane expenses. The tag is automatically added to Amazon EC2 instances that participate in an Amazon EKS cluster. This behavior happens regardless of whether the instances are provisioned using Amazon EKS managed node groups, Karpenter, or directly with Amazon EC2. This specific tag doesn't count towards the 50 tags limit. To use the tag, the account owner must activate it in the {aws} Billing console or by using the API. When an {aws} Organizations management account owner activates the tag, it's also activated for all organization member accounts. You can also organize your billing information based on resources that have the same tag key values. For example, you can tag several resources with a specific application name, and then organize your billing information. That way, you can see the total cost of that application across several services. For more information about setting up a cost allocation report with tags, see link:awsaccountbilling/latest/aboutv2/configurecostallocreport.html[The Monthly Cost Allocation Report,type="documentation"] in the _{aws} Billing User Guide_. @@ -98,23 +93,23 @@ If you just enabled reporting, data for the current month is available for viewi ==== -*Cost Explorer* is a reporting tool that's available as part of the {aws} Free Tier. You can use *Cost Explorer* to view charts of your Amazon EKS resources from the last 13 months. You can also forecast how much you're likely to spend for the next three months. You can see patterns in how much you spend on {aws} resources over time. For example, you can use it to identify areas that need further inquiry and see trends that you can use to understand your costs. You also can specify time ranges for the data, and view time data by day or by month. +*Cost Explorer* is a reporting tool that's available as part of the {aws} Free Tier. You can use *Cost Explorer* to view charts of your Amazon EKS resources from the last 13 months. You can also forecast how much you're likely to spend for the next three months. You can see patterns in how much you spend on {aws} resources over time. For example, you can use it to identify areas that need further inquiry and see trends that you can use to understand your costs. You also can specify time ranges for the data, and view time data by day or by month. -[[tag-resources-console,tag-resources-console.title]] +[#tag-resources-console] == Working with tags using the console Using the Amazon EKS console, you can manage the tags that are associated with new or existing clusters and managed node groups. -When you select a resource-specific page in the Amazon EKS console, the page displays a list of those resources. For example, if you select *Clusters* from the left navigation pane, the console displays a list of Amazon EKS clusters. When you select a resource from one of these lists (for example, a specific cluster) that supports tags, you can view and manage its tags on the *Tags* tab. +When you select a resource-specific page in the Amazon EKS console, the page displays a list of those resources. For example, if you select *Clusters* from the left navigation pane, the console displays a list of Amazon EKS clusters. When you select a resource from one of these lists (for example, a specific cluster) that supports tags, you can view and manage its tags on the *Tags* tab. You can also use *Tag Editor* in the {aws-management-console}, which provides a unified way to manage your tags. For more information, see link:ARG/latest/userguide/tag-editor.html[Tagging your {aws} resources with Tag Editor,type="documentation"] in the _{aws} Tag Editor User Guide_. -[[adding-tags-creation,adding-tags-creation.title]] +[#adding-tags-creation] === Adding tags on a resource on creation You can add tags to Amazon EKS clusters, managed node groups, and Fargate profiles when you create them. For more information, see <>. -[[adding-or-deleting-tags,adding-or-deleting-tags.title]] +[#adding-or-deleting-tags] === Adding and deleting tags on a resource You can add or delete the tags that are associated with your clusters directly from the resource's page. @@ -123,7 +118,7 @@ You can add or delete the tags that are associated with your clusters directly f . On the navigation bar, select the {aws} Region to use. . In the left navigation pane, choose *Clusters*. . Choose a specific cluster. -. Choose the *Tags* tab, and then choose *Manage tags*. +. Choose the *Tags* tab, and then choose *Manage tags*. . On the *Manage tags* page, add or delete your tags as necessary. + ** To add a tag, choose *Add tag*. Then specify the key and value for each tag. @@ -132,12 +127,12 @@ You can add or delete the tags that are associated with your clusters directly f . Choose *Update* to finish. -[[tag-resources-api-sdk,tag-resources-api-sdk.title]] +[#tag-resources-api-sdk] == Working with tags using the CLI, API, or `eksctl` Use the following {aws} CLI commands or Amazon EKS API operations to add, update, list, and delete the tags for your resources. You can only use `eksctl` to add tags while simultaneously creating the new resources with one command. [[tag-eks-resources-table]] -[cols="1,1,1,1", options="header"] +[%header,cols="4"] |=== |Task |{aws} CLI @@ -184,7 +179,7 @@ aws eks list-tags-for-resource --resource-arn resource_ARN When you use some resource-creating actions, you can specify tags at the same time that you create the resource. The following actions support specifying a tag when you create a resource. -[cols="1,1,1,1,1", options="header"] +[%header,cols="5"] |=== |Task |{aws} CLI @@ -212,4 +207,4 @@ When you use some resource-creating actions, you can specify tags at the same ti |`create fargateprofile` |=== -* If you want to also tag the Amazon EC2 instances when you create a managed node group, create the managed node group using a launch template. For more information, see <>. If your instances already exist, you can manually tag the instances. For more information, see link:AWSEC2/latest/UserGuide/Using_Tags.html#tag-resources[Tagging your resources,type="documentation"] in the Amazon EC2 User Guide. +* If you want to also tag the Amazon EC2 instances when you create a managed node group, create the managed node group using a launch template. For more information, see <>. If your instances already exist, you can manually tag the instances. For more information, see link:AWSEC2/latest/UserGuide/Using_Tags.html#tag-resources[Tagging your resources,type="documentation"] in the Amazon EC2 User Guide. \ No newline at end of file diff --git a/latest/ug/clusters/management/helm.adoc b/latest/ug/cluster-management/helm.adoc similarity index 60% rename from latest/ug/clusters/management/helm.adoc rename to latest/ug/cluster-management/helm.adoc index 56c8385ed..b683733c6 100644 --- a/latest/ug/clusters/management/helm.adoc +++ b/latest/ug/cluster-management/helm.adoc @@ -1,20 +1,16 @@ -//!!NODE_ROOT
-include::../../attributes.txt[] +include::../attributes.txt[] [.topic] -[[helm,helm.title]] -= Deploy applications with [.noloc]`Helm` on Amazon EKS -:info_doctype: section -:info_title: Deploy applications with Helm on Amazon EKS +[#helm] += Deploy applications with Helm on Amazon EKS :info_titleabbrev: Deploy apps with Helm -:info_abstract: Learn how to install and use Helm, a package manager for Kubernetes, with your Amazon EKS cluster to manage and deploy applications seamlessly. [abstract] -- Learn how to install and use Helm, a package manager for Kubernetes, with your Amazon EKS cluster to manage and deploy applications seamlessly. -- -The Helm package manager for [.noloc]`Kubernetes` helps you install and manage applications on your [.noloc]`Kubernetes` cluster. For more information, see the https://docs.helm.sh/[Helm documentation]. This topic helps you install and run the Helm binaries so that you can install and manage charts using the Helm CLI on your local system. +The Helm package manager for Kubernetes helps you install and manage applications on your Kubernetes cluster. For more information, see the https://docs.helm.sh/[Helm documentation]. This topic helps you install and run the Helm binaries so that you can install and manage charts using the Helm CLI on your local system. [IMPORTANT] ==== @@ -35,20 +31,7 @@ kubectl get svc ---- brew install helm ---- -** If you're using [.noloc]`Windows` with https://chocolatey.org/[Chocolatey], install the binaries with the following command. -+ -[source,bash,subs="verbatim,attributes"] ----- -choco install kubernetes-helm ----- -** If you're using [.noloc]`Linux`, install the binaries with the following commands. -+ -[source,bash,subs="verbatim,attributes"] ----- -curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 > get_helm.sh -chmod 700 get_helm.sh -./get_helm.sh ----- +** For more installation options, see https://helm.sh/docs/intro/install/[Installing Helm] in the Helm Docs. + NOTE: If you get a message that `openssl` must first be installed, you can install it with the following command. @@ -61,17 +44,19 @@ sudo yum install openssl + [source,bash,subs="verbatim,attributes"] ---- -helm version | cut -d + -f 1 +helm version --template='{{ .Version }}{{ "\n" }}' ---- + An example output is as follows. + [source,bash,subs="verbatim,attributes"] ---- -v3.9.0 +v3.17.2 ---- +. Make sure the version installed is compatible with your cluster version. Check https://helm.sh/docs/topics/version_skew/#supported-version-skew[Supported Version Skew] to learn more. For example, if you are running with `3.17.x`, supported Kubernetes version should not out of the range of `1.29.x` ~ `1.32.x`. ++ . At this point, you can run any Helm commands (such as `helm install [.replaceable]``chart-name```) to install, modify, delete, or query Helm charts in your cluster. If you're new to Helm and don't have a specific chart to install, you can: + ** Experiment by installing an example chart. See https://helm.sh/docs/intro/quickstart#install-an-example-chart[Install an example chart] in the Helm https://helm.sh/docs/intro/quickstart/[Quickstart guide]. ** Create an example chart and push it to Amazon ECR. For more information, see link:AmazonECR/latest/userguide/push-oci-artifact.html[Pushing a Helm chart,type="documentation"] in the _Amazon Elastic Container Registry User Guide_. -** Install an Amazon EKS chart from the https://github.com/aws/eks-charts#eks-charts[eks-charts][.noloc]`GitHub` repo or from https://artifacthub.io/packages/search?page=1&repo=aws[ArtifactHub]. +** Install an Amazon EKS chart from the https://github.com/aws/eks-charts#eks-charts[eks-charts]GitHub repo or from https://artifacthub.io/packages/search?page=1&repo=aws[ArtifactHub]. diff --git a/latest/ug/cluster-management/images b/latest/ug/cluster-management/images new file mode 120000 index 000000000..5e6757319 --- /dev/null +++ b/latest/ug/cluster-management/images @@ -0,0 +1 @@ +../images \ No newline at end of file diff --git a/latest/ug/clusters/management/metrics-server.adoc b/latest/ug/cluster-management/metrics-server.adoc similarity index 65% rename from latest/ug/clusters/management/metrics-server.adoc rename to latest/ug/cluster-management/metrics-server.adoc index c53b83554..c82e0d426 100644 --- a/latest/ug/clusters/management/metrics-server.adoc +++ b/latest/ug/cluster-management/metrics-server.adoc @@ -1,20 +1,16 @@ -//!!NODE_ROOT
-include::../../attributes.txt[] +include::../attributes.txt[] [.topic] -[[metrics-server,metrics-server.title]] -= View resource usage with the [.noloc]`Kubernetes` [.noloc]`Metrics Server` -:info_doctype: section -:info_title: View resource usage with the KubernetesMetrics Server +[#metrics-server] += View resource usage with the Kubernetes Metrics Server :info_titleabbrev: Metrics server -:info_abstract: Use the Kubernetes Metrics Server to view resource usage data on your Amazon EKS cluster for autoscaling and monitoring. [abstract] -- Use the Kubernetes Metrics Server to view resource usage data on your Amazon EKS cluster for autoscaling and monitoring. -- -The [.noloc]`Kubernetes` Metrics Server is an aggregator of resource usage data in your cluster, and it isn't deployed by default in Amazon EKS clusters. For more information, see https://github.com/kubernetes-sigs/metrics-server[Kubernetes Metrics Server] on [.noloc]`GitHub`. The Metrics Server is commonly used by other [.noloc]`Kubernetes` add ons, such as the <> or the <>. For more information, see https://kubernetes.io/docs/tasks/debug/debug-cluster/resource-metrics-pipeline/[Resource metrics pipeline] in the [.noloc]`Kubernetes` documentation. This topic explains how to deploy the [.noloc]`Kubernetes` Metrics Server on your Amazon EKS cluster. +The Kubernetes Metrics Server is an aggregator of resource usage data in your cluster, and it isn't deployed by default in Amazon EKS clusters. For more information, see https://github.com/kubernetes-sigs/metrics-server[Kubernetes Metrics Server] on GitHub. The Metrics Server is commonly used by other Kubernetes add ons, such as the <> or the <>. For more information, see https://kubernetes.io/docs/tasks/debug/debug-cluster/resource-metrics-pipeline/[Resource metrics pipeline] in the Kubernetes documentation. This topic explains how to deploy the Kubernetes Metrics Server on your Amazon EKS cluster. [IMPORTANT] ==== @@ -22,6 +18,11 @@ The [.noloc]`Kubernetes` Metrics Server is an aggregator of resource usage data The metrics are meant for point-in-time analysis and aren't an accurate source for historical analysis. They can't be used as a monitoring solution or for other non-auto scaling purposes. For information about monitoring tools, see <>. ==== +== Considerations + +* If manually deploying Kubernetes Metrics Server onto Fargate nodes using the manifest, configure the `metrics-server` deployment to use a port other than its default of `10250`. This port is reserved for Fargate. The Amazon EKS add-on version of Metrics Server is pre-configured to use port `10251`. +* Ensure security groups and network ACLs allow port `10250` between the `metrics-server` Pods and all other nodes and Pods. The Kubernetes Metrics Server still uses port `10250` to collect metrics from other endpoints in the cluster. If you deploy on Fargate nodes, allow both the configured alternate Metrics Server port and port `10250`. + == Deploy as community add-on with Amazon EKS Add-ons *New: You can now deploy Metrics Server as a community add-on using the {aws} console or Amazon EKS APIs.* @@ -57,7 +58,7 @@ kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/late ---- + If you are using Fargate, you will need to change this file. In the default configuration, the metrics server uses port 10250. This port is reserved on Fargate. Replace references to port 10250 in components.yaml with another port, such as 10251. -. Verify that the `metrics-server` deployment is running the desired number of [.noloc]`Pods` with the following command. +. Verify that the `metrics-server` deployment is running the desired number of Pods with the following command. + [source,bash,subs="verbatim,attributes"] ---- diff --git a/latest/ug/clusters/management/service-quotas.adoc b/latest/ug/cluster-management/service-quotas.adoc similarity index 62% rename from latest/ug/clusters/management/service-quotas.adoc rename to latest/ug/cluster-management/service-quotas.adoc index 132c39421..b4bf1ca86 100644 --- a/latest/ug/clusters/management/service-quotas.adoc +++ b/latest/ug/cluster-management/service-quotas.adoc @@ -1,13 +1,9 @@ -//!!NODE_ROOT
-include::../../attributes.txt[] +include::../attributes.txt[] [.topic] -[[service-quotas,service-quotas.title]] -= View and manage Amazon EKS and [.noloc]`Fargate` service quotas -:info_doctype: section -:info_title: View and manage Amazon EKS and Fargate service quotas +[#service-quotas] += View and manage Amazon EKS and Fargate service quotas :info_titleabbrev: Service quotas -:info_abstract: Use Service Quotas to view and manage Amazon EKS and {aws} Fargate quotas from the {aws-management-console} or {aws} CLI. [abstract] -- @@ -17,16 +13,16 @@ Use Service Quotas to view and manage Amazon EKS and {aws} Fargate quotas from t Amazon EKS has integrated with Service Quotas, an {aws} service that you can use to view and manage your quotas from a central location. For more information, see link:servicequotas/latest/userguide/intro.html[What Is Service Quotas?,type="documentation"] in the _Service Quotas User Guide_. With Service Quotas integration, you can quickly look up the value of your Amazon EKS and {aws} Fargate service quotas using the {aws-management-console} and {aws} CLI. -[[service-quotas-console,service-quotas-console.title]] -== View EKS service quotas in the {aws} Management Console +[#service-quotas-console] +== View EKS service quotas in the {aws-management-console} . Open the link:servicequotas/home/services/eks/quotas["Service Quotas console",type="console"]. . In the left navigation pane, choose *{aws} services*. -. From the *{aws} services* list, search for and select *Amazon Elastic Kubernetes Service (Amazon EKS)* or *{aws} Fargate*. +. From the *{aws} services* list, search for and select *Amazon Elastic Kubernetes Service (Amazon EKS)* or *{aws} Fargate*. + In the *Service quotas* list, you can see the service quota name, applied value (if it's available), {aws} default quota, and whether the quota value is adjustable. . To view additional information about a service quota, such as the description, choose the quota name. -. (Optional) To request a quota increase, select the quota that you want to increase, select *Request quota increase*, enter or select the required information, and select *Request*. +. (Optional) To request a quota increase, select the quota that you want to increase, select *Request quota increase*, enter or select the required information, and select *Request*. To work more with service quotas using the {aws-management-console}, see the link:servicequotas/latest/userguide/intro.html[Service Quotas User Guide,type="documentation"]. To request a quota increase, see link:servicequotas/latest/userguide/request-quota-increase.html[Requesting a Quota Increase,type="documentation"] in the _Service Quotas User Guide_. @@ -54,42 +50,41 @@ aws service-quotas list-aws-default-service-quotas \ --output table ---- -NOTE: The quota returned is the number of Amazon ECS tasks or Amazon EKS [.noloc]`Pods` that can run concurrently on Fargate in this account in the current {aws} Region. +NOTE: The quota returned is the number of Amazon ECS tasks or Amazon EKS Pods that can run concurrently on Fargate in this account in the current {aws} Region. To work more with service quotas using the {aws} CLI, see link:cli/latest/reference/service-quotas/index.html[service-quotas,type="documentation"] in the _{aws} CLI Command Reference_. To request a quota increase, see the link:cli/latest/reference/service-quotas/request-service-quota-increase.html[request-service-quota-increase,type="documentation"] command in the _{aws} CLI Command Reference_. -[[sq-text,sq-text.title]] +[#sq-text] == Amazon EKS service quotas -{aws} recommends using the {aws} management console to view your current quotas. For more information, see <>. +{aws} recommends using the {aws-management-console} to view your current quotas. For more information, see <>. To view the default EKS service quotas, see link:general/latest/gr/eks.html#limits_eks["Amazon Elastic Kubernetes Service endpoints and quotas",type="documentation"] in the _{aws} General Reference_. -These service quotas are listed under *Amazon Elastic Kubernetes Service (Amazon EKS)* in the Service Quotas console. To request a quota increase for values that are shown as adjustable, see link:servicequotas/latest/userguide/request-quota-increase.html[Requesting a quota increase,type="documentation"] in the _Service Quotas User Guide_. +These service quotas are listed under *Amazon Elastic Kubernetes Service (Amazon EKS)* in the Service Quotas console. To request a quota increase for values that are shown as adjustable, see link:servicequotas/latest/userguide/request-quota-increase.html[Requesting a quota increase,type="documentation"] in the _Service Quotas User Guide_. [NOTE] ==== -The following quotas aren't available in Service Quotas: - -* Pod Identity associations per cluster is [.noloc]`1000` in each supported region and this quota isn't adjustable. -* You can use up to 15 CIDRs for Remote Node Networks and 15 CIDRs for Remote Pod Networks per cluster for hybrid nodes. This quota isn't adjustable. +Adjustments to the following components are **not** supported in Service Quotas: +* Pod Identity associations per cluster. For limits, see <>. +* CIDRs for Remote Node Networks or Remote Pod Networks for hybrid nodes. For limits, see <>. ==== -[[service-quotas-eks-fargate,service-quotas-eks-fargate.title]] +[#service-quotas-eks-fargate] == {aws} Fargate service quotas -The *{aws} Fargate* service in the Service Quotas console lists several service quotas. You can configure alarms that alert you when your usage approaches a service quota. For more information, see <>. +The *{aws} Fargate* service in the Service Quotas console lists several service quotas. You can configure alarms that alert you when your usage approaches a service quota. For more information, see <>. New {aws} accounts might have lower initial quotas that can increase over time. Fargate constantly monitors the account usage within each {aws} Region, and then automatically increases the quotas based on the usage. You can also request a quota increase for values that are shown as adjustable. For more information, see link:servicequotas/latest/userguide/request-quota-increase.html[Requesting a quota increase,type="documentation"] in the _Service Quotas User Guide_. -{aws} reccomends using the {aws} management console to view your current quotas. For more information, see <>. +{aws} recomends using the {aws-management-console} to view your current quotas. For more information, see <>. To view default {aws} Fargate on EKS service quotas, see link:general/latest/gr/eks.html#service-quotas-eks-fargate["Fargate service quotas",type="documentation"] in the _{aws} General Reference_. [NOTE] ==== -Fargate additionally enforces Amazon ECS tasks and Amazon EKS [.noloc]`Pods` launch rate quotas. For more information, see link:AmazonECS/latest/developerguide/throttling.html[{aws} Fargate throttling quotas,type="documentation"] in the _Amazon ECS guide_. +Fargate additionally enforces Amazon ECS tasks and Amazon EKS Pods launch rate quotas. For more information, see link:AmazonECS/latest/developerguide/throttling.html[{aws} Fargate throttling quotas,type="documentation"] in the _Amazon ECS guide_. -==== +==== \ No newline at end of file diff --git a/latest/ug/clusters/autoscaling.adoc b/latest/ug/clusters/autoscaling.adoc index 38abb8a6b..5d721142f 100644 --- a/latest/ug/clusters/autoscaling.adoc +++ b/latest/ug/clusters/autoscaling.adoc @@ -1,20 +1,16 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[autoscaling,autoscaling.title]] -= Scale cluster compute with [.noloc]`Karpenter` and [.noloc]`Cluster Autoscaler` -:info_doctype: section -:info_title: Scale cluster compute with Karpenter and Cluster Autoscaler +[#autoscaling] += Scale cluster compute with Karpenter and Cluster Autoscaler :info_titleabbrev: Autoscaling -:info_abstract: Discover how Amazon EKS integrates Kubernetes autoscaling with {aws}, empowering rapid and efficient scaling of compute resources to meet application demands using Karpenter and Cluster Autoscaler. [abstract] -- Discover how Amazon EKS integrates Kubernetes autoscaling with {aws}, empowering rapid and efficient scaling of compute resources to meet application demands using Karpenter and Cluster Autoscaler. -- -Autoscaling is a function that automatically scales your resources out and in to meet changing demands. This is a major [.noloc]`Kubernetes` function that would otherwise require extensive human resources to perform manually. +Autoscaling is a function that automatically scales your resources out and in to meet changing demands. This is a major Kubernetes function that would otherwise require extensive human resources to perform manually. == EKS Auto Mode @@ -32,8 +28,8 @@ Amazon EKS supports two additional autoscaling products: -*[.noloc]`Karpenter`*:: -[.noloc]`Karpenter` is a flexible, high-performance [.noloc]`Kubernetes` cluster autoscaler that helps improve application availability and cluster efficiency. [.noloc]`Karpenter` launches right-sized compute resources (for example, Amazon EC2 instances) in response to changing application load in under a minute. Through integrating [.noloc]`Kubernetes` with {aws}, [.noloc]`Karpenter` can provision just-in-time compute resources that precisely meet the requirements of your workload. [.noloc]`Karpenter` automatically provisions new compute resources based on the specific requirements of cluster workloads. These include compute, storage, acceleration, and scheduling requirements. Amazon EKS supports clusters using [.noloc]`Karpenter`, although [.noloc]`Karpenter` works with any conformant [.noloc]`Kubernetes` cluster. For more information, see the https://karpenter.sh/docs/[Karpenter] documentation. +*Karpenter*:: +Karpenter is a flexible, high-performance Kubernetes cluster autoscaler that helps improve application availability and cluster efficiency. Karpenter launches right-sized compute resources (for example, Amazon EC2 instances) in response to changing application load in under a minute. Through integrating Kubernetes with {aws}, Karpenter can provision just-in-time compute resources that precisely meet the requirements of your workload. Karpenter automatically provisions new compute resources based on the specific requirements of cluster workloads. These include compute, storage, acceleration, and scheduling requirements. Amazon EKS supports clusters using Karpenter, although Karpenter works with any conformant Kubernetes cluster. For more information, see the https://karpenter.sh/docs/[Karpenter] documentation. + [IMPORTANT] ==== @@ -41,4 +37,4 @@ Karpenter is open-source software which {aws} customers are responsible for inst ==== *Cluster Autoscaler*:: -The [.noloc]`Kubernetes` Cluster Autoscaler automatically adjusts the number of nodes in your cluster when pods fail or are rescheduled onto other nodes. The Cluster Autoscaler uses Auto Scaling groups. For more information, see https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md[Cluster Autoscaler on {aws}]. +The Kubernetes Cluster Autoscaler automatically adjusts the number of nodes in your cluster when pods fail or are rescheduled onto other nodes. The Cluster Autoscaler uses Auto Scaling groups. For more information, see https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md[Cluster Autoscaler on {aws}]. \ No newline at end of file diff --git a/latest/ug/clusters/cluster-endpoint.adoc b/latest/ug/clusters/cluster-endpoint.adoc index dc1096ace..154130ff9 100644 --- a/latest/ug/clusters/cluster-endpoint.adoc +++ b/latest/ug/clusters/cluster-endpoint.adoc @@ -1,24 +1,20 @@ -//!!NODE_ROOT
-[.topic] -[[cluster-endpoint,cluster-endpoint.title]] -= Control network access to cluster API server endpoint -:info_doctype: section -:info_title: Control network access to cluster API server endpoint -:info_titleabbrev: Configure endpoint access -:info_abstract: Learn how to enable private access and limit public access to the Amazon EKS cluster Kubernetes API server endpoint for enhanced security with your Amazon EKS cluster. - include::../attributes.txt[] +[.topic] +[#cluster-endpoint] += Cluster API server endpoint +:info_titleabbrev: Cluster endpoint access + [abstract] -- Learn how to enable private access and limit public access to the Amazon EKS cluster Kubernetes API server endpoint for enhanced security with your Amazon EKS cluster. -- -This topic helps you to enable private access for your Amazon EKS cluster's [.noloc]`Kubernetes` API server endpoint and limit, or completely disable, public access from the internet. +This topic helps you to enable private access for your Amazon EKS cluster's Kubernetes API server endpoint and limit, or completely disable, public access from the internet. -When you create a new cluster, Amazon EKS creates an endpoint for the managed [.noloc]`Kubernetes` API server that you use to communicate with your cluster (using [.noloc]`Kubernetes` management tools such as `kubectl`). By default, this API server endpoint is public to the internet, and access to the API server is secured using a combination of {aws} Identity and Access Management (IAM) and native [.noloc]`Kubernetes` https://kubernetes.io/docs/reference/access-authn-authz/rbac/[Role Based Access Control] (RBAC). This endpoint is known as the _cluster public endpoint_. Also there is a _cluster private endpoint_. For more information about the cluster private endpoint, see the following section <>. +When you create a new cluster, Amazon EKS creates an endpoint for the managed Kubernetes API server that you use to communicate with your cluster (using Kubernetes management tools such as `kubectl`). By default, this API server endpoint is public to the internet, and access to the API server is secured using a combination of {aws} Identity and Access Management (IAM) and native Kubernetes https://kubernetes.io/docs/reference/access-authn-authz/rbac/[Role Based Access Control] (RBAC). This endpoint is known as the _cluster public endpoint_. Also there is a _cluster private endpoint_. For more information about the cluster private endpoint, see the following section <>. -[[cluster-endpoint-ipv6,cluster-endpoint-ipv6.title]] +[#cluster-endpoint-ipv6] == `IPv6` cluster endpoint format EKS creates a unique dual-stack endpoint in the following format for new `IPv6` clusters that are made after October 2024. An _IPv6 cluster_ is a cluster that you select `IPv6` in the IP family (`ipFamily`) setting of the cluster. @@ -33,7 +29,7 @@ EKS cluster public/private endpoint: EKS cluster public/private endpoint: `eks-cluster.[.replaceable]``region``.api.aws` -{amazon-web-services} in China:: +Amazon Web Services in China:: EKS cluster public/private endpoint: `eks-cluster.[.replaceable]``region``.api.amazonwebservices.com.cn` @@ -44,7 +40,7 @@ EKS cluster public/private endpoint: The dual-stack cluster endpoint was introduced in October 2024. For more information about `IPv6` clusters, see <>. Clusters made before October 2024, use following endpoint format instead. ==== -[[cluster-endpoint-ipv4,cluster-endpoint-ipv4.title]] +[#cluster-endpoint-ipv4] == `IPv4` cluster endpoint format EKS creates a unique endpoint in the following format for each cluster that select `IPv4` in the IP family (ipFamily) setting of the cluster: @@ -59,9 +55,9 @@ EKS cluster public/private endpoint EKS cluster public/private endpoint `eks-cluster.[.replaceable]``region``.eks.amazonaws.com` -{amazon-web-services} in China:: +Amazon Web Services in China:: EKS cluster public/private endpoint -`eks-cluster.[.replaceable]``region``.api.amazonwebservices.com.cn` +`eks-cluster.[.replaceable]``region``.amazonwebservices.com.cn` ==== @@ -70,28 +66,28 @@ EKS cluster public/private endpoint Before October 2024, `IPv6` clusters used this endpoint format also. For those clusters, both the public endpoint and the private endpoint have only `IPv4` addresses resolve from this endpoint. ==== -[[cluster-endpoint-private,cluster-endpoint-private.title]] +[#cluster-endpoint-private] == Cluster private endpoint -You can enable private access to the [.noloc]`Kubernetes` API server so that all communication between your nodes and the API server stays within your VPC. You can limit the IP addresses that can access your API server from the internet, or completely disable internet access to the API server. +You can enable private access to the Kubernetes API server so that all communication between your nodes and the API server stays within your VPC. You can limit the IP addresses that can access your API server from the internet, or completely disable internet access to the API server. [NOTE] ==== -Because this endpoint is for the [.noloc]`Kubernetes` API server and not a traditional {aws} PrivateLink endpoint for communicating with an {aws} API, it doesn't appear as an endpoint in the Amazon VPC console. +Because this endpoint is for the Kubernetes API server and not a traditional {aws} PrivateLink endpoint for communicating with an {aws} API, it doesn't appear as an endpoint in the Amazon VPC console. ==== -When you enable endpoint private access for your cluster, Amazon EKS creates a Route 53 private hosted zone on your behalf and associates it with your cluster's VPC. This private hosted zone is managed by Amazon EKS, and it doesn't appear in your account's Route 53 resources. In order for the private hosted zone to properly route traffic to your API server, your VPC must have `enableDnsHostnames` and `enableDnsSupport` set to `true`, and the DHCP options set for your VPC must include `AmazonProvidedDNS` in its domain name servers list. For more information, see link:vpc/latest/userguide/vpc-dns.html#vpc-dns-updating[Updating DNS support for your VPC,type="documentation"] in the _Amazon VPC User Guide_. +When you enable endpoint private access for your cluster, Amazon EKS creates a Route 53 private hosted zone on your behalf and associates it with your cluster's VPC. This private hosted zone is managed by Amazon EKS, and it doesn't appear in your account's Route 53 resources. In order for the private hosted zone to properly route traffic to your API server, your VPC must have `enableDnsHostnames` and `enableDnsSupport` set to `true`, and the DHCP options set for your VPC must include `AmazonProvidedDNS` in its domain name servers list. For more information, see link:vpc/latest/userguide/vpc-dns.html#vpc-dns-updating[Updating DNS support for your VPC,type="documentation"] in the _Amazon VPC User Guide_. You can define your API server endpoint access requirements when you create a new cluster, and you can update the API server endpoint access for a cluster at any time. -[[modify-endpoint-access,modify-endpoint-access.title]] +[#modify-endpoint-access] == Modifying cluster endpoint access Use the procedures in this section to modify the endpoint access for an existing cluster. The following table shows the supported API server endpoint access combinations and their associated behavior. -[cols="1,1,1", options="header"] +[%header,cols="1,1,3"] |=== |Endpoint public access |Endpoint private access @@ -103,17 +99,17 @@ Use the procedures in this section to modify the endpoint access for an existing a| * This is the default behavior for new Amazon EKS clusters. -* [.noloc]`Kubernetes` API requests that originate from within your cluster's VPC (such as node to control plane communication) leave the VPC but not Amazon's network. -* Your cluster API server is accessible from the internet. You can, optionally, limit the CIDR blocks that can access the public endpoint. If you limit access to specific CIDR blocks, then it is recommended that you also enable the private endpoint, or ensure that the CIDR blocks that you specify include the addresses that nodes and Fargate [.noloc]`Pods` (if you use them) access the public endpoint from. +* Kubernetes API requests that originate from within your cluster's VPC (such as node to control plane communication) leave the VPC but not Amazon's network. +* Your cluster API server is accessible from the internet. You can, optionally, use the list of _public access CIDRs_ to control access to the public endpoint by a list of CIDR blocks. If you limit access to specific CIDR blocks, then we recommend that you also enable the private endpoint, or ensure that the CIDR blocks that you specify include the addresses that nodes and Fargate Pods (if you use them) access the public endpoint from. |Enabled |Enabled a| -* [.noloc]`Kubernetes` API requests within your cluster's VPC (such as node to control plane communication) use the private VPC endpoint. -* Your cluster API server is accessible from the internet. You can, optionally, limit the CIDR blocks that can access the public endpoint. -* If you are using hybrid nodes with your Amazon EKS cluster, it is not recommended to have both Public and Private cluster endpoint access enabled. Because your hybrid nodes are running outside of your VPC, they will resolve the cluster endpoint to the public IP addresses. It is recommended to use either Public or Private cluster endpoint access for clusters with hybrid nodes. +* Kubernetes API requests within your cluster's VPC (such as node to control plane communication) use the private VPC endpoint. You use the rules on the _cluster security group_ to control access to the private endpoint. +* Your cluster API server is accessible from the internet. You can, optionally, use the list of _public access CIDRs_ to control access to the public endpoint by a list of CIDR blocks. +* If you are using hybrid nodes with your Amazon EKS cluster, it is not recommended to have both Public and Private endpoint access enabled. Because your hybrid nodes are running outside of your VPC, they will resolve the cluster endpoint to the public IP addresses. It is recommended to use either Public or Private endpoint access for clusters with hybrid nodes. |Disabled @@ -122,6 +118,7 @@ a| * All traffic to your cluster API server must come from within your cluster's VPC or a link:whitepapers/latest/aws-vpc-connectivity-options/introduction.html[connected network,type="documentation"]. * There is no public access to your API server from the internet. Any `kubectl` commands must come from within the VPC or a connected network. For connectivity options, see <>. +* You use the rules on the _cluster security group_ to control access to the private endpoint. Note that the public access CIDRs don't affect the private endpoint. * The cluster's API server endpoint is resolved by public DNS servers to a private IP address from the VPC. In the past, the endpoint could only be resolved from within the VPC. + If your endpoint does not resolve to a private IP address within the VPC for an existing cluster, you can: @@ -131,109 +128,34 @@ If your endpoint does not resolve to a private IP address within the VPC for an |=== +*Endpoint access controls* -You can modify your cluster API server endpoint access using the {aws-management-console} or {aws} CLI. +Note that each of the following methods to control endpoint access only affect the respective endpoint. +_Cluster security group_:: +The cluster security group controls two types of connections: connections to the _kubelet API_ and the private endpoint. The connections to the `kubelet` API are used in the `kubectl attach`, `kubectl cp`, `kubectl exec`, `kubectl logs`, and `kubectl port-forward` commands. The cluster security group doesn't affect the public endpoint. -== Configure endpoint access - {aws} console +_Public access CIDRs_:: +The _public access CIDRs_ control access to the public endpoint by a list of CIDR blocks. Note that the public access CIDRs don't affect the private endpoint. The public access CIDRs behave differently on the `IPv6` clusters and `IPv4` clusters depending on the date they were created, which the following describes: -. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. -. Choose the name of the cluster to display your cluster information. -. Choose the *Networking* tab and choose *Update*. -. For *Private access*, choose whether to enable or disable private access for your cluster's [.noloc]`Kubernetes` API server endpoint. If you enable private access, [.noloc]`Kubernetes` API requests that originate from within your cluster's VPC use the private VPC endpoint. You must enable private access to disable public access. -. For *Public access*, choose whether to enable or disable public access for your cluster's [.noloc]`Kubernetes` API server endpoint. If you disable public access, your cluster's [.noloc]`Kubernetes` API server can only receive requests from within the cluster VPC. -. (Optional) If you've enabled *Public access*, you can specify which addresses from the internet can communicate to the public endpoint. Select *Advanced Settings*. Enter a CIDR block, such as [.replaceable]`203.0.113.5/32`. The block cannot include https://en.wikipedia.org/wiki/Reserved_IP_addresses[reserved addresses]. You can enter additional blocks by selecting *Add Source*. There is a maximum number of CIDR blocks that you can specify. For more information, see <>. If you specify no blocks, then the public API server endpoint receives requests from all (`0.0.0.0/0`) IP addresses. If you restrict access to your public endpoint using CIDR blocks, it is recommended that you also enable private endpoint access so that nodes and Fargate [.noloc]`Pods` (if you use them) can communicate with the cluster. Without the private endpoint enabled, your public access endpoint CIDR sources must include the egress sources from your VPC. For example, if you have a node in a private subnet that communicates to the internet through a NAT Gateway, you will need to add the outbound IP address of the NAT gateway as part of an allowed CIDR block on your public endpoint. -. Choose *Update* to finish. +*CIDR blocks in the public endpoint (`IPv6` cluster)* +You can add `IPv6` and `IPv4` CIDR blocks to the public endpoint of an `IPv6` cluster, because the public endpoint is dual-stack. This only applies to new clusters with the `ipFamily` set to `IPv6` that you made in October 2024 or later. You can identify these clusters by the new endpoint domain name `api.aws`. -== Configure endpoint access - {aws} CLI +*CIDR blocks in the public endpoint (`IPv4` cluster)* -Complete the following steps using the {aws} CLI version `1.27.160` or later. You can check your current version with `aws --version`. To install or upgrade the {aws} CLI, see link:cli/latest/userguide/cli-chap-install.html[Installing the {aws} CLI,type="documentation"]. +You can add `IPv4` CIDR blocks to the public endpoint of an `IPv4` cluster. +You can't add `IPv6` CIDR blocks to the public endpoint of an `IPv4` cluster. If you try, EKS returns the following error message: `The following CIDRs are invalid in publicAccessCidrs` -. Update your cluster API server endpoint access with the following {aws} CLI command. Substitute your cluster name and desired endpoint access values. If you set `endpointPublicAccess=true`, then you can (optionally) enter single CIDR block, or a comma-separated list of CIDR blocks for `publicAccessCidrs`. The blocks cannot include https://en.wikipedia.org/wiki/Reserved_IP_addresses[reserved addresses]. If you specify CIDR blocks, then the public API server endpoint will only receive requests from the listed blocks. There is a maximum number of CIDR blocks that you can specify. For more information, see <>. If you restrict access to your public endpoint using CIDR blocks, it is recommended that you also enable private endpoint access so that nodes and Fargate [.noloc]`Pods` (if you use them) can communicate with the cluster. Without the private endpoint enabled, your public access endpoint CIDR sources must include the egress sources from your VPC. For example, if you have a node in a private subnet that communicates to the internet through a NAT Gateway, you will need to add the outbound IP address of the NAT gateway as part of an allowed CIDR block on your public endpoint. If you specify no CIDR blocks, then the public API server endpoint receives requests from all (0.0.0.0/0) IP addresses. -+ -NOTE: The following command enables private access and public access from a single IP address for the API server endpoint. Replace [.replaceable]`203.0.113.5/32` with a single CIDR block, or a comma-separated list of CIDR blocks that you want to restrict network access to. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks update-cluster-config \ - --region region-code \ - --name my-cluster \ - --resources-vpc-config endpointPublicAccess=true,publicAccessCidrs="203.0.113.5/32",endpointPrivateAccess=true ----- -+ -An example output is as follows. -+ -[source,json,subs="verbatim,attributes"] ----- -{ - "update": { - "id": "e6f0905f-a5d4-4a2a-8c49-EXAMPLE00000", - "status": "InProgress", - "type": "EndpointAccessUpdate", - "params": [ - { - "type": "EndpointPublicAccess", - "value": "true" - }, - { - "type": "EndpointPrivateAccess", - "value": "true" - }, - { - "type": "publicAccessCidrs", - "value": "[\203.0.113.5/32\"]" - } - ], - "createdAt": 1576874258.137, - "errors": [] - } -} ----- -. Monitor the status of your endpoint access update with the following command, using the cluster name and update ID that was returned by the previous command. Your update is complete when the status is shown as `Successful`. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks describe-update \ - --region region-code \ - --name my-cluster \ - --update-id e6f0905f-a5d4-4a2a-8c49-EXAMPLE00000 ----- -+ -An example output is as follows. -+ -[source,json,subs="verbatim,attributes"] ----- -{ - "update": { - "id": "e6f0905f-a5d4-4a2a-8c49-EXAMPLE00000", - "status": "Successful", - "type": "EndpointAccessUpdate", - "params": [ - { - "type": "EndpointPublicAccess", - "value": "true" - }, - { - "type": "EndpointPrivateAccess", - "value": "true" - }, - { - "type": "publicAccessCidrs", - "value": "[\203.0.113.5/32\"]" - } - ], - "createdAt": 1576874258.137, - "errors": [] - } -} ----- - - -[[private-access,private-access.title]] +*CIDR blocks in the public endpoint (`IPv6` cluster made before October 2024)* + +You can add `IPv4` CIDR blocks to the public endpoint of the old `IPv6` clusters that you made before October 2024. You can identify these clusters by the `eks.amazonaws.com` endpoint. +You can't add `IPv6` CIDR blocks to the public endpoint of these old `IPv6` clusters that you made before October 2024. If you try, EKS returns the following error message: `The following CIDRs are invalid in publicAccessCidrs` + +[#private-access] == Accessing a private only API server -If you have disabled public access for your cluster's [.noloc]`Kubernetes` API server endpoint, you can only access the API server from within your VPC or a link:whitepapers/latest/aws-vpc-connectivity-options/introduction.html[connected network,type="documentation"]. Here are a few possible ways to access the [.noloc]`Kubernetes` API server endpoint: +If you have disabled public access for your cluster's Kubernetes API server endpoint, you can only access the API server from within your VPC or a link:whitepapers/latest/aws-vpc-connectivity-options/introduction.html[connected network,type="documentation"]. Here are a few possible ways to access the Kubernetes API server endpoint: @@ -244,10 +166,15 @@ Connect your network to the VPC with an link:vpc/latest/tgw/what-is-transit-gate *Amazon EC2 bastion host*:: You can launch an Amazon EC2 instance into a public subnet in your cluster's VPC and then log in via SSH into that instance to run `kubectl` commands. For more information, see link:quickstart/architecture/linux-bastion/[Linux bastion hosts on {aws},type="marketing"]. You must ensure that your Amazon EKS control plane security group contains rules to allow ingress traffic on port 443 from your bastion host. For more information, see <>. + -When you configure `kubectl` for your bastion host, be sure to use {aws} credentials that are already mapped to your cluster's RBAC configuration, or add the link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] that your bastion will use to the RBAC configuration before you remove endpoint public access. For more information, see <> and <>. +When you configure `kubectl` for your bastion host, be sure to use {aws} credentials that are already mapped to your cluster's RBAC configuration, or add the link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] that your bastion will use to the RBAC configuration before you remove endpoint public access. For more information, see <> and <>. *{aws} Cloud9 IDE*:: {aws} Cloud9 is a cloud-based integrated development environment (IDE) that lets you write, run, and debug your code with just a browser. You can create an {aws} Cloud9 IDE in your cluster's VPC and use the IDE to communicate with your cluster. For more information, see link:cloud9/latest/user-guide/create-environment.html[Creating an environment in {aws} Cloud9,type="documentation"]. You must ensure that your Amazon EKS control plane security group contains rules to allow ingress traffic on port 443 from your IDE security group. For more information, see <>. + When you configure `kubectl` for your {aws} Cloud9 IDE, be sure to use {aws} credentials that are already mapped to your cluster's RBAC configuration, or add the IAM principal that your IDE will use to the RBAC configuration before you remove endpoint public access. For more information, see <> and <>. + + +📝 https://github.com/search?q=repo%3Aawsdocs%2Famazon-eks-user-guide+%5B%23cluster-endpoint%5D&type=code[Edit this page on GitHub] + +include::config-cluster-endpoint.adoc[leveloffset=+1] \ No newline at end of file diff --git a/latest/ug/clusters/cluster-insights.adoc b/latest/ug/clusters/cluster-insights.adoc index fa079624d..221d134c1 100644 --- a/latest/ug/clusters/cluster-insights.adoc +++ b/latest/ug/clusters/cluster-insights.adoc @@ -1,201 +1,54 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[cluster-insights,cluster-insights.title]] -= Prepare for [.noloc]`Kubernetes` version upgrades with cluster insights +[#cluster-insights] += Prepare for Kubernetes version upgrades and troubleshoot misconfigurations with cluster insights :info_titleabbrev: Cluster insights -:keywords: cluster, upgrade, insights - -include::../attributes.txt[] [abstract] -- -Discover how Amazon EKS cluster insights help monitor and resolve potential [.noloc]`Kubernetes` version upgrade issues for enhanced reliability and faster adoption of new capabilities. +Discover how Amazon EKS cluster insights help monitor and resolve potential issues for enhanced reliability. -- -Amazon EKS cluster insights provide recommendations to help you follow Amazon EKS and [.noloc]`Kubernetes` best practices. Every Amazon EKS cluster undergoes automatic, recurring checks against an Amazon EKS curated list of insights. These insight checks are fully managed by Amazon EKS and offer recommendations on how to address any findings. - -* Before updating your cluster [.noloc]`Kubernetes` version, check the *Cluster insights* tab of the observability dashboard in the link:eks/home#/clusters[Amazon EKS console.,type="console"] -* If your cluster has identified issues, review them and make appropriate fixes. The issues include links to Amazon EKS and [.noloc]`Kubernetes`. -* After fixing issues, wait for the cluster insights to refresh. If all issues have been resolved, <> - -Amazon EKS returns insights related to [.noloc]`Kubernetes` version upgrade readiness. Upgrade insights identify possible issues that could impact [.noloc]`Kubernetes` cluster upgrades. This minimizes the effort that administrators spend preparing for upgrades and increases the reliability of applications on newer [.noloc]`Kubernetes` versions. Clusters are automatically scanned by Amazon EKS against a list of possible [.noloc]`Kubernetes` version upgrade impacting issues. Amazon EKS frequently updates the list of insight checks based on reviews of changes made in each [.noloc]`Kubernetes` version release. - -Amazon EKS upgrade insights speed up the testing and verification process for new versions. They also allow cluster administrators and application developers to leverage the newest [.noloc]`Kubernetes` capabilities by highlighting concerns and offering remediation advice. To see the list of insight checks performed and any relevant issues that Amazon EKS has identified, you can call the Amazon EKS `ListInsights` API operation or look in the Amazon EKS console. - -Cluster insights update periodically. You cannot manually refresh cluster insights. If you fix a cluster issue, it will take some time for cluster insights to update. To determine if a fix was successful, compare the time the change deployed to the "last refresh time" of the cluster insight. - -[[cluster-insights-console,cluster-insights-console.title]] -== View cluster insights (Console) -. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. -. From the cluster list, choose the name of the Amazon EKS cluster for which you want to see the insights. -. Choose *View dashboard*. -. Choose the *Cluster Insights* tab. -. In the *Upgrade Insights* table, you will see the following columns: -+ -** *Name* – The check that was performed by Amazon EKS against the cluster. -** *Insight status* – An insight with a status of "Error" typically means the impacted [.noloc]`Kubernetes` version is N+1 of the current cluster version, while a status of "Warning" means the insight applies to a future [.noloc]`Kubernetes` version N+2 or more. An insight with status of "Passing" means Amazon EKS has not found any issues associated with this insight check in your cluster. An insight status of "Unknown" means Amazon EKS is unable to determine if your cluster is impacted by this insight check. -** *Version* – The [.noloc]`Kubernetes` version that the insight checked for possible issues. -** *Last refresh time* – The time the status of the insight was last refreshed for this cluster. -** *Last transition time* – The time the status of this insight last changed. -** *Description* – Information from the insight check, which includes the alert and recommended actions for remediation. - - -[[cluster-insights-cli,cluster-insights-cli.title]] -== View cluster insights ({aws} CLI) -. Determine which cluster you would like to check for insights. The following command lists the insights for a specified cluster. Make the following modifications to the command as needed and then run the modified command: -+ -** Replace [.replaceable]`region-code` with the code for your {aws} Region. -** Replace [.replaceable]`my-cluster` with the name of your cluster. -+ -[source,bash,subs="verbatim,attributes,quotes"] ----- -aws eks list-insights --region [.replaceable]`region-code` --cluster-name [.replaceable]`my-cluster` ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -{ -"insights": - [ - { - "id": "a1b2c3d4-5678-90ab-cdef-EXAMPLE11111", - "name": "Deprecated APIs removed in Kubernetes vX.XX", - "category": "UPGRADE_READINESS", - "kubernetesVersion": "X.XX", - "lastRefreshTime": 1734557315.000, - "lastTransitionTime": 1734557309.000, - "description": "Checks for usage of deprecated APIs that are scheduled for removal in Kubernetes vX.XX. Upgrading your cluster before migrating to the updated APIs supported by vX.XX could cause application impact.", - "insightStatus": - { - "status": "PASSING", - "reason": "No deprecated API usage detected within the last 30 days.", - }, - }, - { - "id": "a1b2c3d4-5678-90ab-cdef-EXAMPLE22222", - "name": "Kubelet version skew", - "category": "UPGRADE_READINESS", - "kubernetesVersion": "X.XX", - "lastRefreshTime": 1734557309.000, - "lastTransitionTime": 1734557309.000, - "description": "Checks for kubelet versions of worker nodes in the cluster to see if upgrade would cause non compliance with supported Kubernetes kubelet version skew policy.", - "insightStatus": - { - "status": "UNKNOWN", - "reason": "Unable to determine status of node kubelet versions.", - }, - }, - { - "id": "a1b2c3d4-5678-90ab-cdef-EXAMPLE33333", - "name": "Deprecated APIs removed in Kubernetes vX.XX", - "category": "UPGRADE_READINESS", - "kubernetesVersion": "X.XX", - "lastRefreshTime": 1734557315.000, - "lastTransitionTime": 1734557309.000, - "description": "Checks for usage of deprecated APIs that are scheduled for removal in Kubernetes vX.XX. Upgrading your cluster before migrating to the updated APIs supported by vX.XX could cause application impact.", - "insightStatus": - { - "status": "PASSING", - "reason": "No deprecated API usage detected within the last 30 days.", - }, - }, - { - "id": "a1b2c3d4-5678-90ab-cdef-EXAMPLEaaaaa", - "name": "Cluster health issues", - "category": "UPGRADE_READINESS", - "kubernetesVersion": "X.XX", - "lastRefreshTime": 1734557314.000, - "lastTransitionTime": 1734557309.000, - "description": "Checks for any cluster health issues that prevent successful upgrade to the next Kubernetes version on EKS.", - "insightStatus": - { - "status": "PASSING", - "reason": "No cluster health issues detected.", - }, - }, - { - "id": "a1b2c3d4-5678-90ab-cdef-EXAMPLEbbbbb", - "name": "EKS add-on version compatibility", - "category": "UPGRADE_READINESS", - "kubernetesVersion": "X.XX", - "lastRefreshTime": 1734557314.000, - "lastTransitionTime": 1734557309.000, - "description": "Checks version of installed EKS add-ons to ensure they are compatible with the next version of Kubernetes. ", - "insightStatus": { "status": "PASSING", "reason": "All installed EKS add-on versions are compatible with next Kubernetes version."}, - }, - { - "id": "a1b2c3d4-5678-90ab-cdef-EXAMPLEccccc", - "name": "kube-proxy version skew", - "category": "UPGRADE_READINESS", - "kubernetesVersion": "X.XX", - "lastRefreshTime": 1734557314.000, - "lastTransitionTime": 1734557309.000, - "description": "Checks version of kube-proxy in cluster to see if upgrade would cause non compliance with supported Kubernetes kube-proxy version skew policy.", - "insightStatus": - { - "status": "PASSING", - "reason": "kube-proxy versions match the cluster control plane version.", - }, - }, - { - "id": "a1b2c3d4-5678-90ab-cdef-EXAMPLEddddd", - "name": "Deprecated APIs removed in Kubernetes vX.XX", - "category": "UPGRADE_READINESS", - "kubernetesVersion": "X.XX", - "lastRefreshTime": 1734557315.000, - "lastTransitionTime": 1734557309.000, - "description": "Checks for usage of deprecated APIs that are scheduled for removal in Kubernetes vX.XX. Upgrading your cluster before migrating to the updated APIs supported by vX.XX could cause application impact.", - "insightStatus": - { - "status": "PASSING", - "reason": "No deprecated API usage detected within the last 30 days.", - }, - }, - ], -"nextToken": null, -} ----- -. For descriptive information about the insight, run the following command. Make the following modifications to the command as needed and then run the modified command: -+ -** Replace [.replaceable]`region-code` with the code for your {aws} Region. -** Replace [.replaceable]`a1b2c3d4-5678-90ab-cdef-EXAMPLE22222` with an insight ID retrieved from listing the cluster insights. -** Replace [.replaceable]`my-cluster` with the name of your cluster. -+ -[source,bash,subs="verbatim,attributes,quotes"] ----- -aws eks describe-insight --region region-code --id [.replaceable]`a1b2c3d4-5678-90ab-cdef-EXAMPLE22222` --cluster-name my-cluster ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -{ - "insight": - { - "id": "a1b2c3d4-5678-90ab-cdef-EXAMPLE22222", - "name": "Kubelet version skew", - "category": "UPGRADE_READINESS", - "kubernetesVersion": "1.27", - "lastRefreshTime": 1734557309.000, - "lastTransitionTime": 1734557309.000, - "description": "Checks for kubelet versions of worker nodes in the cluster to see if upgrade would cause non compliance with supported Kubernetes kubelet version skew policy.", - "insightStatus": - { - "status": "UNKNOWN", - "reason": "Unable to determine status of node kubelet versions.", - }, - "recommendation": "Upgrade your worker nodes to match the Kubernetes version of your cluster control plane.", - "additionalInfo": - { - "Kubelet version skew policy": "https://kubernetes.io/releases/version-skew-policy/#kubelet", - "Updating a managed node group": "https://docs.aws.amazon.com/eks/latest/userguide/update-managed-node-group.html", - }, - "resources": [], - "categorySpecificSummary": - { "deprecationDetails": [], "addonCompatibilityDetails": [] }, - }, -} ----- +Amazon EKS cluster insights provide detection of issues and recommendations to resolve them to help you manage your cluster. Every Amazon EKS cluster undergoes automatic, recurring checks against an Amazon EKS curated list of insights. These __insight checks__ are fully managed by Amazon EKS and offer recommendations on how to address any findings. + +[#cluster-insight-types] +== Cluster insight types +* *Configuration insights*: Identifies misconfigurations in your EKS Hybrid Nodes setup that could impair functionality of your cluster or workloads. +* *Upgrade insights*: Identifies issues that could impact your ability to upgrade to new versions of Kubernetes. + +[#cluster-insight-considerations] +== Considerations +* *Frequency*: Amazon EKS refreshes cluster insights every 24 hours, or you can manually refresh them to see the latest status. For example, you can manually refresh cluster insights after addressing an issue to see if the issue was resolved. +* *Permissions*: Amazon EKS automatically creates a cluster access entry for cluster insights in every EKS cluster. This entry gives EKS permission to view information about your cluster. Amazon EKS uses this information to generate the insights. For more information, see <>. + +[#cluster-insights-use-cases] +== Use cases +Cluster insights in Amazon EKS provide automated checks to help maintain the health, reliability, and optimal configuration of your Kubernetes clusters. Below are key use cases for cluster insights, including upgrade readiness and configuration troubleshooting. + +=== Upgrade insights +Upgrade insights are a specific type of insight checks within cluster insights. These checks returns insights related to Kubernetes version upgrade readiness. Amazon EKS runs upgrade insight checks on every EKS cluster. + +[IMPORTANT] +==== +Amazon EKS has temporarily rolled back a feature that would +require you to use a `--force` flag to upgrade your cluster when there were certain cluster insight issues. For more information, see link:https://github.com/aws/containers-roadmap/issues/2570[Temporary rollback of enforcing upgrade insights on update cluster version] on GitHub. + +For more information about updating your cluster, see <>. +==== + +Before updating your cluster Kubernetes version, you can use the *Upgrade insights* tab of the observability dashboard in the link:eks/home#/clusters[Amazon EKS console,type="console"]. If your cluster has identified issues, review them and make appropriate fixes. The issues include links to Amazon EKS and Kubernetes documentation. After fixing the issue, refresh cluster insights on-demand to fetch the latest insights. If all issues have been resolved, <>. + +Amazon EKS returns insights related to Kubernetes version upgrade readiness. Upgrade insights identify possible issues that could impact Kubernetes cluster upgrades. This minimizes the effort that administrators spend preparing for upgrades and increases the reliability of applications on newer Kubernetes versions. Clusters are automatically scanned by Amazon EKS against a list of possible Kubernetes version upgrade impacting issues. Amazon EKS frequently updates the list of insight checks based on reviews of changes made in each Kubernetes version release. + +Amazon EKS upgrade insights speed up the testing and verification process for new versions. They also allow cluster administrators and application developers to leverage the newest Kubernetes capabilities by highlighting concerns and offering remediation advice. + +=== Configuration insights + +EKS cluster insights automatically scans Amazon EKS clusters with hybrid nodes to identify configuration issues impairing Kubernetes control plane-to-webhook communication, kubectl commands like exec and logs, and more. Configuration insights surface issues and provide remediation recommendations, accelerating the time to a fully functioning hybrid nodes setup. + +== Get started + +To see the list of insight checks performed and any relevant issues that Amazon EKS has identified, you can use the {aws-management-console}, the {aws} CLI, {aws} SDKs, and Amazon EKS `ListInsights` API operation. To get started, see <>. + +include::view-cluster-insights.adoc[leveloffset=+1] diff --git a/latest/ug/clusters/clusters.adoc b/latest/ug/clusters/clusters.adoc index 0235a5660..4f2d06ae3 100644 --- a/latest/ug/clusters/clusters.adoc +++ b/latest/ug/clusters/clusters.adoc @@ -1,55 +1,11 @@ -//!!NODE_ROOT include::../attributes.txt[] -[[clusters,clusters.title]] -= Organize workloads with Amazon EKS clusters -:doctype: book -:sectnums: -:toc: left -:icons: font -:experimental: -:idprefix: -:idseparator: - -:sourcedir: . -:info_doctype: chapter -:info_title: Organize workloads with Amazon EKS clusters -:info_titleabbrev: Clusters -An Amazon EKS cluster consists of two primary components: +[#clusters] += Amazon EKS cluster lifecycle and configuration +:info_titleabbrev: Configure clusters -* The Amazon EKS control plane -* Amazon EKS nodes that are registered with the control plane +This section provides in-depth guidance on the full lifecycle management of Kubernetes clusters and different ways to configure them. It covers cluster creation, monitoring cluster health, updating Kubernetes versions, and deleting clusters. It also includes advanced topics on how to configure endpoint access, enable/disable Windows support, set up private clusters, implement autoscaling, and how to enhance resilience with zonal shift for traffic redirection in multi-AZ setups. -The Amazon EKS control plane consists of control plane nodes that run the [.noloc]`Kubernetes` software, such as `etcd` and the [.noloc]`Kubernetes` API server. The control plane runs in an account managed by {aws}, and the [.noloc]`Kubernetes` API is exposed via the Amazon EKS endpoint associated with your cluster. Each Amazon EKS cluster control plane is single-tenant and unique, and runs on its own set of Amazon EC2 instances. - -All of the data stored by the `etcd` nodes and associated Amazon EBS volumes is encrypted using {aws} KMS. The cluster control plane is provisioned across multiple Availability Zones and fronted by an Elastic Load Balancing Network Load Balancer. Amazon EKS also provisions elastic network interfaces in your VPC subnets to provide connectivity from the control plane instances to the nodes (for example, to support `kubectl exec` `logs` `proxy` data flows). - -[IMPORTANT] -==== - -In the Amazon EKS environment, `etcd` storage is limited to 8 GiB as per https://etcd.io/docs/v3.5/dev-guide/limit/#storage-size-limit[upstream] guidance. You can monitor a metric for the current database size by running the following command. If your cluster has a [.noloc]`Kubernetes` version below `1.28`, replace [.replaceable]`apiserver_storage_size_bytes` with the following: - - - -* [.noloc]`Kubernetes` version `1.27` and `1.26` – `apiserver_storage_db_total_size_in_bytes` -* [.noloc]`Kubernetes` version `1.25` and below – `etcd_db_total_size_in_bytes` - -[source,bash,subs="verbatim,attributes"] ----- -kubectl get --raw=/metrics | grep "apiserver_storage_size_bytes" ----- - -==== - -Amazon EKS nodes run in your {aws} account and connect to your cluster's control plane via the API server endpoint and a certificate file that is created for your cluster. - -[NOTE] -==== - - -* You can find out how the different components of Amazon EKS work in <>. -* For connected clusters, see <>. - -==== [.topiclist] [[Topic List]] @@ -58,34 +14,20 @@ include::create-cluster-auto.adoc[leveloffset=+1] include::create-cluster.adoc[leveloffset=+1] - include::cluster-insights.adoc[leveloffset=+1] - include::update-cluster.adoc[leveloffset=+1] - include::delete-cluster.adoc[leveloffset=+1] - include::cluster-endpoint.adoc[leveloffset=+1] - include::windows-support.adoc[leveloffset=+1] - include::disable-windows-support.adoc[leveloffset=+1] - include::private-clusters.adoc[leveloffset=+1] - -include::kubernetes-versions.adoc[leveloffset=+1] - - -include::platform-versions.adoc[leveloffset=+1] - - include::autoscaling.adoc[leveloffset=+1] include::zone-shift.adoc[leveloffset=+1] diff --git a/latest/ug/clusters/config-cluster-endpoint.adoc b/latest/ug/clusters/config-cluster-endpoint.adoc new file mode 100644 index 000000000..1925d8611 --- /dev/null +++ b/latest/ug/clusters/config-cluster-endpoint.adoc @@ -0,0 +1,112 @@ +[.topic] +[#config-cluster-endpoint] += Configure network access to cluster API server endpoint +:info_titleabbrev: Configure endpoint access + +include::../attributes.txt[] + +[abstract] +-- +Learn how to enable private access and limit public access to the Amazon EKS cluster Kubernetes API server endpoint for enhanced security with your Amazon EKS cluster. +-- + +You can modify your cluster API server endpoint access using the {aws-management-console} or {aws} CLI in the following sections. + + + +== Configure endpoint access - {aws} console + +. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. +. Choose the name of the cluster to display your cluster information. +. Choose the *Networking* tab and choose *Manage endpoint access*. +. For *Private access*, choose whether to enable or disable private access for your cluster's [.noloc]`Kubernetes` API server endpoint. If you enable private access, [.noloc]`Kubernetes` API requests that originate from within your cluster's VPC use the private VPC endpoint. You must enable private access to disable public access. +. For *Public access*, choose whether to enable or disable public access for your cluster's [.noloc]`Kubernetes` API server endpoint. If you disable public access, your cluster's [.noloc]`Kubernetes` API server can only receive requests from within the cluster VPC. +. (Optional) If you've enabled *Public access*, you can specify which addresses from the internet can communicate to the public endpoint. Select *Advanced Settings*. Enter a CIDR block, such as [.replaceable]`203.0.113.5/32`. The block cannot include https://en.wikipedia.org/wiki/Reserved_IP_addresses[reserved addresses]. You can enter additional blocks by selecting *Add Source*. There is a maximum number of CIDR blocks that you can specify. For more information, see <>. If you specify no blocks, then the public API server endpoint receives requests from all IP addresses for both `IPv4` (`0.0.0.0/0`) and additionally `IPv6` (`::/0`) for dual-stack `IPv6` cluster. If you restrict access to your public endpoint using CIDR blocks, we recommend that you also enable private endpoint access so that nodes and Fargate [.noloc]`Pods` (if you use them) can communicate with the cluster. Without the private endpoint enabled, your public access endpoint CIDR sources must include the egress sources from your VPC. For example, if you have a node in a private subnet that communicates to the internet through a NAT Gateway, you will need to add the outbound IP address of the NAT gateway as part of an allowed CIDR block on your public endpoint. +. Choose *Update* to finish. + + +== Configure endpoint access - {aws} CLI + +Complete the following steps using the {aws} CLI version `1.27.160` or later. You can check your current version with `aws --version`. To install or upgrade the {aws} CLI, see link:cli/latest/userguide/cli-chap-install.html[Installing the {aws} CLI,type="documentation"]. + +. Update your cluster API server endpoint access with the following {aws} CLI command. Substitute your cluster name and desired endpoint access values. If you set `endpointPublicAccess=true`, then you can (optionally) enter single CIDR block, or a comma-separated list of CIDR blocks for `publicAccessCidrs`. The blocks cannot include https://en.wikipedia.org/wiki/Reserved_IP_addresses[reserved addresses]. If you specify CIDR blocks, then the public API server endpoint will only receive requests from the listed blocks. There is a maximum number of CIDR blocks that you can specify. For more information, see <>. If you restrict access to your public endpoint using CIDR blocks, it is recommended that you also enable private endpoint access so that nodes and Fargate [.noloc]`Pods` (if you use them) can communicate with the cluster. Without the private endpoint enabled, your public access endpoint CIDR sources must include the egress sources from your VPC. For example, if you have a node in a private subnet that communicates to the internet through a NAT Gateway, you will need to add the outbound IP address of the NAT gateway as part of an allowed CIDR block on your public endpoint. If you specify no CIDR blocks, then the public API server endpoint receives requests from all (0.0.0.0/0) IP addresses and additionally `IPv6` (`::/0`) for dual-stack `IPv6` cluster. ++ +NOTE: The following command enables private access and public access from a single IP address for the API server endpoint. Replace [.replaceable]`203.0.113.5/32` with a single CIDR block, or a comma-separated list of CIDR blocks that you want to restrict network access to. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks update-cluster-config \ + --region region-code \ + --name my-cluster \ + --resources-vpc-config endpointPublicAccess=true,publicAccessCidrs="203.0.113.5/32",endpointPrivateAccess=true +---- ++ +An example output is as follows. ++ +[source,json,subs="verbatim,attributes"] +---- +{ + "update": { + "id": "e6f0905f-a5d4-4a2a-8c49-EXAMPLE00000", + "status": "InProgress", + "type": "EndpointAccessUpdate", + "params": [ + { + "type": "EndpointPublicAccess", + "value": "true" + }, + { + "type": "EndpointPrivateAccess", + "value": "true" + }, + { + "type": "publicAccessCidrs", + "value": "[\"203.0.113.5/32\"]" + } + ], + "createdAt": 1576874258.137, + "errors": [] + } +} +---- +. Monitor the status of your endpoint access update with the following command, using the cluster name and update ID that was returned by the previous command. Your update is complete when the status is shown as `Successful`. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks describe-update \ + --region region-code \ + --name my-cluster \ + --update-id e6f0905f-a5d4-4a2a-8c49-EXAMPLE00000 +---- ++ +An example output is as follows. ++ +[source,json,subs="verbatim,attributes"] +---- +{ + "update": { + "id": "e6f0905f-a5d4-4a2a-8c49-EXAMPLE00000", + "status": "Successful", + "type": "EndpointAccessUpdate", + "params": [ + { + "type": "EndpointPublicAccess", + "value": "true" + }, + { + "type": "EndpointPrivateAccess", + "value": "true" + }, + { + "type": "publicAccessCidrs", + "value": "[\"203.0.113.5/32\"]" + } + ], + "createdAt": 1576874258.137, + "errors": [] + } +} +---- + + +📝 https://github.com/search?q=repo%3Aawsdocs%2Famazon-eks-user-guide+%5B%23config-cluster-endpoint%5D&type=code[Edit this page on GitHub] \ No newline at end of file diff --git a/latest/ug/clusters/create-cluster-auto.adoc b/latest/ug/clusters/create-cluster-auto.adoc index 32a496cf3..385a91411 100644 --- a/latest/ug/clusters/create-cluster-auto.adoc +++ b/latest/ug/clusters/create-cluster-auto.adoc @@ -1,23 +1,16 @@ -//!!NODE_ROOT
+include::../attributes.txt[] [.topic] -[[create-cluster-auto,create-cluster-auto.title]] +[#create-cluster-auto] = Create an Amazon EKS Auto Mode cluster -:info_doctype: section -:info_title: Create an Amazon EKS Auto Mode cluster :info_titleabbrev: Create auto cluster -:info_abstract: Learn how to create an Amazon EKS Auto Mode cluster to run Kubernetes applications, including prerequisites, networking options, and add-on configurations. -:idprefix: id_ - -include::../attributes.txt[] - [abstract] -- Learn how to create an Amazon EKS Auto Mode cluster to run Kubernetes applications, including prerequisites, networking options, and add-on configurations. -- -This topic provides detailed instructions for creating an Amazon EKS Auto Mode cluster using advanced configuration options. It covers prerequisites, networking options, and add-on configurations. The process includes setting up IAM roles, configuring cluster settings, specifying networking parameters, and selecting add-ons. Users can create clusters using either the {aws} Management Console or the {aws} CLI, with step-by-step guidance provided for both methods. +This topic provides detailed instructions for creating an Amazon EKS Auto Mode cluster using advanced configuration options. It covers prerequisites, networking options, and add-on configurations. The process includes setting up IAM roles, configuring cluster settings, specifying networking parameters, and selecting add-ons. Users can create clusters using either the {aws-management-console} or the {aws} CLI, with step-by-step guidance provided for both methods. For users seeking a less complex setup process, refer to the following for simplified cluster creation steps: @@ -43,15 +36,15 @@ This topic covers advanced configuration. If you are looking to get started with == Prerequisites * An existing VPC and subnets that meet <>. Before you deploy a cluster for production use, we recommend that you have a thorough understanding of the VPC and subnet requirements. If you don't have a VPC and subnets, you can create them using an <>. -* The `kubectl` command line tool is installed on your device or {aws} CloudShell. The version can be the same as or up to one minor version earlier or later than the [.noloc]`Kubernetes` version of your cluster. For example, if your cluster version is `1.29`, you can use `kubectl` version `1.28`, `1.29`, or `1.30` with it. To install or upgrade `kubectl`, see <>. -* Version `2.12.3` or later or version `1.27.160` or later of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device or {aws} CloudShell. To check your current version, use `aws --version`. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the _{aws} Command Line Interface User Guide_. -* An link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] with permissions to create and modify EKS and IAM resources. +* The `kubectl` command line tool is installed on your device or {aws} CloudShell. The version can be the same as or up to one minor version earlier or later than the Kubernetes version of your cluster. For example, if your cluster version is `1.29`, you can use `kubectl` version `1.28`, `1.29`, or `1.30` with it. To install or upgrade `kubectl`, see <>. +* Version `2.12.3` or later or version `1.27.160` or later of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device or {aws} CloudShell. To check your current version, use `aws --version`. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the _{aws} Command Line Interface User Guide_. +* An link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] with permissions to create and modify EKS and IAM resources. == Create cluster - {aws} console . Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. -. Choose *Add cluster* and then choose *Create*. +. Choose *Add cluster* and then choose *Create*. . Under _Configuration options_, select *Custom configuration*. ** This topic covers custom configuration. For information about Quick configuration, see <>. . Confirm *Use EKS Auto Mode* is enabled. @@ -59,9 +52,9 @@ This topic covers advanced configuration. If you are looking to get started with . On the *Configure cluster* page, enter the following fields: + ** *Name* – A name for your cluster. The name can contain only alphanumeric characters (case-sensitive), hyphens, and underscores. It must start with an alphanumeric character and can't be longer than 100 characters. The name must be unique within the {aws} Region and {aws} account that you're creating the cluster in. -** *Cluster IAM role* – Choose the Amazon EKS cluster IAM role that you created to allow the [.noloc]`Kubernetes` control plane to manage {aws} resources on your behalf. If you haven't previously created a Cluster IAM role for EKS Auto Mode, select the *Create recommended role* button to create the role with the required permissions in the IAM console. -** *[.noloc]`Kubernetes` version* – The version of [.noloc]`Kubernetes` to use for your cluster. We recommend selecting the latest version, unless you need an earlier version. -** *Upgrade policy* -- The [.noloc]`Kubernetes` version policy you would like to set for your cluster. If you want your cluster to only run on a standard support version, you can choose *Standard*. If you want your cluster to enter extended support at the end of standard support for a version, you can choose *Extended*. If you select a [.noloc]`Kubernetes` version that is currently in extended support, you can not select standard support as an option. +** *Cluster IAM role* – Choose the Amazon EKS cluster IAM role that you created to allow the Kubernetes control plane to manage {aws} resources on your behalf. If you haven't previously created a Cluster IAM role for EKS Auto Mode, select the *Create recommended role* button to create the role with the required permissions in the IAM console. +** *Kubernetes version* – The version of Kubernetes to use for your cluster. We recommend selecting the latest version, unless you need an earlier version. +** *Upgrade policy* -- The Kubernetes version policy you would like to set for your cluster. If you want your cluster to only run on a standard support version, you can choose *Standard*. If you want your cluster to enter extended support at the end of standard support for a version, you can choose *Extended*. If you select a Kubernetes version that is currently in extended support, you can not select standard support as an option. . In the *Auto Mode Compute* section of the configure cluster page, enter the following fields: ** *Node pools* -- Determine if you want to use the build in node pools. For more information, see <>. ** *Node IAM role* -- If you enable any of the built-in node pools, you need to select a Node IAM Role. EKS Auto Mode will assign this role to new nodes. You cannot change this value after the cluster is created. If you haven't previously created a Node IAM role for EKS Auto Mode, select the Create recommended role button to create the role with the required permissions. For more information about this role, see <>. @@ -69,14 +62,14 @@ This topic covers advanced configuration. If you are looking to get started with ** *Bootstrap cluster administrator access* -- The cluster creator is automatically a Kubernetes administrator. If you want to disable this, select *Disallow cluster administrator access*. ** *Cluster authentication mode* -- EKS Auto Mode requires EKS access entries, the EKS API authentication mode. You can optionally enable the `ConfigMap` authentication mode by selecting *EKS API and ConfigMap*. . Enter the remaining fields on the configure cluster page: -** *Secrets encryption* – (Optional) Choose to enable secrets encryption of [.noloc]`Kubernetes` secrets using a KMS key. You can also enable this after you create your cluster. Before you enable this capability, make sure that you're familiar with the information in xref:enable-kms[Encrypt Kubernetes secrets with {aws} KMS on existing clusters,linkend=enable-kms]. +** *Secrets encryption* – (Optional) Choose to enable secrets encryption of Kubernetes secrets using a KMS key. You can also enable this after you create your cluster. Before you enable this capability, make sure that you're familiar with the information in <>. ** *ARC Zonal shift* -- EKS Auto Mode does not support Arc Zonal shift. ** *Tags* – (Optional) Add any tags to your cluster. For more information, see <>. + When you're done with this page, choose *Next*. . On the *Specify networking* page, select values for the following fields: + -** *VPC* – Choose an existing VPC that meets xref:network-requirements-vpc[Amazon EKS VPC requirements,linkend=network-requirements-vpc] to create your cluster in. Before choosing a VPC, we recommend that you're familiar with all of the requirements and considerations in xref:network-reqs[View Amazon EKS networking requirements for VPC and subnets,linkend=network-reqs]. You can't change which VPC you want to use after cluster creation. If no VPCs are listed, then you need to create one first. For more information, see <>. +** *VPC* – Choose an existing VPC that meets <> to create your cluster in. Before choosing a VPC, we recommend that you're familiar with all of the requirements and considerations in <>. You can't change which VPC you want to use after cluster creation. If no VPCs are listed, then you need to create one first. For more information, see <>. ** *Subnets* – By default, all available subnets in the VPC specified in the previous field are preselected. You must select at least two. + The subnets that you choose must meet the <>. Before selecting subnets, we recommend that you're familiar with all of the <>. @@ -84,13 +77,13 @@ The subnets that you choose must meet the <>. You can modify the rules in the cluster security group that Amazon EKS creates. -** *Choose cluster IP address family* – You can choose either *IPv4* and *IPv6*. +** *Choose cluster IP address family* – You can choose either *IPv4* and *IPv6*. + -[.noloc]`Kubernetes` assigns `IPv4` addresses to [.noloc]`Pods` and services, by default. Before deciding to use the `IPv6` family, make sure that you're familiar with all of the considerations and requirements in the <>, <>, <>, and <> topics. If you choose the `IPv6` family, you can't specify an address range for [.noloc]`Kubernetes` to assign `IPv6` service addresses from like you can for the `IPv4` family. [.noloc]`Kubernetes` assigns service addresses from the unique local address range (`fc00::/7`). +Kubernetes assigns `IPv4` addresses to Pods and services, by default. Before deciding to use the `IPv6` family, make sure that you're familiar with all of the considerations and requirements in the <>, <>, <>, and <> topics. If you choose the `IPv6` family, you can't specify an address range for Kubernetes to assign `IPv6` service addresses from like you can for the `IPv4` family. Kubernetes assigns service addresses from the unique local address range (`fc00::/7`). + -** (Optional) Choose *Configure [.noloc]`Kubernetes` Service IP address range* and specify a *Service `IPv4` range*. +** (Optional) Choose *Configure Kubernetes Service IP address range* and specify a *Service `IPv4` range*. + -Specifying your own range can help prevent conflicts between [.noloc]`Kubernetes` services and other networks peered or connected to your VPC. Enter a range in CIDR notation. For example: `10.2.0.0/16`. +Specifying your own range can help prevent conflicts between Kubernetes services and other networks peered or connected to your VPC. Enter a range in CIDR notation. For example: `10.2.0.0/16`. + The CIDR block must meet the following requirements: + @@ -99,24 +92,24 @@ The CIDR block must meet the following requirements: *** Not overlap with the range of the VPC for your Amazon EKS resources. + -You can only specify this option when using the `IPv4` address family and only at cluster creation. If you don't specify this, then [.noloc]`Kubernetes` assigns service IP addresses from either the `10.100.0.0/16` or `172.20.0.0/16` CIDR blocks. +You can only specify this option when using the `IPv4` address family and only at cluster creation. If you don't specify this, then Kubernetes assigns service IP addresses from either the `10.100.0.0/16` or `172.20.0.0/16` CIDR blocks. ** For *Cluster endpoint access*, select an option. After your cluster is created, you can change this option. Before selecting a non-default option, make sure to familiarize yourself with the options and their implications. For more information, see <>. + When you're done with this page, choose *Next*. -. (Optional) On the *Configure observability* page, choose which *Metrics* and *Control plane logging* options to turn on. By default, each log type is turned off. +. (Optional) On the *Configure observability* page, choose which *Metrics* and *Control plane logging* options to turn on. By default, each log type is turned off. + -** For more information about the [.noloc]`Prometheus` metrics option, see <>. +** For more information about the Prometheus metrics option, see <>. ** For more information about the *Control plane logging* options, see <>. -** When you're done with this page, choose *Next*. -. On the *Select add-ons* page, choose the add-ons that you want to add to your cluster. You can choose as many *Amazon EKS add-ons* and *{aws} Marketplace add-ons* as you require. If the *{aws} Marketplace add-ons* that you want to install isn't listed, you can click the page numbering to view additional page results or search for available *{aws} Marketplace add-ons* by entering text in the search box. You can also filter by *category*, *vendor*, or *pricing model* and then choose the add-ons from the search results. When creating a cluster, you can view, select, and install any add-on that supports EKS Pod Identities as detailed in <>. +** When you're done with this page, choose *Next*. +. On the *Select add-ons* page, choose the add-ons that you want to add to your cluster. You can choose as many *Amazon EKS add-ons* and *{aws} Marketplace add-ons* as you require. If the *{aws} Marketplace add-ons* that you want to install isn't listed, you can click the page numbering to view additional page results or search for available *{aws} Marketplace add-ons* by entering text in the search box. You can also filter by *category*, *vendor*, or *pricing model* and then choose the add-ons from the search results. When creating a cluster, you can view, select, and install any add-on that supports EKS Pod Identities as detailed in <>. ** EKS Auto Mode automates the functionality of certain add-ons. If you plan to deploy EKS Managed Node Groups to your EKS Auto Mode Cluster, select *Additional Amazon EKS Add-ons* and review the options. You may need to install add-ons such as CoreDNS and kube-proxy. EKS will only install the add-ons in this section on self-managed nodes and node groups. -** When you're done with this page, choose *Next*. +** When you're done with this page, choose *Next*. . On the *Configure selected add-ons settings* page, select the version that you want to install. You can always update to a later version after cluster creation. + For add-ons that support EKS Pod Identities, you can use the console to automatically generate the role with the name, {aws} managed policy, and trust policy prepopulated specifically for the add-on. You can re-use existing roles or create new roles for supported add-ons. For the steps to use the console to create roles for add-ons that support EKS Pod Identities, see <<_create_add_on_console>>. If an add-on does not support EKS Pod Identity, a message displays with instructions to use the wizard to create the IAM roles for service accounts (IRSA) after the cluster is created. + You can update the configuration of each add-on after cluster creation. For more information about configuring add-ons, see <>. When you're done with this page, choose *Next*. -. On the *Review and create* page, review the information that you entered or selected on the previous pages. If you need to make changes, choose *Edit*. When you're satisfied, choose *Create*. The *Status* field shows *CREATING* while the cluster is provisioned. +. On the *Review and create* page, review the information that you entered or selected on the previous pages. If you need to make changes, choose *Edit*. When you're satisfied, choose *Create*. The *Status* field shows *CREATING* while the cluster is provisioned. + NOTE: You might receive an error that one of the Availability Zones in your request doesn't have sufficient capacity to create an Amazon EKS cluster. If this happens, the error output contains the Availability Zones that can support a new cluster. Retry creating your cluster with at least two subnets that are located in the supported Availability Zones for your account. For more information, see <>. + @@ -129,29 +122,16 @@ The following CLI instructions cover creating IAM resources and creating the clu === Create an EKS Auto Mode Cluster IAM Role -#### Step 1: Create the Trust Policy +==== Step 1: Create the Trust Policy Create a trust policy that allows the Amazon EKS service to assume the role. Save the policy as `trust-policy.json`: -``` -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Service": "eks.amazonaws.com" - }, - "Action": [ - "sts:AssumeRole", - "sts:TagSession" - ] - } - ] -} -``` +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:ec7fbb57-62f5-4aef-8c1f-f24b1fa7654d[] +---- -#### Step 2: Create the IAM Role +==== Step 2: Create the IAM Role Use the trust policy to create the Cluster IAM Role: @@ -161,7 +141,7 @@ aws iam create-role \ --assume-role-policy-document file://trust-policy.json ``` -#### Step 3: Note the Role ARN +==== Step 3: Note the Role ARN Retrieve and save the ARN of the new role for use in subsequent steps: @@ -169,74 +149,69 @@ Retrieve and save the ARN of the new role for use in subsequent steps: aws iam get-role --role-name AmazonEKSAutoClusterRole --query "Role.Arn" --output text ``` -#### Step 4: Attach Required Policies +==== Step 4: Attach Required Policies Attach the following {aws} managed policies to the Cluster IAM Role to grant the necessary permissions: -**AmazonEKSClusterPolicy**: +*AmazonEKSClusterPolicy*: -``` +[source,bash,subs="verbatim,attributes"] +---- aws iam attach-role-policy \ --role-name AmazonEKSAutoClusterRole \ - --policy-arn arn:aws:iam::aws:policy/AmazonEKSClusterPolicy -``` + --policy-arn {arn-aws}iam::aws:policy/AmazonEKSClusterPolicy +---- -**AmazonEKSComputePolicy**: +*AmazonEKSComputePolicy*: -``` +[source,bash,subs="verbatim,attributes"] +---- aws iam attach-role-policy \ --role-name AmazonEKSAutoClusterRole \ - --policy-arn arn:aws:iam::aws:policy/AmazonEKSComputePolicy -``` + --policy-arn {arn-aws}iam::aws:policy/AmazonEKSComputePolicy +---- -**AmazonEKSBlockStoragePolicy**: +*AmazonEKSBlockStoragePolicy*: -``` +[source,bash,subs="verbatim,attributes"] +---- aws iam attach-role-policy \ --role-name AmazonEKSAutoClusterRole \ - --policy-arn arn:aws:iam::aws:policy/AmazonEKSBlockStoragePolicy -``` + --policy-arn {arn-aws}iam::aws:policy/AmazonEKSBlockStoragePolicy +---- -**AmazonEKSLoadBalancingPolicy**: +*AmazonEKSLoadBalancingPolicy*: -``` +[source,bash,subs="verbatim,attributes"] +---- aws iam attach-role-policy \ --role-name AmazonEKSAutoClusterRole \ - --policy-arn arn:aws:iam::aws:policy/AmazonEKSLoadBalancingPolicy -``` + --policy-arn {arn-aws}iam::aws:policy/AmazonEKSLoadBalancingPolicy +---- -**AmazonEKSNetworkingPolicy**: +*AmazonEKSNetworkingPolicy*: -``` +[source,bash,subs="verbatim,attributes"] +---- aws iam attach-role-policy \ --role-name AmazonEKSAutoClusterRole \ - --policy-arn arn:aws:iam::aws:policy/AmazonEKSNetworkingPolicy -``` + --policy-arn {arn-aws}iam::aws:policy/AmazonEKSNetworkingPolicy +---- -### Create an EKS Auto Mode Node IAM Role +=== Create an EKS Auto Mode Node IAM Role -#### Step 1: Create the Trust Policy +==== Step 1: Create the Trust Policy Create a trust policy that allows the Amazon EKS service to assume the role. Save the policy as `node-trust-policy.json`: -``` -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Service": "ec2.amazonaws.com" - }, - "Action": "sts:AssumeRole" - } - ] -} -``` +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:f2d6ec6e-f3d9-46bb-930f-5ec41ae792ab[] +---- -#### Step 2: Create the Node IAM Role +==== Step 2: Create the Node IAM Role -Use the **node-trust-policy.json** file from the previous step to define which entities can assume the role. Run the following command to create the Node IAM Role: +Use the *node-trust-policy.json* file from the previous step to define which entities can assume the role. Run the following command to create the Node IAM Role: ``` aws iam create-role \ @@ -244,7 +219,7 @@ aws iam create-role \ --assume-role-policy-document file://node-trust-policy.json ``` -#### Step 3: Note the Role ARN +==== Step 3: Note the Role ARN After creating the role, retrieve and save the ARN of the Node IAM Role. You will need this ARN in subsequent steps. Use the following command to get the ARN: @@ -252,38 +227,41 @@ After creating the role, retrieve and save the ARN of the Node IAM Role. You wil aws iam get-role --role-name AmazonEKSAutoNodeRole --query "Role.Arn" --output text ``` -#### Step 4: Attach Required Policies +==== Step 4: Attach Required Policies Attach the following {aws} managed policies to the Node IAM Role to provide the necessary permissions: -**AmazonEKSWorkerNodeMinimalPolicy**: +*AmazonEKSWorkerNodeMinimalPolicy*: -``` +[source,bash,subs="verbatim,attributes"] +---- aws iam attach-role-policy \ --role-name AmazonEKSAutoNodeRole \ - --policy-arn arn:aws:iam::aws:policy/AmazonEKSWorkerNodeMinimalPolicy -``` + --policy-arn {arn-aws}iam::aws:policy/AmazonEKSWorkerNodeMinimalPolicy +---- -**AmazonEC2ContainerRegistryPullOnly**: +*AmazonEC2ContainerRegistryPullOnly*: -``` +[source,bash,subs="verbatim,attributes"] +---- aws iam attach-role-policy \ --role-name AmazonEKSAutoNodeRole \ - --policy-arn arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPullOnly -``` + --policy-arn {arn-aws}iam::aws:policy/AmazonEC2ContainerRegistryPullOnly +---- +[#create-cluster-auto-create-cluster] === Create cluster . Create your cluster with the command that follows. Before running the command, make the following replacements: + ** Replace [.replaceable]`region-code` with the {aws} Region that you want to create your cluster in. ** Replace [.replaceable]`my-cluster` with a name for your cluster. The name can contain only alphanumeric characters (case-sensitive), hyphens, and underscores. It must start with an alphanumeric character and can't be longer than 100 characters. The name must be unique within the {aws} Region and {aws} account that you're creating the cluster in. -** Replace [.replaceable]`1.30` with any xref:kubernetes-versions[Amazon EKS supported version,linkend=kubernetes-versions]. +** Replace [.replaceable]`1.30` with any link:eks/latest/userguide/kubernetes-versions.html[Amazon EKS supported version,type="documentation"]. ** Replace [.replaceable]`111122223333` with your account ID ** If you have created differently named IAM Roles for the Cluster and Node roles, replace the ARNs. ** Replace the values for `subnetIds` with your own. You can also add additional IDs. You must specify at least two subnet IDs. + -The subnets that you choose must meet the xref:network-requirements-subnets[Amazon EKS subnet requirements,linkend=network-requirements-subnets]. Before selecting subnets, we recommend that you're familiar with all of the xref:network-reqs[Amazon EKS VPC and subnet requirements and considerations,linkend=network-reqs]. +The subnets that you choose must meet the <>. Before selecting subnets, we recommend that you're familiar with all of the <>. ** If you don't want to specify a security group ID, remove `,securityGroupIds=sg-` from the command. If you want to specify one or more security group IDs, replace the values for `securityGroupIds` with your own. You can also add additional IDs. + Whether you choose any security groups or not, Amazon EKS creates a security group that enables communication between your cluster and your VPC. Amazon EKS associates this security group, and any that you choose, to the network interfaces that it creates. For more information about the cluster security group that Amazon EKS creates, see <>. You can modify the rules in the cluster security group that Amazon EKS creates. @@ -294,9 +272,9 @@ aws eks create-cluster \ --region region-code \ --name my-cluster \ --kubernetes-version 1.30 \ - --role-arn arn:aws:iam::111122223333:role/AmazonEKSAutoClusterRole \ + --role-arn {arn-aws}iam::111122223333:role/AmazonEKSAutoClusterRole \ --resources-vpc-config '{"subnetIds": ["subnet-ExampleID1","subnet-ExampleID2"], "securityGroupIds": ["sg-ExampleID1"], "endpointPublicAccess": true, "endpointPrivateAccess": true}' \ - --compute-config '{"enabled": true, "nodeRoleArn": "arn:aws:iam::111122223333:role/AmazonEKSAutoNodeRole", "nodePools": ["general-purpose", "system"]}' \ + --compute-config '{"enabled": true, "nodeRoleArn": "{arn-aws}iam::111122223333:role/AmazonEKSAutoNodeRole", "nodePools": ["general-purpose", "system"]}' \ --kubernetes-network-config '{"elasticLoadBalancing": {"enabled": true}}' \ --storage-config '{"blockStorage": {"enabled": true}}' \ --access-config '{"authenticationMode": "API"}' @@ -305,9 +283,9 @@ aws eks create-cluster \ NOTE: You might receive an error that one of the Availability Zones in your request doesn't have sufficient capacity to create an Amazon EKS cluster. If this happens, the error output contains the Availability Zones that can support a new cluster. Retry creating your cluster with at least two subnets that are located in the supported Availability Zones for your account. For more information, see <>. + The following are optional settings that, if required, must be added to the previous command. You can only enable these options when you create the cluster, not after. -** If you want to specify which `IPv4` Classless Inter-domain Routing (CIDR) block [.noloc]`Kubernetes` assigns service IP addresses from, you must specify it by adding the `--kubernetes-network-config serviceIpv4Cidr=` to the following command. +** If you want to specify which `IPv4` Classless Inter-domain Routing (CIDR) block Kubernetes assigns service IP addresses from, you must specify it by adding the `--kubernetes-network-config serviceIpv4Cidr=` to the following command. + -Specifying your own range can help prevent conflicts between [.noloc]`Kubernetes` services and other networks peered or connected to your VPC. Enter a range in CIDR notation. For example: `10.2.0.0/16`. +Specifying your own range can help prevent conflicts between Kubernetes services and other networks peered or connected to your VPC. Enter a range in CIDR notation. For example: `10.2.0.0/16`. + The CIDR block must meet the following requirements: + @@ -315,10 +293,10 @@ The CIDR block must meet the following requirements: *** Have a minimum size of `/24` and a maximum size of `/12`. *** Not overlap with the range of the VPC for your Amazon EKS resources. + -You can only specify this option when using the `IPv4` address family and only at cluster creation. If you don't specify this, then [.noloc]`Kubernetes` assigns service IP addresses from either the `10.100.0.0/16` or `172.20.0.0/16` CIDR blocks. -** If you're creating a cluster and want the cluster to assign `IPv6` addresses to [.noloc]`Pods` and services instead of `IPv4` addresses, add `--kubernetes-network-config ipFamily=ipv6` to the following command. +You can only specify this option when using the `IPv4` address family and only at cluster creation. If you don't specify this, then Kubernetes assigns service IP addresses from either the `10.100.0.0/16` or `172.20.0.0/16` CIDR blocks. +** If you're creating a cluster and want the cluster to assign `IPv6` addresses to Pods and services instead of `IPv4` addresses, add `--kubernetes-network-config ipFamily=ipv6` to the following command. + -[.noloc]`Kubernetes` assigns `IPv4` addresses to [.noloc]`Pods` and services, by default. Before deciding to use the `IPv6` family, make sure that you're familiar with all of the considerations and requirements in the <>, <>, <>, and <> topics. If you choose the `IPv6` family, you can't specify an address range for [.noloc]`Kubernetes` to assign `IPv6` service addresses from like you can for the `IPv4` family. [.noloc]`Kubernetes` assigns service addresses from the unique local address range (`fc00::/7`). +Kubernetes assigns `IPv4` addresses to Pods and services, by default. Before deciding to use the `IPv6` family, make sure that you're familiar with all of the considerations and requirements in the <>, <>, <>, and <> topics. If you choose the `IPv6` family, you can't specify an address range for Kubernetes to assign `IPv6` service addresses from like you can for the `IPv4` family. Kubernetes assigns service addresses from the unique local address range (`fc00::/7`). + . It takes several minutes to provision the cluster. You can query the status of your cluster with the following command. + @@ -333,4 +311,4 @@ aws eks describe-cluster --region region-code --name my-cluster --query "cluster * <> * <>. * <>. -* <>. +* <>. \ No newline at end of file diff --git a/latest/ug/clusters/create-cluster.adoc b/latest/ug/clusters/create-cluster.adoc index db3350e77..8527e0b9e 100644 --- a/latest/ug/clusters/create-cluster.adoc +++ b/latest/ug/clusters/create-cluster.adoc @@ -1,15 +1,9 @@ -//!!NODE_ROOT
+include::../attributes.txt[] [.topic] -[[create-cluster,create-cluster.title]] +[#create-cluster] = Create an Amazon EKS cluster -:info_doctype: section -:info_title: Create an Amazon EKS cluster :info_titleabbrev: Create a cluster -:info_abstract: Learn how to create an Amazon EKS cluster to run Kubernetes applications, including prerequisites, networking options, and add-on configurations. - -include::../attributes.txt[] - [abstract] -- @@ -26,14 +20,14 @@ To get started with EKS Auto Mode, see <>. ==== -This topic provides an overview of the available options and describes what to consider when you create an Amazon EKS cluster. If you need to create a cluster with your on-premises infrastructure as the compute for nodes, see <>. If this is your first time creating an Amazon EKS cluster, we recommend that you follow one of our guides in <>. These guides help you to create a simple, default cluster without expanding into all of the available options. +This topic provides an overview of the available options and describes what to consider when you create an Amazon EKS cluster. If you need to create a cluster with your on-premises infrastructure as the compute for nodes, see <>. If this is your first time creating an Amazon EKS cluster, we recommend that you follow one of our guides in <>. These guides help you to create a simple, default cluster without expanding into all of the available options. == Prerequisites * An existing VPC and subnets that meet <>. Before you deploy a cluster for production use, we recommend that you have a thorough understanding of the VPC and subnet requirements. If you don't have a VPC and subnets, you can create them using an <>. -* The `kubectl` command line tool is installed on your device or {aws} CloudShell. The version can be the same as or up to one minor version earlier or later than the [.noloc]`Kubernetes` version of your cluster. For example, if your cluster version is `1.29`, you can use `kubectl` version `1.28`, `1.29`, or `1.30` with it. To install or upgrade `kubectl`, see <>. -* Version `2.12.3` or later or version `1.27.160` or later of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device or {aws} CloudShell. To check your current version, use `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or [.noloc]`Homebrew` for [.noloc]`macOS` are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the _{aws} Command Line Interface User Guide_. The {aws} CLI version that is installed in {aws} CloudShell might also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the _{aws} CloudShell User Guide_. -* An link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] with permissions to `create` and `describe` an Amazon EKS cluster. For more information, see <> and <>. +* The `kubectl` command line tool is installed on your device or {aws} CloudShell. The version can be the same as or up to one minor version earlier or later than the Kubernetes version of your cluster. To install or upgrade `kubectl`, see <>. +* Version `2.12.3` or later or version `1.27.160` or later of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device or {aws} CloudShell. To check your current version, use `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or Homebrew for macOS are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the _{aws} Command Line Interface User Guide_. The {aws} CLI version that is installed in {aws} CloudShell might also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the _{aws} CloudShell User Guide_. +* An link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] with permissions to `create` and `describe` an Amazon EKS cluster. For more information, see <> and <>. == Step 1: Create cluster IAM role @@ -43,22 +37,9 @@ This topic provides an overview of the available options and describes what to c + [source,json,subs="verbatim,attributes"] ---- -cat >eks-cluster-role-trust-policy.json <>. + -Attach the Amazon EKS managed policy named link:aws-managed-policy/latest/reference/AmazonEKSClusterPolicy.html#AmazonEKSClusterPolicy-json[AmazonEKSClusterPolicy,type="documentation"] to the role. To attach an IAM policy to an link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"], the principal that is attaching the policy must be assigned one of the following IAM actions (permissions): `iam:AttachUserPolicy` or `iam:AttachRolePolicy`. +Attach the Amazon EKS managed policy named link:aws-managed-policy/latest/reference/AmazonEKSClusterPolicy.html#AmazonEKSClusterPolicy-json[AmazonEKSClusterPolicy,type="documentation"] to the role. To attach an IAM policy to an link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"], the principal that is attaching the policy must be assigned one of the following IAM actions (permissions): `iam:AttachUserPolicy` or `iam:AttachRolePolicy`. + [source,bash,subs="verbatim,attributes"] ---- aws iam attach-role-policy --policy-arn {arn-aws}iam::aws:policy/AmazonEKSClusterPolicy --role-name myAmazonEKSClusterRole ---- +=== Service Linked Role + +Amazon EKS automatically creates a service linked role called `AWSServiceRoleForAmazonEKS`. + +This is in addition to the cluster IAM role. A service-linked role is a unique type of IAM role that is linked directly to Amazon EKS. The role allows Amazon EKS to manage clusters in your account. For more information, see <>. + +The IAM Identity you use to create the EKS cluster must have permission to create the service-linked role. This includes the `iam:CreateServiceLinkedRole` permission. + +If the service linked role doesn't already exist, and your current IAM role doesn't have sufficient permissions to create it, the cluster create operation will fail. + == Step 2: Create cluster You can create a cluster by using: -* xref:step2-eksctl[`eksctl`] -* xref:step2-console[the {aws-management-console}] -* xref:step2-cli[the {aws} CLI] +* <> +* <> +* <> -[[step2-eksctl,step2-eksctl.title]] +[#step2-eksctl] === Create cluster - eksctl . You need version `{eksctl-min-version}` or later of the `eksctl` command line tool installed on your device or {aws} CloudShell. To install or update `eksctl`, see https://eksctl.io/installation[Installation] in the `eksctl` documentation. -. Create an Amazon EKS `IPv4` cluster with the Amazon EKS default [.noloc]`Kubernetes` version in your default {aws} Region. Before running command, make the following replacements: +. Create an Amazon EKS `IPv4` cluster with the Amazon EKS default Kubernetes version in your default {aws} Region. Before running command, make the following replacements: . Replace [.replaceable]`region-code` with the {aws} Region that you want to create your cluster in. . Replace [.replaceable]`my-cluster` with a name for your cluster. The name can contain only alphanumeric characters (case-sensitive) and hyphens. It must start with an alphanumeric character and can't be longer than 100 characters. The name must be unique within the {aws} Region and {aws} account that you're creating the cluster in. -. Replace [.replaceable]`1.29` with any xref:kubernetes-versions[Amazon EKS supported version,linkend=kubernetes-versions]. +. Replace [.replaceable]`{k8s-n}` with any link:eks/latest/userguide/kubernetes-versions.html[Amazon EKS supported version,type="documentation"]. . Change the values for `vpc-private-subnets` to meet your requirements. You can also add additional IDs. You must specify at least two subnet IDs. If you'd rather specify public subnets, you can change `--vpc-private-subnets` to `--vpc-public-subnets`. Public subnets have an associated route table with a route to an internet gateway, but private subnets don't have an associated route table. We recommend using private subnets whenever possible. + The subnets that you choose must meet the <>. Before selecting subnets, we recommend that you're familiar with all of the <>. + -[source,bash,subs="verbatim,attributes"] . Run the following command: + +[source,bash,subs="verbatim,attributes"] ---- -eksctl create cluster --name my-cluster --region region-code --version 1.29 --vpc-private-subnets subnet-ExampleID1,subnet-ExampleID2 --without-nodegroup +eksctl create cluster --name my-cluster --region region-code --version {k8s-n} --vpc-private-subnets subnet-ExampleID1,subnet-ExampleID2 --without-nodegroup ---- + Cluster provisioning takes several minutes. While the cluster is being created, several lines of output appear. The last line of output is similar to the following example line. @@ -110,16 +101,16 @@ Cluster provisioning takes several minutes. While the cluster is being created, ==== Optional Settings -To see the most options that you can specify when creating a cluster with `eksctl`, use the `eksctl create cluster --help` command. To see all the available options, you can use a `config` file. For more information, see https://eksctl.io/usage/creating-and-managing-clusters/#using-config-files[Using config files] and the https://eksctl.io/usage/schema/[config file schema] in the `eksctl` documentation. You can find https://github.com/weaveworks/eksctl/tree/master/examples[config file examples] on [.noloc]`GitHub`. +To see the most options that you can specify when creating a cluster with `eksctl`, use the `eksctl create cluster --help` command. To see all the available options, you can use a `config` file. For more information, see https://eksctl.io/usage/creating-and-managing-clusters/#using-config-files[Using config files] and the https://eksctl.io/usage/schema/[config file schema] in the `eksctl` documentation. You can find https://github.com/weaveworks/eksctl/tree/master/examples[config file examples] on GitHub. The following are optional settings that, if required, must be added to the previous command. You can only enable these options when you create the cluster, not after. If you need to specify these options, you must create the cluster with an https://eksctl.io/usage/creating-and-managing-clusters/#using-config-files[eksctl config file] and specify the settings, rather than using the previous command. * If you want to specify one or more security groups that Amazon EKS assigns to the network interfaces that it creates, specify the https://eksctl.io/usage/schema/#vpc-securityGroup[securityGroup] option. + Whether you choose any security groups or not, Amazon EKS creates a security group that enables communication between your cluster and your VPC. Amazon EKS associates this security group, and any that you choose, to the network interfaces that it creates. For more information about the cluster security group that Amazon EKS creates, see <>. You can modify the rules in the cluster security group that Amazon EKS creates. -* If you want to specify which `IPv4` Classless Inter-domain Routing (CIDR) block [.noloc]`Kubernetes` assigns service IP addresses from, specify the https://eksctl.io/usage/schema/#kubernetesNetworkConfig-serviceIPv4CIDR[serviceIPv4CIDR] option. +* If you want to specify which `IPv4` Classless Inter-domain Routing (CIDR) block Kubernetes assigns service IP addresses from, specify the https://eksctl.io/usage/schema/#kubernetesNetworkConfig-serviceIPv4CIDR[serviceIPv4CIDR] option. + -Specifying your own range can help prevent conflicts between [.noloc]`Kubernetes` services and other networks peered or connected to your VPC. Enter a range in CIDR notation. For example: `10.2.0.0/16`. +Specifying your own range can help prevent conflicts between Kubernetes services and other networks peered or connected to your VPC. Enter a range in CIDR notation. For example: `10.2.0.0/16`. + The CIDR block must meet the following requirements: + @@ -127,16 +118,16 @@ The CIDR block must meet the following requirements: ** Have a minimum size of `/24` and a maximum size of `/12`. ** Not overlap with the range of the VPC for your Amazon EKS resources. + -You can only specify this option when using the `IPv4` address family and only at cluster creation. If you don't specify this, then [.noloc]`Kubernetes` assigns service IP addresses from either the `10.100.0.0/16` or `172.20.0.0/16` CIDR blocks. -* If you're creating cluster and want the cluster to assign `IPv6` addresses to [.noloc]`Pods` and services instead of `IPv4` addresses, specify the https://eksctl.io/usage/schema/#kubernetesNetworkConfig-ipFamily[ipFamily] option. +You can only specify this option when using the `IPv4` address family and only at cluster creation. If you don't specify this, then Kubernetes assigns service IP addresses from either the `10.100.0.0/16` or `172.20.0.0/16` CIDR blocks. +* If you're creating cluster and want the cluster to assign `IPv6` addresses to Pods and services instead of `IPv4` addresses, specify the https://eksctl.io/usage/schema/#kubernetesNetworkConfig-ipFamily[ipFamily] option. + -[.noloc]`Kubernetes` assigns `IPv4` addresses to [.noloc]`Pods` and services, by default. Before deciding to use the `IPv6` family, make sure that you're familiar with all of the considerations and requirements in the xref:network-requirements-vpc[VPC requirements and considerations,linkend=network-requirements-vpc], xref:network-requirements-subnets[Subnet requirements and considerations,linkend=network-requirements-subnets], xref:sec-group-reqs[View Amazon EKS security group requirements for clusters,linkend=sec-group-reqs], and <> topics. If you choose the `IPv6` family, you can't specify an address range for [.noloc]`Kubernetes` to assign `IPv6` service addresses from like you can for the `IPv4` family. [.noloc]`Kubernetes` assigns service addresses from the unique local address range (`fc00::/7`). +Kubernetes assigns `IPv4` addresses to Pods and services, by default. Before deciding to use the `IPv6` family, make sure that you're familiar with all of the considerations and requirements in the <>, <>, <>, and <> topics. If you choose the `IPv6` family, you can't specify an address range for Kubernetes to assign `IPv6` service addresses from like you can for the `IPv4` family. Kubernetes assigns service addresses from the unique local address range (`fc00::/7`). -[[step2-console,step2-console.title]] +[#step2-console] === Create cluster - {aws} console . Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. -. Choose *Add cluster* and then choose *Create*. +. Choose *Add cluster* and then choose *Create*. . Under *Configuration options* select *Custom configuration* ** For information about quickly creating a cluster wih EKS Auto Mode, see <>. . Under *EKS Auto Mode*, toggle *Use EKS Auto Mode* off. @@ -144,10 +135,10 @@ You can only specify this option when using the `IPv4` address family and only a . On the *Configure cluster* page, enter the following fields: + ** *Name* – A name for your cluster. The name can contain only alphanumeric characters (case-sensitive), hyphens, and underscores. It must start with an alphanumeric character and can't be longer than 100 characters. The name must be unique within the {aws} Region and {aws} account that you're creating the cluster in. -** *Cluster IAM role* – Choose the Amazon EKS cluster IAM role that you created to allow the [.noloc]`Kubernetes` control plane to manage {aws} resources on your behalf. -** *[.noloc]`Kubernetes` version* – The version of [.noloc]`Kubernetes` to use for your cluster. We recommend selecting the latest version, unless you need an earlier version. -** *Support type* -- The [.noloc]`Kubernetes` version policy you would like to set for your cluster. If you want your cluster to only run on a standard support version, you can choose *Standard support*. If you want your cluster to enter extended support at the end of standard support for a version, you can choose *Extended support*. If you select a [.noloc]`Kubernetes` version that is currently in extended support, you can not select standard support as an option. -** *Secrets encryption* – (Optional) Choose to enable secrets encryption of [.noloc]`Kubernetes` secrets using a KMS key. You can also enable this after you create your cluster. Before you enable this capability, make sure that you're familiar with the information in xref:enable-kms[Encrypt Kubernetes secrets with {aws} KMS on existing clusters,linkend=enable-kms]. +** *Cluster IAM role* – Choose the Amazon EKS cluster IAM role that you created to allow the Kubernetes control plane to manage {aws} resources on your behalf. +** *Kubernetes version* – The version of Kubernetes to use for your cluster. We recommend selecting the latest version, unless you need an earlier version. +** *Support type* -- The Kubernetes version policy you would like to set for your cluster. If you want your cluster to only run on a standard support version, you can choose *Standard support*. If you want your cluster to enter extended support at the end of standard support for a version, you can choose *Extended support*. If you select a Kubernetes version that is currently in extended support, you can not select standard support as an option. +** *Secrets encryption* – (Optional) Choose to enable secrets encryption of Kubernetes secrets using a KMS key. You can also enable this after you create your cluster. Before you enable this capability, make sure that you're familiar with the information in <>. ** *Tags* – (Optional) Add any tags to your cluster. For more information, see <>. ** *ARC Zonal shift* - (Optional) You can use Route53 Application Recovery controller to mitigate impaired availability zones. For more information, see <>. . In the *Cluster access* section of the configure cluster page, enter the following fields: @@ -157,7 +148,7 @@ You can only specify this option when using the `IPv4` address family and only a When you're done with this page, choose *Next*. . On the *Specify networking* page, select values for the following fields: + -** *VPC* – Choose an existing VPC that meets xref:network-requirements-vpc[Amazon EKS VPC requirements,linkend=network-requirements-vpc] to create your cluster in. Before choosing a VPC, we recommend that you're familiar with all of the requirements and considerations in xref:network-reqs[View Amazon EKS networking requirements for VPC and subnets,linkend=network-reqs]. You can't change which VPC you want to use after cluster creation. If no VPCs are listed, then you need to create one first. For more information, see <>. +** *VPC* – Choose an existing VPC that meets <> to create your cluster in. Before choosing a VPC, we recommend that you're familiar with all of the requirements and considerations in <>. You can't change which VPC you want to use after cluster creation. If no VPCs are listed, then you need to create one first. For more information, see <>. ** *Subnets* – By default, all available subnets in the VPC specified in the previous field are preselected. You must select at least two. + The subnets that you choose must meet the <>. Before selecting subnets, we recommend that you're familiar with all of the <>. @@ -165,13 +156,13 @@ The subnets that you choose must meet the <>. You can modify the rules in the cluster security group that Amazon EKS creates. -** *Choose cluster IP address family* – You can choose either *IPv4* and *IPv6*. +** *Choose cluster IP address family* – You can choose either *IPv4* and *IPv6*. + -[.noloc]`Kubernetes` assigns `IPv4` addresses to [.noloc]`Pods` and services, by default. Before deciding to use the `IPv6` family, make sure that you're familiar with all of the considerations and requirements in the <>, <>, <>, and <> topics. If you choose the `IPv6` family, you can't specify an address range for [.noloc]`Kubernetes` to assign `IPv6` service addresses from like you can for the `IPv4` family. [.noloc]`Kubernetes` assigns service addresses from the unique local address range (`fc00::/7`). +Kubernetes assigns `IPv4` addresses to Pods and services, by default. Before deciding to use the `IPv6` family, make sure that you're familiar with all of the considerations and requirements in the <>, <>, <>, and <> topics. If you choose the `IPv6` family, you can't specify an address range for Kubernetes to assign `IPv6` service addresses from like you can for the `IPv4` family. Kubernetes assigns service addresses from the unique local address range (`fc00::/7`). + -** (Optional) Choose *Configure [.noloc]`Kubernetes` Service IP address range* and specify a *Service `IPv4` range*. +** (Optional) Choose *Configure Kubernetes Service IP address range* and specify a *Service `IPv4` range*. + -Specifying your own range can help prevent conflicts between [.noloc]`Kubernetes` services and other networks peered or connected to your VPC. Enter a range in CIDR notation. For example: `10.2.0.0/16`. +Specifying your own range can help prevent conflicts between Kubernetes services and other networks peered or connected to your VPC. Enter a range in CIDR notation. For example: `10.2.0.0/16`. + The CIDR block must meet the following requirements: + @@ -180,20 +171,20 @@ The CIDR block must meet the following requirements: *** Not overlap with the range of the VPC for your Amazon EKS resources. + -You can only specify this option when using the `IPv4` address family and only at cluster creation. If you don't specify this, then [.noloc]`Kubernetes` assigns service IP addresses from either the `10.100.0.0/16` or `172.20.0.0/16` CIDR blocks. +You can only specify this option when using the `IPv4` address family and only at cluster creation. If you don't specify this, then Kubernetes assigns service IP addresses from either the `10.100.0.0/16` or `172.20.0.0/16` CIDR blocks. ** For *Cluster endpoint access*, select an option. After your cluster is created, you can change this option. Before selecting a non-default option, make sure to familiarize yourself with the options and their implications. For more information, see <>. + When you're done with this page, choose *Next*. -. (Optional) On the *Configure observability* page, choose which *Metrics* and *Control plane logging* options to turn on. By default, each log type is turned off. +. (Optional) On the *Configure observability* page, choose which *Metrics* and *Control plane logging* options to turn on. By default, each log type is turned off. + -** For more information about the [.noloc]`Prometheus` metrics option, see <>. +** For more information about the Prometheus metrics option, see <>. ** For more information about the *Control plane logging* options, see <>. + -When you're done with this page, choose *Next*. -. On the *Select add-ons* page, choose the add-ons that you want to add to your cluster. Certain add-ons are pre-selected. You can choose as many *Amazon EKS add-ons* and *{aws} Marketplace add-ons* as you require. If the *{aws} Marketplace add-ons* that you want to install isn't listed, you can click the page numbering to view additional page results or search for available *{aws} Marketplace add-ons* by entering text in the search box. You can also filter by *category*, *vendor*, or *pricing model* and then choose the add-ons from the search results. When creating a cluster, you can view, select, and install any add-on that supports EKS Pod Identities as detailed in <>. +When you're done with this page, choose *Next*. +. On the *Select add-ons* page, choose the add-ons that you want to add to your cluster. Certain add-ons are pre-selected. You can choose as many *Amazon EKS add-ons* and *{aws} Marketplace add-ons* as you require. If the *{aws} Marketplace add-ons* that you want to install isn't listed, you can click the page numbering to view additional page results or search for available *{aws} Marketplace add-ons* by entering text in the search box. You can also filter by *category*, *vendor*, or *pricing model* and then choose the add-ons from the search results. When creating a cluster, you can view, select, and install any add-on that supports EKS Pod Identities as detailed in <>. + -When you're done with this page, choose *Next*. +When you're done with this page, choose *Next*. + Some add-ons, such as Amazon VPC CNI, CoreDNS, and kube-proxy, are installed by default. If you disable any of the default add-ons, this may affect your ability to run Kubernetes applications. . On the *Configure selected add-ons settings* page, select the version that you want to install. You can always update to a later version after cluster creation. @@ -201,31 +192,31 @@ Some add-ons, such as Amazon VPC CNI, CoreDNS, and kube-proxy, are installed by For add-ons that support EKS Pod Identities, you can use the console to automatically generate the role with the name, {aws} managed policy, and trust policy prepopulated specifically for the add-on. You can re-use existing roles or create new roles for supported add-ons. For the steps to use the console to create roles for add-ons that support EKS Pod Identities, see <<_create_add_on_console>>. If an add-on does not support EKS Pod Identity, a message displays with instructions to use the wizard to create the IAM roles for service accounts (IRSA) after the cluster is created. + You can update the configuration of each add-on after cluster creation. For more information about configuring add-ons, see <>. When you're done with this page, choose *Next*. -. On the *Review and create* page, review the information that you entered or selected on the previous pages. If you need to make changes, choose *Edit*. When you're satisfied, choose *Create*. The *Status* field shows *CREATING* while the cluster is provisioned. +. On the *Review and create* page, review the information that you entered or selected on the previous pages. If you need to make changes, choose *Edit*. When you're satisfied, choose *Create*. The *Status* field shows *CREATING* while the cluster is provisioned. + NOTE: You might receive an error that one of the Availability Zones in your request doesn't have sufficient capacity to create an Amazon EKS cluster. If this happens, the error output contains the Availability Zones that can support a new cluster. Retry creating your cluster with at least two subnets that are located in the supported Availability Zones for your account. For more information, see <>. + Cluster provisioning takes several minutes. . Continue with <> -[[step2-cli,step2-cli.title]] +[#step2-cli] === Create cluster - {aws} CLI . Create your cluster with the command that follows. Before running the command, make the following replacements: + ** Replace [.replaceable]`region-code` with the {aws} Region that you want to create your cluster in. ** Replace [.replaceable]`my-cluster` with a name for your cluster. The name can contain only alphanumeric characters (case-sensitive), hyphens, and underscores. It must start with an alphanumeric character and can't be longer than 100 characters. The name must be unique within the {aws} Region and {aws} account that you're creating the cluster in. -** Replace [.replaceable]`1.30` with any xref:kubernetes-versions[Amazon EKS supported version,linkend=kubernetes-versions]. +** Replace [.replaceable]`{k8s-n}` with any link:eks/latest/userguide/kubernetes-versions.html[Amazon EKS supported version,type="documentation"]. ** Replace [.replaceable]`111122223333` with your account ID and [.replaceable]`myAmazonEKSClusterRole` with the name of your cluster IAM role. ** Replace the values for `subnetIds` with your own. You can also add additional IDs. You must specify at least two subnet IDs. + -The subnets that you choose must meet the xref:network-requirements-subnets[Amazon EKS subnet requirements,linkend=network-requirements-subnets]. Before selecting subnets, we recommend that you're familiar with all of the xref:network-reqs[Amazon EKS VPC and subnet requirements and considerations,linkend=network-reqs]. +The subnets that you choose must meet the <>. Before selecting subnets, we recommend that you're familiar with all of the <>. ** If you don't want to specify a security group ID, remove `,securityGroupIds=sg-` from the command. If you want to specify one or more security group IDs, replace the values for `securityGroupIds` with your own. You can also add additional IDs. + Whether you choose any security groups or not, Amazon EKS creates a security group that enables communication between your cluster and your VPC. Amazon EKS associates this security group, and any that you choose, to the network interfaces that it creates. For more information about the cluster security group that Amazon EKS creates, see <>. You can modify the rules in the cluster security group that Amazon EKS creates. + [source,bash,subs="verbatim,attributes"] ---- -aws eks create-cluster --region region-code --name my-cluster --kubernetes-version 1.30 \ +aws eks create-cluster --region region-code --name my-cluster --kubernetes-version {k8s-n} \ --role-arn {arn-aws}iam::111122223333:role/myAmazonEKSClusterRole \ --resources-vpc-config subnetIds=subnet-ExampleID1,subnet-ExampleID2,securityGroupIds=sg-ExampleID1 ---- @@ -236,12 +227,12 @@ The following are optional settings that, if required, must be added to the prev ** By default, EKS installs multiple networking add-ons during cluster creation. This includes the Amazon VPC CNI, CoreDNS, and kube-proxy. + -If you'd like to disable the installation of these default networking add-ons, use the parameter below. This may be used for alternate CNIs, such as Cilium. Review the link:eks/latest/APIReference/API_CreateCluster.html[EKS API reference,type="documentation"] for more information. +If you'd like to disable the installation of these default networking add-ons, use the parameter below. This may be used for alternate CNIs, such as Cilium. Review the link:eks/latest/APIReference/API_CreateCluster.html[EKS API reference,type="documentation"] for more information. + `aws eks create-cluster --bootstrapSelfManagedAddons false` -** If you want to specify which `IPv4` Classless Inter-domain Routing (CIDR) block [.noloc]`Kubernetes` assigns service IP addresses from, you must specify it by adding the `--kubernetes-network-config serviceIpv4Cidr=` to the following command. +** If you want to specify which `IPv4` Classless Inter-domain Routing (CIDR) block Kubernetes assigns service IP addresses from, you must specify it by adding the `--kubernetes-network-config serviceIpv4Cidr=` to the following command. + -Specifying your own range can help prevent conflicts between [.noloc]`Kubernetes` services and other networks peered or connected to your VPC. Enter a range in CIDR notation. For example: `10.2.0.0/16`. +Specifying your own range can help prevent conflicts between Kubernetes services and other networks peered or connected to your VPC. Enter a range in CIDR notation. For example: `10.2.0.0/16`. + The CIDR block must meet the following requirements: + @@ -250,10 +241,10 @@ The CIDR block must meet the following requirements: *** Not overlap with the range of the VPC for your Amazon EKS resources. + -You can only specify this option when using the `IPv4` address family and only at cluster creation. If you don't specify this, then [.noloc]`Kubernetes` assigns service IP addresses from either the `10.100.0.0/16` or `172.20.0.0/16` CIDR blocks. -** If you're creating a cluster and want the cluster to assign `IPv6` addresses to [.noloc]`Pods` and services instead of `IPv4` addresses, add `--kubernetes-network-config ipFamily=ipv6` to the following command. +You can only specify this option when using the `IPv4` address family and only at cluster creation. If you don't specify this, then Kubernetes assigns service IP addresses from either the `10.100.0.0/16` or `172.20.0.0/16` CIDR blocks. +** If you're creating a cluster and want the cluster to assign `IPv6` addresses to Pods and services instead of `IPv4` addresses, add `--kubernetes-network-config ipFamily=ipv6` to the following command. + -[.noloc]`Kubernetes` assigns `IPv4` addresses to [.noloc]`Pods` and services, by default. Before deciding to use the `IPv6` family, make sure that you're familiar with all of the considerations and requirements in the <>, <>, <>, and <> topics. If you choose the `IPv6` family, you can't specify an address range for [.noloc]`Kubernetes` to assign `IPv6` service addresses from like you can for the `IPv4` family. [.noloc]`Kubernetes` assigns service addresses from the unique local address range (`fc00::/7`). +Kubernetes assigns `IPv4` addresses to Pods and services, by default. Before deciding to use the `IPv6` family, make sure that you're familiar with all of the considerations and requirements in the <>, <>, <>, and <> topics. If you choose the `IPv6` family, you can't specify an address range for Kubernetes to assign `IPv6` service addresses from like you can for the `IPv4` family. Kubernetes assigns service addresses from the unique local address range (`fc00::/7`). + . It takes several minutes to provision the cluster. You can query the status of your cluster with the following command. + @@ -265,7 +256,7 @@ aws eks describe-cluster --region region-code --name my-cluster --query "cluster Don't proceed to the next step until the output returned is `ACTIVE`. . Continue with <> -[[step3,step3.title]] +[#step3] == Step 3: Update kubeconfig . If you created your cluster using `eksctl`, then you can skip this step. This is because `eksctl` already completed this step for you. Enable `kubectl` to communicate with your cluster by adding a new context to the `kubectl` `config` file. For more information about how to create and update the file, see <>. + @@ -297,21 +288,21 @@ kubernetes ClusterIP 10.100.0.1 443/TCP 28h == Step 4: Cluster setup -. (Recommended) To use some Amazon EKS add-ons, or to enable individual [.noloc]`Kubernetes` workloads to have specific {aws} Identity and Access Management (IAM) permissions, xref:enable-iam-roles-for-service-accounts[create an IAM OpenID Connect (OIDC) provider,linkend=enable-iam-roles-for-service-accounts] for your cluster. You only need to create an IAM [.noloc]`OIDC` provider for your cluster once. To learn more about Amazon EKS add-ons, see <>. To learn more about assigning specific IAM permissions to your workloads, see <>. -. (Recommended) Configure your cluster for the [.noloc]`Amazon VPC CNI plugin for Kubernetes` plugin before deploying Amazon EC2 nodes to your cluster. By default, the plugin was installed with your cluster. When you add Amazon EC2 nodes to your cluster, the plugin is automatically deployed to each Amazon EC2 node that you add. The plugin requires you to attach one of the following IAM policies to an IAM role. If your cluster uses the `IPv4` family, use the link:aws-managed-policy/latest/reference/AmazonEKS_CNI_Policy.html[AmazonEKS_CNI_Policy,type="documentation"] managed IAM policy. If your cluster uses the `IPv6` family, use an xref:cni-iam-role-create-ipv6-policy[IAM policy that you create,linkend=cni-iam-role-create-ipv6-policy]. +. (Recommended) To use some Amazon EKS add-ons, or to enable individual Kubernetes workloads to have specific {aws} Identity and Access Management (IAM) permissions, <> for your cluster. You only need to create an IAM OIDC provider for your cluster once. To learn more about Amazon EKS add-ons, see <>. To learn more about assigning specific IAM permissions to your workloads, see <>. +. (Recommended) Configure your cluster for the Amazon VPC CNI plugin for Kubernetes plugin before deploying Amazon EC2 nodes to your cluster. By default, the plugin was installed with your cluster. When you add Amazon EC2 nodes to your cluster, the plugin is automatically deployed to each Amazon EC2 node that you add. The plugin requires you to attach one of the following IAM policies to an IAM role. If your cluster uses the `IPv4` family, use the link:aws-managed-policy/latest/reference/AmazonEKS_CNI_Policy.html[AmazonEKS_CNI_Policy,type="documentation"] managed IAM policy. If your cluster uses the `IPv6` family, use an <>. + The IAM role that you attach the policy to can be the node IAM role, or a dedicated role used only for the plugin. We recommend attaching the policy to this role. For more information about creating the role, see <> or <>. -. If you deployed your cluster using the {aws-management-console}, you can skip this step. The {aws-management-console} deploys the [.noloc]`Amazon VPC CNI plugin for Kubernetes`, [.noloc]`CoreDNS`, and `kube-proxy` Amazon EKS add-ons, by default. +. If you deployed your cluster using the {aws-management-console}, you can skip this step. The {aws-management-console} deploys the Amazon VPC CNI plugin for Kubernetes, CoreDNS, and `kube-proxy` Amazon EKS add-ons, by default. + -If you deploy your cluster using either `eksctl` or the {aws} CLI, then the [.noloc]`Amazon VPC CNI plugin for Kubernetes`, [.noloc]`CoreDNS`, and `kube-proxy` self-managed add-ons are deployed. You can migrate the [.noloc]`Amazon VPC CNI plugin for Kubernetes`, [.noloc]`CoreDNS`, and `kube-proxy` self-managed add-ons that are deployed with your cluster to Amazon EKS add-ons. For more information, see <>. -. (Optional) If you haven't already done so, you can enable [.noloc]`Prometheus` metrics for your cluster. For more information, see link:prometheus/latest/userguide/AMP-collector-how-to.html#AMP-collector-create[Create a scraper,type="documentation"] in the _Amazon Managed Service for Prometheus User Guide_. -. If you plan to deploy workloads to your cluster that use Amazon EBS volumes , and you created a `1.23` or later cluster, then you must install the xref:ebs-csi[Amazon EBS CSI,linkend=ebs-csi] to your cluster before deploying the workloads. +If you deploy your cluster using either `eksctl` or the {aws} CLI, then the Amazon VPC CNI plugin for Kubernetes, CoreDNS, and `kube-proxy` self-managed add-ons are deployed. You can migrate the Amazon VPC CNI plugin for Kubernetes, CoreDNS, and `kube-proxy` self-managed add-ons that are deployed with your cluster to Amazon EKS add-ons. For more information, see <>. +. (Optional) If you haven't already done so, you can enable Prometheus metrics for your cluster. For more information, see link:prometheus/latest/userguide/AMP-collector-how-to.html#AMP-collector-create[Create a scraper,type="documentation"] in the _Amazon Managed Service for Prometheus User Guide_. +. If you plan to deploy workloads to your cluster that use Amazon EBS volumes, then you must install the <> to your cluster before deploying the workloads. == Next steps -* The link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] that created the cluster is the only principal that has access to the cluster. <> so they can access your cluster. +* The link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] that created the cluster is the only principal that has access to the cluster. <> so they can access your cluster. * If the IAM principal that created the cluster only has the minimum IAM permissions referenced in the prerequisites, then you might want to add additional Amazon EKS permissions for that principal. For more information about granting Amazon EKS permissions to IAM principals, see <>. -* If you want the IAM principal that created the cluster, or any other principals to view [.noloc]`Kubernetes` resources in the Amazon EKS console, grant the <> to the entities. +* If you want the IAM principal that created the cluster, or any other principals to view Kubernetes resources in the Amazon EKS console, grant the <> to the entities. * If you want nodes and IAM principals to access your cluster from within your VPC, enable the private endpoint for your cluster. The public endpoint is enabled by default. You can disable the public endpoint once you've enabled the private endpoint, if desired. For more information, see <>. * <>. * <>. diff --git a/latest/ug/clusters/delete-cluster.adoc b/latest/ug/clusters/delete-cluster.adoc index d9da28153..919af670a 100644 --- a/latest/ug/clusters/delete-cluster.adoc +++ b/latest/ug/clusters/delete-cluster.adoc @@ -1,13 +1,9 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[delete-cluster,delete-cluster.title]] +[#delete-cluster] = Delete a cluster -:info_doctype: section -:info_title: Delete a cluster :info_titleabbrev: Delete a cluster -:info_abstract: Learn how to delete Amazon EKS clusters, including managed and self-managed node groups, Fargate profiles, related services, and {aws} CloudFormation stacks using eksctl, {aws-management-console}, or {aws} CLI for cost optimization and resource cleanup. [abstract] -- @@ -22,9 +18,8 @@ You can delete a cluster with `eksctl`, the {aws-management-console}, or the {aw == Considerations -* If you have active services in your cluster that are associated with a load balancer, you must delete those services before deleting the cluster so that the load balancers are deleted properly. Otherwise, you can have orphaned resources in your VPC that prevent you from being able to delete the VPC. * If you receive an error because the cluster creator has been removed, see link:premiumsupport/knowledge-center/eks-api-server-unauthorized-error[this article,type="marketing"] to resolve. -* Amazon Managed Service for Prometheus resources are outside of the cluster lifecycle and need to be maintained independent of the cluster. When you delete your cluster, make sure to also delete any applicable scrapers to stop applicable costs. For more information, see link:prometheus/latest/userguide/AMP-collector-how-to.html#AMP-collector-list-delete[Find and delete scrapers,type="documentation"] in the _Amazon Managed Service for Prometheus User Guide_. +* Amazon Managed Service for Prometheus resources are outside of the cluster lifecycle and need to be maintained independent of the cluster. When you delete your cluster, make sure to also delete any applicable scrapers to stop applicable costs. For more information, see link:prometheus/latest/userguide/AMP-collector-how-to.html#AMP-collector-list-delete[Find and delete scrapers,type="documentation"] in the _Amazon Managed Service for Prometheus User Guide_. * To remove a connected cluster, see <> === Considerations for EKS Auto Mode @@ -51,7 +46,8 @@ For instructions on how to install or upgrade `eksctl`, see https://eksctl.io/in ---- kubectl get svc --all-namespaces ---- -.. Delete any services that have an associated `EXTERNAL-IP` value. These services are fronted by an Elastic Load Balancing load balancer, and you must delete them in [.noloc]`Kubernetes` to allow the load balancer and associated resources to be properly released. +.. Delete any services that have an associated `EXTERNAL-IP` value. These services are fronted by an Elastic Load Balancing load balancer, and you must delete them in Kubernetes to allow the load balancer and associated resources to be properly released. +Replace [.replaceable]`service-name` with the name of each service listed as described. + [source,bash,subs="verbatim,attributes"] ---- @@ -84,7 +80,8 @@ Output: ---- kubectl get svc --all-namespaces ---- -. Delete any services that have an associated `EXTERNAL-IP` value. These services are fronted by an Elastic Load Balancing load balancer, and you must delete them in [.noloc]`Kubernetes` to allow the load balancer and associated resources to be properly released. +. Delete any services that have an associated `EXTERNAL-IP` value. These services are fronted by an Elastic Load Balancing load balancer, and you must delete them in Kubernetes to allow the load balancer and associated resources to be properly released. +Replace [.replaceable]`service-name` with the name of each service listed as described. + [source,bash,subs="verbatim,attributes"] ---- @@ -94,15 +91,15 @@ kubectl delete svc service-name + .. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. .. In the left navigation pane, choose Amazon EKS *Clusters*, and then in the tabbed list of clusters, choose the name of the cluster that you want to delete. -.. Choose the *Compute* tab and choose a node group to delete. Choose *Delete*, enter the name of the node group, and then choose *Delete*. Delete all node groups in the cluster. +.. Choose the *Compute* tab and choose a node group to delete. Choose *Delete*, enter the name of the node group, and then choose *Delete*. Delete all node groups in the cluster. + -NOTE: The node groups listed are xref:managed-node-groups[managed node groups,linkend=managed-node-groups] only. -.. Choose a *Fargate Profile* to delete, select *Delete*, enter the name of the profile, and then choose *Delete*. Delete all Fargate profiles in the cluster. +NOTE: The node groups listed are <> only. +.. Choose a *Fargate Profile* to delete, select *Delete*, enter the name of the profile, and then choose *Delete*. Delete all Fargate profiles in the cluster. . Delete all self-managed node {aws} CloudFormation stacks. + .. Open the link:cloudformation/[{aws} CloudFormation console,type="console"]. .. Choose the node stack to delete, and then choose *Delete*. -.. In the *Delete stack* confirmation dialog box, choose *Delete stack*. Delete all self-managed node stacks in the cluster. +.. In the *Delete stack* confirmation dialog box, choose *Delete stack*. Delete all self-managed node stacks in the cluster. . Delete the cluster. + .. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. @@ -112,7 +109,7 @@ NOTE: The node groups listed are xref:managed-node-groups[managed node groups,li + .. Open the link:cloudformation/[{aws} CloudFormation console,type="console"]. .. Select the VPC stack to delete, and then choose *Delete*. -.. In the *Delete stack* confirmation dialog box, choose *Delete stack*. +.. In the *Delete stack* confirmation dialog box, choose *Delete stack*. == Delete cluster ({aws} CLI) @@ -123,7 +120,8 @@ NOTE: The node groups listed are xref:managed-node-groups[managed node groups,li ---- kubectl get svc --all-namespaces ---- -. Delete any services that have an associated `EXTERNAL-IP` value. These services are fronted by an Elastic Load Balancing load balancer, and you must delete them in [.noloc]`Kubernetes` to allow the load balancer and associated resources to be properly released. +. Delete any services that have an associated `EXTERNAL-IP` value. These services are fronted by an Elastic Load Balancing load balancer, and you must delete them in Kubernetes to allow the load balancer and associated resources to be properly released. +Replace [.replaceable]`service-name` with the name of each service listed as described. + [source,bash,subs="verbatim,attributes"] ---- @@ -138,7 +136,7 @@ kubectl delete svc service-name aws eks list-nodegroups --cluster-name my-cluster ---- + -NOTE: The node groups listed are xref:managed-node-groups[managed node groups,linkend=managed-node-groups] only. +NOTE: The node groups listed are <> only. .. Delete each node group with the following command. Delete all node groups in the cluster. + [source,bash,subs="verbatim,attributes"] @@ -191,3 +189,5 @@ aws cloudformation list-stacks --query "StackSummaries[].StackName" ---- aws cloudformation delete-stack --stack-name my-vpc-stack ---- + +include::deletion-protection.adoc[leveloffset=+1] diff --git a/latest/ug/clusters/deletion-protection.adoc b/latest/ug/clusters/deletion-protection.adoc new file mode 100644 index 000000000..6308a0ec7 --- /dev/null +++ b/latest/ug/clusters/deletion-protection.adoc @@ -0,0 +1,34 @@ +include::../attributes.txt[] + +[.topic] +[#deletion-protection] += Protect EKS clusters from accidental deletion + +Accidentally deleting an EKS cluster may impair Kubernetes cluster operations. + +You can now protect EKS clusters from accidental deletion. If you enable deletion protection on a cluster, you must first disable deletion protection before you can delete the cluster. + +The purpose of deletion protection is to prevent accidents. You should carefully restrict who is authorized to delete clusters. + +If you try to delete an active cluster that has deletion protection turned on, you will receive a `InvalidRequestException` . + +IMPORTANT: If you enable deletion protection on a cluster, you must have **both** the UpdateClusterConfig and DeleteCluster IAM permissions to first remove the deletion protection, and finally delete the cluster. + +NOTE: If the cluster state is creating, failed, or deleting, you can delete the cluster even if deletion protection is turned on. + +## To enable deletion protection for an existing cluster + +You can only run this on a cluster in the active status. + +``` +aws eks update-cluster-config --name --region --deletion-protection +``` + +## To disable deletion protection for an existing cluster + +``` +aws eks update-cluster-config --name --region --no-deletion-protection +``` + + + diff --git a/latest/ug/clusters/disable-extended-support.adoc b/latest/ug/clusters/disable-extended-support.adoc deleted file mode 100644 index eb45b0932..000000000 --- a/latest/ug/clusters/disable-extended-support.adoc +++ /dev/null @@ -1,38 +0,0 @@ -include::../attributes.txt[] -[.topic] -[[disable-extended-support,disable-extended-support.title]] -= Prevent increased cluster costs by disabling EKS extended support -:info_titleabbrev: Disable extended support - -This topic describes how to set the _upgrade policy_ of an EKS cluster to disable extended support. The upgrade policy of an EKS cluster determines what happens when a cluster reaches the end of the standard _support period_. If a cluster upgrade policy has extended support disabled, it will be automatically upgraded to the next [.noloc]`Kubernetes` version. - -For more information about upgrade policies, see <>. - -[IMPORTANT] -==== - -You cannot disable extended support once your cluster has entered it. You can only disable extended support for clusters on standard support. - -{aws} recommends upgrading your cluster to a version in the standard support period. - -==== - -[[disable-support-policy-console,disable-support-policy-console.title]] -== Disable EKS extended support ({aws} Console) -. Navigate to your EKS cluster in the {aws} Console. Select the *Overview* tab on the *Cluster Info* page. -. In the *Kubernetes version setting* section, select *Manage*. -. Select *Standard support* and then *Save changes*. - - -[[disable-support-policy-cli,disable-support-policy-cli.title]] -== Disable EKS extended support ({aws} CLI) -. Verify the {aws} CLI is installed and you are logged in. link:cli/latest/userguide/getting-started-install.html[Learn how to update and install the {aws} CLI.,type="documentation"] -. Determine the name of your EKS cluster. -. Run the following command: -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks update-cluster-config \ ---name \ ---upgrade-policy supportType=STANDARD ----- diff --git a/latest/ug/clusters/disable-windows-support.adoc b/latest/ug/clusters/disable-windows-support.adoc index e884af946..e3f5658cf 100644 --- a/latest/ug/clusters/disable-windows-support.adoc +++ b/latest/ug/clusters/disable-windows-support.adoc @@ -1,15 +1,12 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[disable-windows-support,disable-windows-support.title]] -= Disable [.noloc]`Windows` support -:info_doctype: section -:info_title: Disable Windows support +[#disable-windows-support] += Disable Windows support -. If your cluster contains Amazon Linux nodes and you use xref:security-groups-for-pods[security groups for Pods,linkend=security-groups-for-pods] with them, then skip this step. +. If your cluster contains Amazon Linux nodes and you use <> with them, then skip this step. + -Remove the `AmazonVPCResourceController` managed IAM policy from your <>. Replace [.replaceable]`eksClusterRole` with the name of your cluster role and [.replaceable]`111122223333` with your account ID. +Remove the `AmazonVPCResourceController` managed IAM policy from your <>. Replace [.replaceable]`eksClusterRole` with the name of your cluster role. + [source,bash,subs="verbatim,attributes"] ---- @@ -17,7 +14,7 @@ aws iam detach-role-policy \ --role-name eksClusterRole \ --policy-arn {arn-aws}iam::aws:policy/AmazonEKSVPCResourceController ---- -. Disable [.noloc]`Windows` IPAM in the `amazon-vpc-cni` ConfigMap. +. Disable Windows IPAM in the `amazon-vpc-cni` ConfigMap. + [source,bash,subs="verbatim,attributes"] ---- diff --git a/latest/ug/clusters/enable-extended-support.adoc b/latest/ug/clusters/enable-extended-support.adoc deleted file mode 100644 index 2e6dd4b78..000000000 --- a/latest/ug/clusters/enable-extended-support.adoc +++ /dev/null @@ -1,42 +0,0 @@ -include::../attributes.txt[] -[.topic] -[[enable-extended-support,enable-extended-support.title]] -= Add flexibility to plan Kubernetes version upgrades by enabling EKS extended support -:info_titleabbrev: Enable extended support - -This topic describes how to set the _upgrade policy_ of an EKS cluster to enable extended support. The upgrade policy of an EKS cluster determines what happens when a cluster reaches the end of the standard _support period_. If a cluster upgrade policy has extended support enabled, it will enter the extended support period at the end of the standard support period. The cluster will not be automatically upgraded at the end of the standard support period. - -Clusters actually in the _extended support period_ incur higher costs. If a cluster merely has the upgrade policy set to enable extended support, and is otherwise in the _standard support period_, it incurs standard costs. - -EKS Clusters have the upgrade policy set to enable extended support by default. - -For more information about upgrade policies, see <>. - -[IMPORTANT] -==== - -If you want your cluster to stay on its current [.noloc]`Kubernetes` version to take advantage of the extended support period, you must enable the extended support upgrade policy before the end of standard support period. - -If you do not enable extended support, your cluster will be automatically upgraded. - -==== - -[[enable-support-policy-console,enable-support-policy-console.title]] -== Enable EKS extended support ({aws} Console) -. Navigate to your EKS cluster in the {aws} Console. Select the *Overview* tab on the *Cluster Info* page. -. In the *Kubernetes version settings* section, select *Manage*. -. Select *Extended support* and then *Save changes*. - - -[[enable-support-policy-cli,enable-support-policy-cli.title]] -== Enable EKS extended support ({aws} CLI) -. Verify the {aws} CLI is installed and you are logged in. link:cli/latest/userguide/getting-started-install.html[Learn how to update and install the {aws} CLI.,type="documentation"] -. Determine the name of your EKS cluster. -. Run the following command: -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks update-cluster-config \ ---name \ ---upgrade-policy supportType=EXTENDED ----- diff --git a/latest/ug/clusters/kubernetes-versions-extended.adoc b/latest/ug/clusters/kubernetes-versions-extended.adoc deleted file mode 100644 index 7974fd6af..000000000 --- a/latest/ug/clusters/kubernetes-versions-extended.adoc +++ /dev/null @@ -1,161 +0,0 @@ -//!!NODE_ROOT
-include::../attributes.txt[] -[.topic] -[[kubernetes-versions-extended,kubernetes-versions-extended.title]] -= Review release notes for [.noloc]`Kubernetes` versions on extended support -:info_titleabbrev: Extended support versions - -[abstract] --- -This topic gives important changes to be aware of for each [.noloc]`Kubernetes` version in extended support. --- - -This topic gives important changes to be aware of for each [.noloc]`Kubernetes` version in extended support. When upgrading, carefully review the changes that have occurred between the old and new versions for your cluster. - -[[kubernetes-1.28,kubernetes-1.28.title]] -== [.noloc]`Kubernetes` 1.28 - -[.noloc]`Kubernetes` `1.28` is now available in Amazon EKS. For more information about [.noloc]`Kubernetes` `1.28`, see the https://kubernetes.io/blog/2023/08/15/kubernetes-v1-28-release/[official release announcement]. - - - -* [.noloc]`Kubernetes` `v1.28` expanded the supported skew between core node and control plane components by one minor version, from `n-2` to `n-3`, so that node components (``kubelet`` and `kube-proxy`) for the oldest supported minor version can work with control plane components (``kube-apiserver``, `kube-scheduler`, `kube-controller-manager`, `cloud-controller-manager`) for the newest supported minor version. -* Metrics `force_delete_pods_total` and `force_delete_pod_errors_total` in the `Pod GC Controller` are enhanced to account for all forceful pods deletion. A reason is added to the metric to indicate whether the pod is forcefully deleted because it's terminated, orphaned, terminating with the out-of-service taint, or terminating and unscheduled. -* The `PersistentVolume (PV)` controller has been modified to automatically assign a default `StorageClass` to any unbound `PersistentVolumeClaim` with the `storageClassName` not set. Additionally, the `PersistentVolumeClaim` admission validation mechanism within the API server has been adjusted to allow changing values from an unset state to an actual `StorageClass` name. - -For the complete [.noloc]`Kubernetes` `1.28` changelog, see https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.28.md#changelog-since-v1270. - -[[kubernetes-1.27,kubernetes-1.27.title]] -== [.noloc]`Kubernetes` 1.27 - -[.noloc]`Kubernetes` `1.27` is now available in Amazon EKS. For more information about [.noloc]`Kubernetes` `1.27`, see the https://kubernetes.io/blog/2023/04/11/kubernetes-v1-27-release/[official release announcement]. - -[IMPORTANT] -==== - - -* The support for the alpha `seccomp` annotations `seccomp.security.alpha.kubernetes.io/pod` and `container.seccomp.security.alpha.kubernetes.io` annotations was removed. The alpha `seccomp` annotations was deprecated in `1.19`, and with their removal in `1.27`, `seccomp` fields will no longer auto-populate for `Pods` with `seccomp` annotations. Instead, use the `securityContext.seccompProfile` field for `Pods` or containers to configure `seccomp` profiles. To check whether you are using the deprecated alpha `seccomp` annotations in your cluster, run the following command: -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl get pods --all-namespaces -o json | grep -E 'seccomp.security.alpha.kubernetes.io/pod|container.seccomp.security.alpha.kubernetes.io' ----- -* The `--container-runtime` command line argument for the `kubelet` was removed. The default container runtime for Amazon EKS has been `containerd` since `1.24`, which eliminates the need to specify the container runtime. From `1.27` onwards, Amazon EKS will ignore the `--container-runtime` argument passed to any bootstrap scripts. It is important that you don't pass this argument to `--kubelet-extra-args` in order to prevent errors during the node bootstrap process. You must remove the `--container-runtime` argument from all of your node creation workflows and build scripts. - -==== - -* The `kubelet` in [.noloc]`Kubernetes` `1.27` increased the default `kubeAPIQPS` to `50` and `kubeAPIBurst` to `100`. These enhancements allow the `kubelet` to handle a higher volume of API queries, improving response times and performance. When the demands for `Pods` increase, due to scaling requirements, the revised defaults ensure that the `kubelet` can efficiently manage the increased workload. As a result, `Pod` launches are quicker and cluster operations are more effective. -* You can use more fine grained `Pod` topology to spread policies such as `minDomain`. This parameter gives you the ability to specify the minimum number of domains your `Pods` should be spread across. `nodeAffinityPolicy` and `nodeTaintPolicy` allow for an extra level of granularity in governing `Pod` distribution. This is in accordance to node affinities, taints, and the `matchLabelKeys` field in the `topologySpreadConstraints` of your `Pod's` specification. This permits the selection of `Pods` for spreading calculations following a rolling upgrade. -* [.noloc]`Kubernetes` `1.27` promoted to beta a new policy mechanism for `StatefulSets` that controls the lifetime of their `PersistentVolumeClaims`(`PVCs`). The new `PVC` retention policy lets you specify if the `PVCs` generated from the `StatefulSet` spec template will be automatically deleted or retained when the `StatefulSet` is deleted or replicas in the `StatefulSet` are scaled down. -* The https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/[goaway-chance] option in the [.noloc]`Kubernetes` API server helps prevent `HTTP/2` client connections from being stuck on a single API server instance, by randomly closing a connection. When the connection is closed, the client will try to reconnect, and will likely land on a different API server as a result of load balancing. Amazon EKS version `1.27` has enabled `goaway-chance` flag. If your workload running on Amazon EKS cluster uses a client that is not compatible with https://www.rfc-editor.org/rfc/rfc7540#section-6.8[HTTP GOAWAY], we recommend that you update your client to handle `GOAWAY` by reconnecting on connection termination. - -For the complete [.noloc]`Kubernetes` `1.27` changelog, see https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.27.md#changelog-since-v1260. - -[[kubernetes-1.26,kubernetes-1.26.title]] -== [.noloc]`Kubernetes` 1.26 - -[.noloc]`Kubernetes` `1.26` is now available in Amazon EKS. For more information about [.noloc]`Kubernetes` `1.26`, see the https://kubernetes.io/blog/2022/12/09/kubernetes-v1-26-release/[official release announcement]. - -[IMPORTANT] -==== - -[.noloc]`Kubernetes` `1.26` no longer supports [.noloc]`CRI` `v1alpha2`. This results in the `kubelet` no longer registering the node if the container runtime doesn't support [.noloc]`CRI` `v1`. This also means that [.noloc]`Kubernetes` `1.26` doesn't support containerd minor version `1.5` and earlier. If you're using containerd, you need to upgrade to containerd version `1.6.0` or later before you upgrade any nodes to [.noloc]`Kubernetes` `1.26`. You also need to upgrade any other container runtimes that only support the `v1alpha2`. For more information, defer to the container runtime vendor. By default, [.noloc]`Amazon Linux` and [.noloc]`Bottlerocket` AMIs include containerd version `1.6.6`. - -==== - -* Before you upgrade to [.noloc]`Kubernetes` `1.26`, upgrade your [.noloc]`Amazon VPC CNI plugin for Kubernetes` to version `1.12` or later. If you don't upgrade to [.noloc]`Amazon VPC CNI plugin for Kubernetes` version `1.12` or later, the [.noloc]`Amazon VPC CNI plugin for Kubernetes` will crash. For more information, see <>. -* The https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/[goaway-chance] option in the [.noloc]`Kubernetes` API server helps prevent `HTTP/2` client connections from being stuck on a single API server instance, by randomly closing a connection. When the connection is closed, the client will try to reconnect, and will likely land on a different API server as a result of load balancing. Amazon EKS version `1.26` has enabled `goaway-chance` flag. If your workload running on Amazon EKS cluster uses a client that is not compatible with https://www.rfc-editor.org/rfc/rfc7540#section-6.8[HTTP GOAWAY], we recommend that you update your client to handle `GOAWAY` by reconnecting on connection termination. - -For the complete [.noloc]`Kubernetes` `1.26` changelog, see https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.26.md#changelog-since-v1250. - -[[kubernetes-1.25,kubernetes-1.25.title]] -== [.noloc]`Kubernetes` 1.25 - -[.noloc]`Kubernetes` `1.25` is now available in Amazon EKS. For more information about [.noloc]`Kubernetes` `1.25`, see the https://kubernetes.io/blog/2022/08/23/kubernetes-v1-25-release/[official release announcement]. - -[IMPORTANT] -==== - - -* Amazon EC2 `P2` instances aren't supported on Amazon EKS because they require `NVIDIA` driver version 470 or earlier. -* `PodSecurityPolicy` ([.noloc]`PSP`) is removed in [.noloc]`Kubernetes` `1.25`. [.noloc]`PSPs` are replaced with https://kubernetes.io/docs/concepts/security/pod-security-admission/[Pod Security Admission (PSA)] and Pod Security Standards [.noloc]`(PSS)`. [.noloc]`PSA` is a built-in admission controller that implements the security controls outlined in the https://kubernetes.io/docs/concepts/security/pod-security-standards/[PSS]. [.noloc]`PSA` and [.noloc]`PSS` are graduated to stable in [.noloc]`Kubernetes` `1.25` and are enabled in Amazon EKS by default. If you have [.noloc]`PSPs` in your cluster, make sure to migrate from [.noloc]`PSP` to the built-in [.noloc]`Kubernetes` [.noloc]`PSS` or to a policy-as-code solution before upgrading your cluster to version `1.25`. If you don't migrate from PSP, you might encounter interruptions to your workloads. For more information, see the xref:pod-security-policy-removal-faq[Migrate from legacy pod security policies (PSP),linkend=pod-security-policy-removal-faq]. -* [.noloc]`Kubernetes` version `1.25` contains changes that alter the behavior of an existing feature known as API Priority and Fairness (APF). APF serves to shield the API server from potential overload during periods of heightened request volumes. It does this by placing restrictions on the number of concurrent requests that can be processed at any given time. This is achieved through the application of distinct priority levels and limits to requests originating from various workloads or users. This approach ensures that critical applications or high-priority requests receive preferential treatment, while simultaneously preventing lower priority requests from overwhelming the API server. For more information, see https://kubernetes.io/docs/concepts/cluster-administration/flow-control/[API Priority and Fairness] in the [.noloc]`Kubernetes` documentation or https://aws.github.io/aws-eks-best-practices/scalability/docs/control-plane/#api-priority-and-fairness[API Priority and Fairness] in the EKS Best Practices Guide. -+ -These updates were introduced in https://github.com/kubernetes/kubernetes/pull/103521[PR #10352] and https://github.com/kubernetes/kubernetes/pull/118601[PR #118601]. Previously, APF treated all types of requests uniformly, with each request consuming a single unit of the concurrent request limit. The APF behavior change assigns higher units of concurrency to `LIST` requests due to the exceptionally heavy burden put on the API server by these requests. The API server estimates the number of objects that will be returned by a `LIST` request. It assigns a unit of concurrency that is proportional to the number of objects returned. -+ -Upon upgrading to Amazon EKS version `1.25` or higher, this updated behavior might cause workloads with heavy `LIST` requests (that previously functioned without issue) to encounter rate limiting. This would be indicated by an HTTP 429 response code. To avoid potential workload disruption due to `LIST` requests being rate limited, we strongly encourage you to restructure your workloads to reduce the rate of these requests. Alternatively, you can address this issue by adjusting the APF settings to allocate more capacity for essential requests while reducing the capacity allocated to non-essential ones. For more information about these mitigation techniques, see https://aws.github.io/aws-eks-best-practices/scalability/docs/control-plane/#preventing-dropped-requests[Preventing Dropped Requests] in the EKS Best Practices Guide. -* Amazon EKS `1.25` includes enhancements to cluster authentication that contain updated [.noloc]`YAML` libraries. If a [.noloc]`YAML` value in the `aws-auth` `ConfigMap` found in the `kube-system` namespace starts with a macro, where the first character is a curly brace, you should add quotation marks (`" "`) before and after the curly braces (`{ }`). This is required to ensure that `aws-iam-authenticator` version `v0.6.3` accurately parses the `aws-auth` `ConfigMap` in Amazon EKS `1.25`. -* The beta API version (`discovery.k8s.io/v1beta1`) of `EndpointSlice` was deprecated in [.noloc]`Kubernetes` `1.21` and is no longer served as of [.noloc]`Kubernetes` `1.25`. This API has been updated to `discovery.k8s.io/v1`. For more information, see https://kubernetes.io/docs/reference/using-api/deprecation-guide/#endpointslice-v125[EndpointSlice] in the [.noloc]`Kubernetes` documentation. The [.noloc]`{aws} Load Balancer Controller` `v2.4.6` and earlier used the `v1beta1` endpoint to communicate with `EndpointSlices`. If you're using the `EndpointSlices` configuration for the [.noloc]`{aws} Load Balancer Controller`, you must upgrade to [.noloc]`{aws} Load Balancer Controller` `v2.4.7` _before_ upgrading your Amazon EKS cluster to `1.25`. If you upgrade to `1.25` while using the `EndpointSlices` configuration for the [.noloc]`{aws} Load Balancer Controller`, the controller will crash and result in interruptions to your workloads. To upgrade the controller, see <>. -* The beta API version (`autoscaling/v2beta1`) of HorizontalPodAutoscaler is no longer served as of Kubernetes `1.25`. This API was deprecated in version `1.23`. Migrate manifests and API clients to use the `autoscaling/v2` HorizontalPodAutoscaler API version. For more information, see https://kubernetes.io/docs/reference/using-api/deprecation-guide/#horizontalpodautoscaler-v125[the Kubernetes documentation]. - -==== - -* `SeccompDefault` is promoted to beta in [.noloc]`Kubernetes` `1.25`. By setting the `--seccomp-default` flag when you configure `kubelet`, the container runtime uses its `RuntimeDefaultseccomp` profile, rather than the unconfined (`seccomp disabled`) mode. The default profiles provide a strong set of security defaults, while preserving the functionality of the workload. Although this flag is available, Amazon EKS doesn't enable this flag by default, so Amazon EKS behavior is effectively unchanged. If you want to, you can start enabling this on your nodes. For more details, see the tutorial https://kubernetes.io/docs/tutorials/security/seccomp/#enable-the-use-of-runtimedefault-as-the-default-seccomp-profile-for-all-workloads/[Restrict a Container's Syscalls with seccomp] in the [.noloc]`Kubernetes` documentation. -* Support for the Container Runtime Interface (CRI) for [.noloc]`Docker` (also known as [.noloc]`dockershim`) was removed from [.noloc]`Kubernetes` `1.24` and later. The only container runtime in Amazon EKS official [.noloc]`AMIs` for [.noloc]`Kubernetes` `1.24` and later clusters is [.noloc]`containerd`. Before upgrading to Amazon EKS `1.24` or later, remove any reference to bootstrap script flags that aren't supported anymore. For more information, see <>. -* The support for wildcard queries was deprecated in [.noloc]`CoreDNS` `1.8.7` and removed in [.noloc]`CoreDNS` `1.9`. This was done as a security measure. Wildcard queries no longer work and return [.noloc]`NXDOMAIN` instead of an IP address. -* The https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/[goaway-chance] option in the [.noloc]`Kubernetes` API server helps prevent `HTTP/2` client connections from being stuck on a single API server instance, by randomly closing a connection. When the connection is closed, the client will try to reconnect, and will likely land on a different API server as a result of load balancing. Amazon EKS version `1.25` has enabled `goaway-chance` flag. If your workload running on Amazon EKS cluster uses a client that is not compatible with https://www.rfc-editor.org/rfc/rfc7540#section-6.8[HTTP GOAWAY], we recommend that you update your client to handle `GOAWAY` by reconnecting on connection termination. - -For the complete [.noloc]`Kubernetes` `1.25` changelog, see https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.25.md#changelog-since-v1240. - -[[kubernetes-1.24,kubernetes-1.24.title]] -== [.noloc]`Kubernetes` 1.24 - -[.noloc]`Kubernetes` `1.24` is now available in Amazon EKS. For more information about [.noloc]`Kubernetes` `1.24`, see the https://kubernetes.io/blog/2022/05/03/kubernetes-1-24-release-announcement/[official release announcement]. - -[IMPORTANT] -==== - - -* Starting with [.noloc]`Kubernetes` `1.24`, new beta APIs aren't enabled in clusters by default. By default, existing beta APIs and new versions of existing beta APIs continue to be enabled. Amazon EKS follows the same behavior as upstream [.noloc]`Kubernetes` `1.24`. The feature gates that control new features for both new and existing API operations are enabled by default. This is in alignment with upstream [.noloc]`Kubernetes`. For more information, see https://github.com/kubernetes/enhancements/blob/master/keps/sig-architecture/3136-beta-apis-off-by-default/README.md[KEP-3136: Beta APIs Are Off by Default] on GitHub. -* Support for Container Runtime Interface (CRI) for [.noloc]`Docker` (also known as `dockershim`) is removed from [.noloc]`Kubernetes` `1.24`. Amazon EKS official AMIs have [.noloc]`containerd` as the only runtime. Before moving to Amazon EKS `1.24` or higher, you must remove any reference to bootstrap script flags that aren't supported anymore. You must also make sure that IP forwarding is enabled for your worker nodes. For more information, see <>. -* If you already have [.noloc]`Fluentd` configured for [.noloc]`Container Insights`, then you must migrate [.noloc]`Fluentd` to [.noloc]`Fluent Bit` before updating your cluster. The [.noloc]`Fluentd` parsers are configured to only parse log messages in JSON format. Unlike `dockerd`, the `containerd` container runtime has log messages that aren't in JSON format. If you don't migrate to [.noloc]`Fluent Bit`, some of the configured [.noloc]`Fluentd's` parsers will generate a massive amount of errors inside the [.noloc]`Fluentd` container. For more information on migrating, see link:AmazonCloudWatch/latest/monitoring/Container-Insights-setup-logs-FluentBit.html[Set up Fluent Bit as a DaemonSet to send logs to CloudWatch Logs,type="documentation"]. -* In [.noloc]`Kubernetes` `1.23` and earlier, `kubelet` serving certificates with unverifiable IP and DNS Subject Alternative Names (SANs) are automatically issued with unverifiable SANs. These unverifiable SANs are omitted from the provisioned certificate. In version `1.24` and later clusters, `kubelet` serving certificates aren't issued if any SAN can't be verified. This prevents `kubectl` exec and `kubectl` logs commands from working. For more information, see <>. -* When upgrading an Amazon EKS `1.23` cluster that uses [.noloc]`Fluent Bit`, you must make sure that it's running `k8s/1.3.12` or later. You can do this by reapplying the latest applicable [.noloc]`Fluent Bit` YAML file from [.noloc]`GitHub`. For more information, see link:AmazonCloudWatch/latest/monitoring/Container-Insights-setup-logs-FluentBit.html#Container-Insights-FluentBit-setup[Setting up Fluent Bit,type="documentation"] in the _Amazon CloudWatch User Guide_. - -==== - -* You can use Topology Aware Hints to indicate your preference for keeping traffic in zone when cluster worker nodes are deployed across multiple availability zones. Routing traffic within a zone can help reduce costs and improve network performance. By default, Topology Aware Hints are enabled in Amazon EKS `1.24`. For more information, see https://kubernetes.io/docs/concepts/services-networking/topology-aware-hints/[Topology Aware Hints] in the [.noloc]`Kubernetes` documentation. -* The `PodSecurityPolicy` ([.noloc]`PSP`) is scheduled for removal in [.noloc]`Kubernetes` `1.25`. [.noloc]`PSPs` are being replaced with https://kubernetes.io/docs/concepts/security/pod-security-admission/[Pod Security Admission (PSA)]. PSA is a built-in admission controller that uses the security controls that are outlined in the https://kubernetes.io/docs/concepts/security/pod-security-standards/[Pod Security Standards (PSS)]. PSA and PSS are both beta features and are enabled in Amazon EKS by default. To address the removal of [.noloc]`PSP` in version `1.25`, we recommend that you implement PSS in Amazon EKS. For more information, see link:containers/implementing-pod-security-standards-in-amazon-eks[Implementing Pod Security Standards in Amazon EKS,type="blog"] on the {aws} blog. -* The `client.authentication.k8s.io/v1alpha1` ExecCredential is removed in [.noloc]`Kubernetes` `1.24`. The ExecCredential API was generally available in [.noloc]`Kubernetes` `1.22`. If you use a client-go credential plugin that relies on the `v1alpha1` API, contact the distributor of your plugin on how to migrate to the `v1` API. -* For [.noloc]`Kubernetes` `1.24`, we contributed a feature to the upstream Cluster Autoscaler project that simplifies scaling Amazon EKS managed node groups to and from zero nodes. Previously, for the Cluster Autoscaler to understand the resources, labels, and taints of a managed node group that was scaled to zero nodes, you needed to tag the underlying Amazon EC2 Auto Scaling group with the details of the nodes that it was responsible for. Now, when there are no running nodes in the managed node group, the Cluster Autoscaler calls the Amazon EKS `DescribeNodegroup` API operation. This API operation provides the information that the Cluster Autoscaler requires of the managed node group's resources, labels, and taints. This feature requires that you add the `eks:DescribeNodegroup` permission to the Cluster Autoscaler service account IAM policy. When the value of a Cluster Autoscaler tag on the Auto Scaling group powering an Amazon EKS managed node group conflicts with the node group itself, the Cluster Autoscaler prefers the value of the Auto Scaling group tag. This is so that you can override values as needed. For more information, see https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md[Cluster Autoscaler]. -* If you intend to use [.noloc]`Inferentia` or [.noloc]`Trainium` instance types with Amazon EKS `1.24`, you must upgrade to the {aws} [.noloc]`Neuron` device plugin version 1.9.3.0 or later. For more information, see https://awsdocs-neuron.readthedocs-hosted.com/en/latest/release-notes/containers/neuron-k8.html#id46[Neuron K8 release [1.9.3.0]] in the {aws} [.noloc]`Neuron` Documentation. -* `Containerd` has `IPv6` enabled for [.noloc]`Pods`, by default. It applies node kernel settings to [.noloc]`Pod` network namespaces. Because of this, containers in a [.noloc]`Pod` bind to both `IPv4` (`127.0.0.1`) and `IPv6` (`::1`) loopback addresses. `IPv6` is the default protocol for communication. Before updating your cluster to version `1.24`, we recommend that you test your multi-container [.noloc]`Pods`. Modify apps so that they can bind to all IP addresses on loopback interfaces. The majority of libraries enable `IPv6` binding, which is backward compatible with `IPv4`. When it's not possible to modify your application code, you have two options: -+ -** Run an `init` container and set `disable ipv6` to `true` (`sysctl -w net.ipv6.conf.all.disable_ipv6=1`). -** Configure a https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#mutatingadmissionwebhook[mutating admission webhook] to inject an `init` container alongside your application [.noloc]`Pods`. - -+ -If you need to block `IPv6` for all [.noloc]`Pods` across all nodes, you might have to disable `IPv6` on your instances. -* The https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/[goaway-chance] option in the [.noloc]`Kubernetes` API server helps prevent `HTTP/2` client connections from being stuck on a single API server instance, by randomly closing a connection. When the connection is closed, the client will try to reconnect, and will likely land on a different API server as a result of load balancing. Amazon EKS version `1.24` has enabled `goaway-chance` flag. If your workload running on Amazon EKS cluster uses a client that is not compatible with https://www.rfc-editor.org/rfc/rfc7540#section-6.8[HTTP GOAWAY], we recommend that you update your client to handle `GOAWAY` by reconnecting on connection termination. - -For the complete [.noloc]`Kubernetes` `1.24` changelog, see https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.24.md#changelog-since-v1230. - -[[kubernetes-1.23,kubernetes-1.23.title]] -== [.noloc]`Kubernetes` 1.23 - -[.noloc]`Kubernetes` `1.23` is now available in Amazon EKS. For more information about [.noloc]`Kubernetes` `1.23`, see the https://kubernetes.io/blog/2021/12/07/kubernetes-1-23-release-announcement/[official release announcement]. - -[IMPORTANT] -==== - - -* The [.noloc]`Kubernetes` in-tree to container storage interface (CSI) volume migration feature is enabled. This feature enables the replacement of existing [.noloc]`Kubernetes` in-tree storage plugins for Amazon EBS with a corresponding Amazon EBS CSI driver. For more information, see https://kubernetes.io/blog/2019/12/09/kubernetes-1-17-feature-csi-migration-beta/[Kubernetes 1.17 Feature: Kubernetes In-Tree to CSI Volume Migration Moves to Beta] on the [.noloc]`Kubernetes` blog. -+ -The feature translates in-tree APIs to equivalent CSI APIs and delegates operations to a replacement CSI driver. With this feature, if you use existing `StorageClass`, `PersistentVolume`, and `PersistentVolumeClaim` objects that belong to these workloads, there likely won't be any noticeable change. The feature enables [.noloc]`Kubernetes` to delegate all storage management operations from the in-tree plugin to the CSI driver. If you use Amazon EBS volumes in an existing cluster, install the Amazon EBS CSI driver in your cluster before you update your cluster to version `1.23`. If you don't install the driver before updating an existing cluster, interruptions to your workloads might occur. If you plan to deploy workloads that use Amazon EBS volumes in a new `1.23` cluster, install the Amazon EBS CSI driver in your cluster before deploying the workloads your cluster. For instructions on how to install the Amazon EBS CSI driver on your cluster, see <>. For frequently asked questions about the migration feature, see <>. -* Extended Support for Amazon EKS optimized [.noloc]`Windows` AMIs that are published by {aws} isn't available for [.noloc]`Kubernetes` version `1.23` but is available for [.noloc]`Kubernetes` version `1.24` and higher. - -==== - -* [.noloc]`Kubernetes` stopped supporting `dockershim` in version `1.20` and removed `dockershim` in version `1.24`. For more information, see https://kubernetes.io/blog/2022/01/07/kubernetes-is-moving-on-from-dockershim/[Kubernetes is Moving on From Dockershim: Commitments and Next Steps] in the [.noloc]`Kubernetes` blog. Amazon EKS will end support for `dockershim` starting in Amazon EKS version `1.24`. Starting with Amazon EKS version `1.24`, Amazon EKS official AMIs will have `containerd` as the only runtime. -+ -Even though Amazon EKS version `1.23` continues to support `dockershim`, we recommend that you start testing your applications now to identify and remove any [.noloc]`Docker` dependencies. This way, you are prepared to update your cluster to version `1.24`. For more information about `dockershim` removal, see <>. -* [.noloc]`Kubernetes` graduated `IPv4`/``IPv6`` dual-stack networking for [.noloc]`Pods`, services, and nodes to general availability. However, Amazon EKS and the [.noloc]`Amazon VPC CNI plugin for Kubernetes` don't support dual-stack networking. Your clusters can assign `IPv4` or `IPv6` addresses to [.noloc]`Pods` and services, but can't assign both address types. -* [.noloc]`Kubernetes` graduated the Pod Security Admission (PSA) feature to beta. The feature is enabled by default. For more information, see https://kubernetes.io/docs/concepts/security/pod-security-admission/[Pod Security Admission] in the [.noloc]`Kubernetes` documentation. PSA replaces the https://aws.github.io/aws-eks-best-practices/security/docs/pods/#pod-security-solutions[Pod Security Policy] ([.noloc]`PSP`) admission controller. The PSP admission controller isn't supported and is scheduled for removal in [.noloc]`Kubernetes` version `1.25`. -+ -The [.noloc]`PSP` admission controller enforces [.noloc]`Pod` security standards on [.noloc]`Pods` in a namespace based on specific namespace labels that set the enforcement level. For more information, see https://aws.github.io/aws-eks-best-practices/security/docs/pods/#pod-security-standards-pss-and-pod-security-admission-psa[Pod Security Standards (PSS) and Pod Security Admission (PSA)] in the Amazon EKS best practices guide. -* The `kube-proxy` image deployed with clusters is now the https://gallery.ecr.aws/eks-distro-build-tooling/eks-distro-minimal-base-iptables[minimal base image] maintained by Amazon EKS Distro (EKS-D). The image contains minimal packages and doesn't have shells or package managers. -* [.noloc]`Kubernetes` graduated ephemeral containers to beta. Ephemeral containers are temporary containers that run in the same namespace as an existing [.noloc]`Pod`. You can use them to observe the state of [.noloc]`Pods` and containers for troubleshooting and debugging purposes. This is especially useful for interactive troubleshooting when `kubectl exec` is insufficient because either a container has crashed or a container image doesn't include debugging utilities. An example of a container that includes a debugging utility is https://github.com/GoogleContainerTools/distroless#distroless-container-images[distroless images]. For more information, see https://kubernetes.io/docs/tasks/debug/debug-application/debug-running-pod/#ephemeral-container[Debugging with an ephemeral debug container] in the [.noloc]`Kubernetes` documentation. -* [.noloc]`Kubernetes` graduated the `HorizontalPodAutoscaler` `autoscaling/v2` stable API to general availability. The `HorizontalPodAutoscaler` `autoscaling/v2beta2` API is deprecated. It will be unavailable in `1.26`. -* The https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/[goaway-chance] option in the [.noloc]`Kubernetes` API server helps prevent `HTTP/2` client connections from being stuck on a single API server instance, by randomly closing a connection. When the connection is closed, the client will try to reconnect, and will likely land on a different API server as a result of load balancing. Amazon EKS version `1.23` has enabled `goaway-chance` flag. If your workload running on Amazon EKS cluster uses a client that is not compatible with https://www.rfc-editor.org/rfc/rfc7540#section-6.8[HTTP GOAWAY], we recommend that you update your client to handle `GOAWAY` by reconnecting on connection termination. - -For the complete [.noloc]`Kubernetes` `1.23` changelog, see https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.23.md#changelog-since-v1220. diff --git a/latest/ug/clusters/kubernetes-versions-standard.adoc b/latest/ug/clusters/kubernetes-versions-standard.adoc deleted file mode 100644 index ad1034fce..000000000 --- a/latest/ug/clusters/kubernetes-versions-standard.adoc +++ /dev/null @@ -1,88 +0,0 @@ -//!!NODE_ROOT
- -[.topic] -[[kubernetes-versions-standard,kubernetes-versions-standard.title]] -= Review release notes for [.noloc]`Kubernetes` versions on standard support -:info_titleabbrev: Standard support versions - -include::../attributes.txt[] - -[abstract] --- -This topic gives important changes to be aware of for each [.noloc]`Kubernetes` version in standard support. --- - -This topic gives important changes to be aware of for each [.noloc]`Kubernetes` version in standard support. When upgrading, carefully review the changes that have occurred between the old and new versions for your cluster. - -[NOTE] -==== - -For `1.24` and later clusters, officially published Amazon EKS AMIs include `containerd` as the only runtime. [.noloc]`Kubernetes` versions earlier than `1.24` use [.noloc]`Docker` as the default runtime. These versions have a bootstrap flag option that you can use to test out your workloads on any supported cluster with `containerd`. For more information, see <>. - -==== - - -[[kubernetes-1.31,kubernetes-1.31.title]] -== [.noloc]`Kubernetes` 1.31 - -[.noloc]`Kubernetes` `1.31` is now available in Amazon EKS. For more information about [.noloc]`Kubernetes` `1.31`, see the https://kubernetes.io/blog/2024/08/13/kubernetes-v1-31-release/[official release announcement]. - -[IMPORTANT] -==== - - -* The kubelet flag `--keep-terminated-pod-volumes` deprecated since 2017 has been removed as part of the `v1.31` release. This change impacts how terminated pod volumes are handled by the kubelet. If you are using this flag in your node configurations, you must update your bootstrap scripts and launch templates to remove it before upgrading. - -==== - -* The beta `VolumeAttributesClass` feature gate and API resource is enabled in Amazon EKS `v1.31`. This feature allows cluster operators to modify mutable properties of Persistent Volumes (PVs) managed by compatible CSI Drivers, including the Amazon EBS CSI Driver. To leverage this feature, ensure that your CSI Driver supports the `VolumeAttributesClass` feature (for the Amazon EBS CSI Driver, upgrade to version `v1.35.0` or later to automatically enable the feature). You will be able to create `VolumeAttributesClass` objects to define the desired volume attributes, such as volume type and throughput, and associate them with your Persistent Volume Claims (PVCs). See the https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/[official Kubernetes documentation] as well as the documentation of your CSI driver for more information. -** For more information about the Amazon EBS CSI Driver, see <>. -* Kubernetes support for https://apparmor.net/[AppArmor] has graduated to stable and is now generally available for public use. This feature allows you to protect your containers with AppArmor by setting the `appArmorProfile.type` field in the container's `securityContext`. Prior to Kubernetes `v1.30`, AppArmor was controlled by annotations. Starting with `v1.30`, it is controlled using fields. To leverage this feature, we recommend migrating away from annotations and using the `appArmorProfile.type` field to ensure that your workloads are compatible. -* The PersistentVolume last phase transition time feature has graduated to stable and is now generally available for public use in Kubernetes `v1.31`. This feature introduces a new field, `.status.lastTransitionTime`, in the PersistentVolumeStatus, which provides a timestamp of when a PersistentVolume last transitioned to a different phase. This enhancement allows for better tracking and management of PersistentVolumes, particularly in scenarios where understanding the lifecycle of volumes is important. - -For the complete [.noloc]`Kubernetes` `1.31` changelog, see https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.31.md - -[[kubernetes-1.30,kubernetes-1.30.title]] -== [.noloc]`Kubernetes` 1.30 - -[.noloc]`Kubernetes` `1.30` is now available in Amazon EKS. For more information about [.noloc]`Kubernetes` `1.30`, see the https://kubernetes.io/blog/2024/04/17/kubernetes-v1-30-release/[official release announcement]. - -[IMPORTANT] -==== - - -* Starting with Amazon EKS version `1.30` or newer, any newly created managed node groups will automatically default to using Amazon Linux 2023 (AL2023) as the node operating system. Previously, new node groups would default to Amazon Linux 2 (AL2). You can continue to use AL2 by choosing it as the AMI type when creating a new node group. -+ -** For more information about Amazon Linux, see link:linux/al2023/ug/compare-with-al2.html[Comparing AL2 and AL2023,type="documentation"] in the Amazon Linux User Guide. -** For more information about specifiying the operating system for a managed node group, see <>. - -==== - -* With Amazon EKS `1.30`, the `topology.k8s.aws/zone-id` label is added to worker nodes. You can use Availability Zone IDs (AZ IDs) to determine the location of resources in one account relative to the resources in another account. For more information, see link:ram/latest/userguide/working-with-az-ids.html[Availability Zone IDs for your {aws} resources,type="documentation"] in the _{aws} RAM User Guide_. -* Starting with `1.30`, Amazon EKS no longer includes the `default` annotation on the `gp2 StorageClass` resource applied to newly created clusters. This has no impact if you are referencing this storage class by name. You must take action if you were relying on having a default `StorageClass` in the cluster. You should reference the `StorageClass` by the name `gp2`. Alternatively, you can deploy the Amazon EBS recommended default storage class by setting the `defaultStorageClass.enabled` parameter to true when installing `v1.31.0` or later of the `aws-ebs-csi-driver add-on`. -* The minimum required IAM policy for the Amazon EKS cluster IAM role has changed. The action `ec2:DescribeAvailabilityZones` is required. For more information, see <>. - -For the complete [.noloc]`Kubernetes` `1.30` changelog, see https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.30.md. - -[[kubernetes-1.29,kubernetes-1.29.title]] -== [.noloc]`Kubernetes` 1.29 - -[.noloc]`Kubernetes` `1.29` is now available in Amazon EKS. For more information about [.noloc]`Kubernetes` `1.29`, see the https://kubernetes.io/blog/2023/12/13/kubernetes-v1-29-release/[official release announcement]. - -[IMPORTANT] -==== - - -* The deprecated `flowcontrol.apiserver.k8s.io/v1beta2` API version of `FlowSchema` and `PriorityLevelConfiguration` are no longer served in [.noloc]`Kubernetes` `v1.29`. If you have manifests or client software that uses the deprecated beta API group, you should change these before you upgrade to `v1.29`. - -==== - -* The `.status.kubeProxyVersion` field for node objects is now deprecated, and the [.noloc]`Kubernetes` project is proposing to remove that field in a future release. The deprecated field is not accurate and has historically been managed by `kubelet` - which does not actually know the `kube-proxy` version, or even whether `kube-proxy` is running. If you've been using this field in client software, stop - the information isn't reliable and the field is now deprecated. -* In [.noloc]`Kubernetes` `1.29` to reduce potential attack surface, the `LegacyServiceAccountTokenCleanUp` feature labels legacy auto-generated secret-based tokens as invalid if they have not been used for a long time (1 year by default), and automatically removes them if use is not attempted for a long time after being marked as invalid (1 additional year by default). To identify such tokens, a you can run: -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl get cm kube-apiserver-legacy-service-account-token-tracking -n kube-system ----- - -For the complete [.noloc]`Kubernetes` `1.29` changelog, see https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.29.md#changelog-since-v1280. diff --git a/latest/ug/clusters/kubernetes-versions.adoc b/latest/ug/clusters/kubernetes-versions.adoc deleted file mode 100644 index fdf3e63ea..000000000 --- a/latest/ug/clusters/kubernetes-versions.adoc +++ /dev/null @@ -1,276 +0,0 @@ -//!!NODE_ROOT
- -[.topic] -[[kubernetes-versions,kubernetes-versions.title]] -= Understand the [.noloc]`Kubernetes` version lifecycle on EKS -:info_doctype: section -:info_title: Understand the Kubernetes version lifecycle on EKS -:info_titleabbrev: Kubernetes versions -:keywords: Amazon EKS, available, Kubernetes, version, release notes -:info_abstract: Learn how Amazon EKS supports Kubernetes versions with standard and extended \ - support periods, allowing you to proactively update clusters with the latest \ - versions, features, and security patches.. - - -include::../attributes.txt[] - -[abstract] --- -Learn how Amazon EKS supports Kubernetes versions with standard and extended support periods, allowing you to proactively update clusters with the latest versions, features, and security patches.. --- - -[.noloc]`Kubernetes` rapidly evolves with new features, design updates, and bug fixes. The community releases new [.noloc]`Kubernetes` minor versions (such as `1.30`) on average once every four months. Amazon EKS follows the upstream release and deprecation cycle for minor versions. As new [.noloc]`Kubernetes` versions become available in Amazon EKS, we recommend that you proactively update your clusters to use the latest available version. - -A minor version is under standard support in Amazon EKS for the first 14 months after it's released. Once a version is past the end of standard support date, it enters extended support for the next 12 months. Extended support allows you to stay at a specific [.noloc]`Kubernetes` version for longer at an additional cost per cluster hour. If you haven't updated your cluster before the extended support period ends, your cluster is auto-upgraded to the oldest currently supported extended version. - -Extended support is enabled by default. <> - -We recommend that you create your cluster with the latest available [.noloc]`Kubernetes` version supported by Amazon EKS. If your application requires a specific version of [.noloc]`Kubernetes`, you can select older versions. You can create new Amazon EKS clusters on any version offered in standard or extended support. - - - -video::_dJdAZ_J_jw[youtube,align = center,height = 405,fileref = https://www.youtube.com/embed/_dJdAZ_J_jw,width = 720] - - -[[available-versions,available-versions.title]] -== Available versions on standard support - -The following [.noloc]`Kubernetes` versions are currently available in Amazon EKS standard support: - - -* `1.31` -* `1.30` -* `1.29` - -For important changes to be aware of for each version in standard support, see <>. - -[[available-versions-extended,available-versions-extended.title]] -== Available versions on extended support - -The following [.noloc]`Kubernetes` versions are currently available in Amazon EKS extended support: - - -* `1.28` -* `1.27` -* `1.26` -* `1.25` -* `1.24` - -For important changes to be aware of for each version in extended support, see <>. - -[[kubernetes-release-calendar,kubernetes-release-calendar.title]] -== Amazon EKS [.noloc]`Kubernetes` release calendar - -The following table shows important release and support dates to consider for each [.noloc]`Kubernetes` version. Billing for extended support starts at the beginning of the day that the version reaches end of standard support. - -[NOTE] -==== - -Dates with only a month and a year are approximate and are updated with an exact date when it's known. - -==== - -[cols="1,1,1,1,1", options="header"] -|=== -|Kubernetes version -|Upstream release -|Amazon EKS release -|End of standard support -|End of extended support - - -|`1.31` -|August 13, 2024 -|September 26, 2024 -|November 26, 2025 -|November 26, 2026 - -|`1.30` -|April 17, 2024 -|May 23, 2024 -|July 23, 2025 -|July 23, 2026 - -|`1.29` -|December 13, 2023 -|January 23, 2024 -|March 23, 2025 -|March 23, 2026 - -|`1.28` -|August 15, 2023 -|September 26, 2023 -|November 26, 2024 -|November 26, 2025 - -|`1.27` -|April 11, 2023 -|May 24, 2023 -|July 24, 2024 -|July 24, 2025 - -|`1.26` -|December 9, 2022 -|April 11, 2023 -|June 11, 2024 -|June 11, 2025 - -|`1.25` -|August 23, 2022 -|February 22, 2023 -|May 1, 2024 -|May 1, 2025 - -|`1.24` -|May 3, 2022 -|November 15, 2022 -|January 31, 2024 -|January 31, 2025 - -|=== - -[[version-cli,version-cli.title]] -== Get version information with {AWS} CLI - -You can use the {aws} CLI to get information about Kubernetes versions available on EKS, such as the end date of Standard Support. - -=== To retrieve information about available Kubernetes versions on EKS using the {aws} CLI - -. Open your terminal. -. Ensure you have the {aws} CLI installed and configured. For more information, see link:cli/latest/userguide/getting-started-install.html["Installing or updating to the latest version of the CLI",type="documentation"]. -. Run the following command: -+ -``` -aws eks describe-cluster-versions -``` -. The command will return a JSON output with details about the available cluster versions. Here's an example of the output: -+ -```json -{ - "clusterVersions": [ - { - "clusterVersion": "1.31", - "clusterType": "eks", - "defaultPlatformVersion": "eks.21", - "defaultVersion": true, - "releaseDate": "2024-09-25T17:00:00-07:00", - "endOfStandardSupportDate": "2025-11-25T16:00:00-08:00", - "endOfExtendedSupportDate": "2026-11-25T16:00:00-08:00", - "status": "STANDARD_SUPPORT", - "kubernetesPatchVersion": "1.31.3" - } - ] -} -``` - -*The output provides the following information for each cluster version:* - -* `clusterVersion`: The Kubernetes version of the EKS cluster -* `clusterType`: The type of cluster (e.g., "eks") -* `defaultPlatformVersion`: The default EKS platform version -* `defaultVersion`: Whether this is the default version -* `releaseDate`: The date when this version was released -* `endOfStandardSupportDate`: The date when standard support ends -* `endOfExtendedSupportDate`: The date when extended support ends -* `status`: The current support status of the version, such as `STANDARD_SUPPORT` or `EXTENDED_SUPPORT` -* `kubernetesPatchVersion`: The specific Kubernetes patch version - -[[version-faqs,version-faqs.title]] -== Amazon EKS version FAQs - -*How many [.noloc]`Kubernetes` versions are available in standard support?*:: -In line with the [.noloc]`Kubernetes` community support for [.noloc]`Kubernetes` versions, Amazon EKS is committed to offering support for three [.noloc]`Kubernetes` versions at any given time. We will announce the end of standard support date of a given [.noloc]`Kubernetes` minor version at least 60 days in advance. Because of the Amazon EKS qualification and release process for new [.noloc]`Kubernetes` versions, the end of standard support date of a [.noloc]`Kubernetes` version on Amazon EKS will be after the date that the [.noloc]`Kubernetes` project stops supporting the version upstream. - -*How long does a [.noloc]`Kubernetes` receive standard support by Amazon EKS?*:: -A [.noloc]`Kubernetes` version received standard support for 14 months after first being available on Amazon EKS. This is true even if upstream [.noloc]`Kubernetes` no longer support a version that's available on Amazon EKS. We backport security patches that are applicable to the [.noloc]`Kubernetes` versions that are supported on Amazon EKS. - - -*Am I notified when standard support is ending for a [.noloc]`Kubernetes` version on Amazon EKS?*:: -Yes. If any clusters in your account are running the version nearing the end of support, Amazon EKS sends out a notice through the {aws} Health Dashboard approximately 12 months after the [.noloc]`Kubernetes` version was released on Amazon EKS. The notice includes the end of support date. This is at least 60 days from the date of the notice. - - -*Which [.noloc]`Kubernetes` features are supported by Amazon EKS?*:: -Amazon EKS supports all generally available (GA) features of the [.noloc]`Kubernetes` API. Starting with [.noloc]`Kubernetes` version `1.24`, new beta APIs aren't enabled in clusters by default. However, previously existing beta APIs and new versions of existing beta APIs continue to be enabled by default. Alpha features aren't supported. - - -*Are Amazon EKS managed node groups automatically updated along with the cluster control plane version?*:: -No. A managed node group creates Amazon EC2 instances in your account. These instances aren't automatically upgraded when you or Amazon EKS update your control plane. For more information, see <>. We recommend maintaining the same [.noloc]`Kubernetes` version on your control plane and nodes. - - -*Are self-managed node groups automatically updated along with the cluster control plane version?*:: -No. A self-managed node group includes Amazon EC2 instances in your account. These instances aren't automatically upgraded when you or Amazon EKS update the control plane version on your behalf. A self-managed node group doesn't have any indication in the console that it needs updating. You can view the `kubelet` version installed on a node by selecting the node in the *Nodes* list on the *Overview* tab of your cluster to determine which nodes need updating. You must manually update the nodes. For more information, see <>. -+ -The [.noloc]`Kubernetes` project tests compatibility between the control plane and nodes for up to three minor versions. For example, `1.27` nodes continue to operate when orchestrated by a `1.30` control plane. However, running a cluster with nodes that are persistently three minor versions behind the control plane isn't recommended. For more information, see https://kubernetes.io/docs/setup/version-skew-policy/[Kubernetes version and version skew support policy] in the [.noloc]`Kubernetes` documentation. We recommend maintaining the same [.noloc]`Kubernetes` version on your control plane and nodes. - - -*Are [.noloc]`Pods` running on Fargate automatically upgraded with an automatic cluster control plane version upgrade?*:: -No. We strongly recommend running Fargate [.noloc]`Pods` as part of a replication controller, such as a [.noloc]`Kubernetes` deployment. Then do a rolling restart of all Fargate [.noloc]`Pods`. The new version of the Fargate [.noloc]`Pod` is deployed with a `kubelet` version that's the same version as your updated cluster control plane version. For more information, see https://kubernetes.io/docs/concepts/workloads/controllers/deployment[Deployments] in the [.noloc]`Kubernetes` documentation. -+ -IMPORTANT: If you update the control plane, you must still update the Fargate nodes yourself. To update Fargate nodes, delete the Fargate [.noloc]`Pod` represented by the node and redeploy the [.noloc]`Pod`. The new [.noloc]`Pod` is deployed with a `kubelet` version that's the same version as your cluster. - - -*What Kubernetes versions are supported for hybrid nodes?*:: -Amazon EKS Hybrid Nodes supports the same Kubernetes versions as Amazon EKS clusters with other node compute types, including standard and extended Kubernetes version support. Hybrid nodes are not automatically upgraded when you upgrade your control plane version and you are responsible for upgrading your hybrid nodes. For more information, see <>. - - -[[extended-support-faqs,extended-support-faqs.title]] -== Amazon EKS extended support FAQs - -*The standard support and extended support terminology is new to me. What do those terms mean?*:: -Standard support for a [.noloc]`Kubernetes` version in Amazon EKS begins when a [.noloc]`Kubernetes` version is released on Amazon EKS, and will end 14 months after the release date. Extended support for a [.noloc]`Kubernetes` version will begin immediately after the end of standard support, and will end after the next 12 months. For example, standard support for version `1.23` in Amazon EKS ended on October 11, 2023. Extended support for version `1.23` began on October 12, 2023 and ended on October 11, 2024. - - -*What do I need to do to get extended support for Amazon EKS clusters?*:: -You will need to enable extended support (see <>) for your cluster by changing the cluster upgrade policy to EXTENDED. By default, for all new and existing clusters, the upgrade policy is set to EXTENDED, unless specified otherwise. See <> to view the upgrade policy for your cluster. Standard support will begin when a [.noloc]`Kubernetes` version is released on Amazon EKS, and will end 14 months after the release date. Extended support for a [.noloc]`Kubernetes` version will begin immediately after the end of standard support, and will end after the next 12 months. - - -*For which [.noloc]`Kubernetes` versions can I get extended support?*:: -Extended support is available for [.noloc]`Kubernetes` versions `1.23` and higher. You can run clusters on any version for up to 12 months after the end of standard support for that version. This means that each version will be supported for 26 months in Amazon EKS (14 months of standard support plus 12 months of extended support). - - -*What if I don't want to use extended support?*:: -If you don't want to be automatically enrolled in extended support, you can upgrade your cluster to a [.noloc]`Kubernetes` version that's in standard Amazon EKS support. See <> to learn how to disable extended support. Note: If you disable extended support, your cluster will be auto upgraded at the end of standard support. - - -*What will happen at the end of 12 months of extended support?*:: -Clusters running on a [.noloc]`Kubernetes` version that has completed its 26-month lifecycle (14 months of standard support plus 12 months of extended support) will be auto-upgraded to the next version. The auto-upgrade includes only the Kubernetes control plane. If you have EKS Auto Mode nodes, they may automatically update. Self managed nodes and EKS Managed Node Groups will remain on the previous version. -+ -On the end of extended support date, you can no longer create new Amazon EKS clusters with the unsupported version. Existing control planes are automatically updated by Amazon EKS to the earliest supported version through a gradual deployment process after the end of support date. After the automatic control plane update, make sure to manually update cluster add-ons and Amazon EC2 nodes. For more information, see <>. - - -*When exactly is my control plane automatically updated after the end of extended support date?*:: -Amazon EKS can't provide specific time frames. Automatic updates can happen at any time after the end of extended support date. You won't receive any notification before the update. We recommend that you proactively update your control plane without relying on the Amazon EKS automatic update process. For more information, see <>. - - -*Can I leave my control plane on a [.noloc]`Kubernetes` version indefinitely?*:: -No. Cloud security at {aws} is the highest priority. Past a certain point (usually one year), the [.noloc]`Kubernetes` community stops releasing common vulnerabilities and exposures ([.noloc]`CVE`) patches and discourages CVE submission for unsupported versions. This means that vulnerabilities specific to an older version of [.noloc]`Kubernetes` might not even be reported. This leaves clusters exposed with no notice and no remediation options in the event of a vulnerability. Given this, Amazon EKS doesn't allow control planes to stay on a version that reached end of extended support. - - -*Is there additional cost to get extended support?*:: -Yes, there is additional cost for Amazon EKS clusters running in extended support. For pricing details, see link:containers/amazon-eks-extended-support-for-kubernetes-versions-pricing[Amazon EKS extended support for Kubernetes version pricing,type="blog"] on the {aws} blog or our https://aws.amazon.com/eks/pricing/[pricing page]. - - -*What is included in extended support?*:: -Amazon EKS clusters in Extended Support receive ongoing security patches for the [.noloc]`Kubernetes` control plane. Additionally, Amazon EKS will release patches for the Amazon VPC CNI, `kube-proxy`, and [.noloc]`CoreDNS` add-ons for Extended Support versions. Amazon EKS will also release patches for {aws}-published Amazon EKS optimized AMIs for Amazon Linux, [.noloc]`Bottlerocket`, and Windows, as well as Amazon EKS Fargate nodes for those versions. All clusters in Extended Support will continue to get access to technical support from {aws}. -+ -NOTE: Extended Support for Amazon EKS optimized [.noloc]`Windows` AMIs that are published by {aws} isn't available for [.noloc]`Kubernetes` version `1.23` but is available for [.noloc]`Kubernetes` version `1.24` and higher. - - -*Are there any limitations to patches for non-[.noloc]`Kubernetes` components in extended support?*:: -While Extended Support covers all of the [.noloc]`Kubernetes` specific components from {aws}, it will only provide support for {aws}-published Amazon EKS optimized AMIs for Amazon Linux, [.noloc]`Bottlerocket`, and Windows at all times. This means, you will potentially have newer components (such as OS or kernel) on your Amazon EKS optimized AMI while using Extended Support. For example, once Amazon Linux 2 reaches the link:amazon-linux-2/faqs/[end of its lifecycle in 2025,type="marketing"], the Amazon EKS optimized Amazon Linux AMIs will be built using a newer Amazon Linux OS. Amazon EKS will announce and document important support lifecycle discrepancies such as this for each [.noloc]`Kubernetes` version. - - -*Can I create new clusters using a version on extended support?*:: -Yes. - -include::kubernetes-versions-standard.adoc[leveloffset=+1] - -include::kubernetes-versions-extended.adoc[leveloffset=+1] - -include::view-support-status.adoc[leveloffset=+1] - -include::view-upgrade-policy.adoc[leveloffset=+1] - -include::enable-extended-support.adoc[leveloffset=+1] - -include::disable-extended-support.adoc[leveloffset=+1] diff --git a/latest/ug/clusters/management/cost-monitoring-aws.adoc b/latest/ug/clusters/management/cost-monitoring-aws.adoc deleted file mode 100644 index 8c773eae2..000000000 --- a/latest/ug/clusters/management/cost-monitoring-aws.adoc +++ /dev/null @@ -1,29 +0,0 @@ -include::../../attributes.txt[] - -[.topic] -[[cost-monitoring-aws,cost-monitoring-aws.title]] -= View costs by pod in {aws} billing with split cost allocation -:info_doctype: section - -.Cost monitoring using {aws} split cost allocation data for Amazon EKS -You can use {aws} split cost allocation data for Amazon EKS to get granular cost visibility for your Amazon EKS clusters. This enables you to analyze, optimize, and chargeback cost and usage for your [.noloc]`Kubernetes` applications. You allocate application costs to individual business units and teams based on Amazon EC2 CPU and memory resources consumed by your [.noloc]`Kubernetes` application. Split cost allocation data for Amazon EKS gives visibility into cost per Pod, and enables you to aggregate the cost data per Pod using namespace, cluster, and other [.noloc]`Kubernetes` primitives. The following are examples of [.noloc]`Kubernetes` primitives that you can use to analyze Amazon EKS cost allocation data. - -* Cluster name -* Deployment -* Namespace -* Node -* Workload Name -* Workload Type - -For more information about using split cost allocation data, see link:cur/latest/userguide/split-cost-allocation-data.html[Understanding split cost allocation data,type="documentation"] in the {aws} Billing User Guide. - -[[task-cur-setup,task-cur-setup.title]] -== Set up Cost and Usage Reports - -You can turn on Split Cost Allocation Data for EKS in the Cost Management Console, {aws} Command Line Interface, or the {aws} SDKs. - -Use the following for _Split Cost Allocation Data_: - -. Opt in to Split Cost Allocation Data. For more information, see link:cur/latest/userguide/enabling-split-cost-allocation-data.html[Enabling split cost allocation data,type="documentation"] in the {aws} Cost and Usage Report User Guide. -. Include the data in a new or existing report. -. View the report. You can use the Billing and Cost Management console or view the report files in Amazon Simple Storage Service. diff --git a/latest/ug/clusters/management/cost-monitoring-kubecost-bundles.adoc b/latest/ug/clusters/management/cost-monitoring-kubecost-bundles.adoc deleted file mode 100644 index c3297daa1..000000000 --- a/latest/ug/clusters/management/cost-monitoring-kubecost-bundles.adoc +++ /dev/null @@ -1,330 +0,0 @@ -//!!NODE_ROOT
-[.topic] -[[cost-monitoring-kubecost-bundles,cost-monitoring-kubecost-bundles.title]] -= Learn more about Kubecost -:info_doctype: section - -include::../../attributes.txt[] - -Amazon EKS provides an {aws} optimized bundle of [.noloc]`Kubecost` for cluster cost visibility. Amazon EKS supports [.noloc]`Kubecost`, which you can use to monitor your costs broken down by [.noloc]`Kubernetes` resources including [.noloc]`Pods`, nodes, namespaces, and labels. - -This topic covers the available versions of [.noloc]`Kubecost`, and the differences between the available tiers. EKS supports [.noloc]`Kubecost` Version 1 and Version 2. Each version is available in different tiers. You can use _Amazon EKS optimized [.noloc]`Kubecost` custom bundle_ for your EKS clusters at no additional cost. You may be charged for use of associated {aws} services, such as Amazon Managed Service for Prometheus. Also, you can use your existing {aws} support agreements to obtain support. - -As a [.noloc]`Kubernetes` platform administrator and finance leader, you can use [.noloc]`Kubecost` to visualize a breakdown of Amazon EKS charges, allocate costs, and charge back organizational units such as application teams. You can provide your internal teams and business units with transparent and accurate cost data based on their actual {aws} bill. Moreover, you can also get customized recommendations for cost optimization based on their infrastructure environment and usage patterns within their clusters. For more information about [.noloc]`Kubecost`, see the https://guide.kubecost.com[Kubecost] documentation. - -*What is the difference between the custom bundle of [.noloc]`Kubecost` and the free version of [.noloc]`Kubecost` (also known as [.noloc]`OpenCost`)?* - -{aws} and [.noloc]`Kubecost` collaborated to offer a customized version of [.noloc]`Kubecost`. This version includes a subset of commercial features at no additional charge. See the tables below for features that are included with in the custom bundle of [.noloc]`Kubecost`. - -[[kubecost-v2,kubecost-v2.title]] -== Kubecost v2 - -*What is the difference between [.noloc]`Kubecost` v1 and v2?* - -Kubecost 2.0 is a major upgrade from previous versions and includes major new features including a brand new API Backend. Note the https://docs.kubecost.com/apis/monitoring-apis/api-allocation[Allocation] and https://docs.kubecost.com/apis/monitoring-apis/assets-api[Assets] APIs are fully backwards compatible. https://docs.kubecost.com/install-and-configure/install/kubecostv2[Please review the Kubecost documentation to ensure a smooth transition.] For the full list of enhancements, https://github.com/kubecost/cost-analyzer-helm-chart/releases/tag/v2.0.0[please see the Kubecost release notes] - -[IMPORTANT] -==== - -https://docs.kubecost.com/install-and-configure/install/kubecostv2[Review the Kubecost documentation before upgrading.] Upgrading may impact report availability. - -==== - -*Core features comparison:* - -[cols="1,1,1,1", options="header"] -|=== - -| Feature -| Kubecost free tier 2.0 -| Amazon EKS optimized Kubecost bundle 2.0 -| Kubecost Enterprise 2.0 - - -| Cluster cost visibility -| Single clusters up to 250 cores -| Unified multi-cluster without core limits when integrated with Amazon Managed Service for Prometheus -| Unified and unlimited number of clusters across unlimited numbers of environments (i.e. multi-cloud) - -| Deployment -| User hosted -| User hosted -| User hosted, Kubecost hosted (dedicated tenant), SaaS - -| Databases supported -| Local Prometheus -| Amazon Managed Service for Prometheus or Local Prometheus -| Any prometheus flavor and custom databases - -| Database retention support (raw metrics) -| 15 days -| Unlimited historical data -| Unlimited historical data - -| Kubecost API and UI retention (ETL) -| 15 days -| 15 days -| Unlimited - -| Hybrid cloud visibility -| - -| Amazon EKS and Amazon EKS Anywhere clusters -| Multi-cloud and hybrid cloud - -| Alerts and recurring reports -| Only supported on the primary cluster, limited to 250 cores -| Efficiency alerts, budget alerts, spend change alerts, and https://docs.kubecost.com/using-kubecost/navigating-the-kubecost-ui/alerts[more supported] across all clusters -| Efficiency alerts, budget alerts, spend change alerts, and https://docs.kubecost.com/using-kubecost/navigating-the-kubecost-ui/alerts[more supported] across all clusters - -| Saved reports -| - -| Reports using 15 days of metrics -| Reports using unlimited historical data and metrics - -| Cloud billing integration -| Only supported on the primary cluster, limited to 250 cores -| Custom pricing support for {aws} (including multiple clusters and multiple accounts) -| Custom pricing support for any cloud - -| Savings recommendations -| Only supported on the primary cluster, limited to 250 cores -| Primary cluster insights, but there is no 250 core limit -| Multi-cluster insights - -| Governance: Audits -| - -| - -| Audit historical cost events - -| Single sign-on (SSO) support -| - -| Amazon Cognito supported -| Okta, Auth0, PingID, KeyCloak, and anything else custom - -| Role-based access control (RBAC) with SAML 2.0 -| - -| - -| Okta, Auth0, PingID, KeyCloak, and anything else custom - -| Enterprise training and onboarding -| - -| - -| Full-service training and FinOps onboarding - -| Teams -| - -| - -| Yes - -|=== - -*New Features:* - -The following features have metric limits: - - - -* Kubecost Aggregator -* Network Monitoring -* Kubecost Actions -* Collections -* Anomaly detection -* Container Request Right-Sizing -* Kubecost Forecasting -* Autocomplete for filtering and aggregation - -*Metric limits:* - -[cols="1,1,1,1", options="header"] -|=== -|Metric -|Kubecost Free Tier 2.0 -|Amazon EKS Optimized Kubecost Custom Bundle - 2.0 -|Kubecost Enterprise 2.0 - - -|Cluster size -|Limited to 250 cores -|Unlimited -|Unlimited - -|Metric retention -|15 days -|15 days -|Unlimited - -|Multi-cluster support -|Not available -|Available -|Available - -|Core limits -|250 cores per cluster -|No core limits -|No core limits -|=== - -[[kubecost-v1,kubecost-v1.title]] -== Kubecost v1 - -[cols="1,1,1,1", options="header"] -|=== -|Feature -|Kubecost free tier -|Amazon EKS optimized Kubecost custom bundle -|Kubecost Enterprise - - -|*Deployment* -|User hosted -|User hosted -|User hosted or [.noloc]`Kubecost` hosted (SaaS) - -|*Number of clusters supported* -|Unlimited -|Unlimited -|Unlimited - -|*Databases supported* -|Local [.noloc]`Prometheus` -|Local [.noloc]`Prometheus` or Amazon Managed Service for Prometheus -|[.noloc]`Prometheus`, Amazon Managed Service for Prometheus, [.noloc]`Cortex`, or [.noloc]`Thanos` - -|*Database retention support* -|15 days -|Unlimited historical data -|Unlimited historical data - -|*[.noloc]`Kubecost` API retention (ETL)* -|15 days -|15 days -|Unlimited historical data - -|*Cluster cost visibility* -|Single clusters -|Unified multi-cluster -|Unified multi-cluster - -|*Hybrid cloud visibility* -|- -|Amazon EKS and Amazon EKS Anywhere clusters -|Multi-cloud and hybrid-cloud support - -|*Alerts and recurring reports* -|- -|Efficiency alerts, budget alerts, spend change alerts, and more supported -|Efficiency alerts, budget alerts, spend change alerts, and more supported - -|*Saved reports* -|- -|Reports using 15 days data -|Reports using unlimited historical data - -|*Cloud billing integration* -|Required for each individual cluster -|Custom pricing support for {aws} (including multiple clusters and multiple accounts) -|Custom pricing support for {aws} (including multiple clusters and multiple accounts) - -|*Savings recommendations* -|Single cluster insights -|Single cluster insights -|Multi-cluster insights - -|*Governance: Audits* -|- -|- -|Audit historical cost events - -|*Single sign-on (SSO) support* -|- -|Amazon Cognito supported -|[.noloc]`Okta`, [.noloc]`Auth0`, [.noloc]`PingID`, KeyCloak - -|*Role-based access control (RBAC) with SAML `2.0`* -|- -|- -|[.noloc]`Okta`, [.noloc]`Auth0`, [.noloc]`PingID`, [.noloc]`Keycloak` - -|*Enterprise training and onboarding* -|- -|- -|Full-service training and [.noloc]`FinOps` onboarding -|=== - -[[cost-monitoring-faq,cost-monitoring-faq.title]] -== Frequently asked questions - -See the following common questions and answers about using [.noloc]`Kubecost` with Amazon EKS. - -*What is the Kubecost API retention (ETL) feature?* - -The Kubecost ETL feature aggregates and organizes metrics to surface cost visibility at various levels of granularity (such as `namespace-level`, `pod-level`, and `deployment-level`). For the custom Kubecost bundle, customers get data and insights from metrics for the last 15 days. - -*What is the alerts and recurring reports feature? What alerts and reports does it include?* - -Kubecost alerts allow teams to receive updates on real-time Kubernetes spend as well as cloud spend. Recurring reports enable teams to receive customized views of historical Kubernetes and cloud spend. Both are configurable using the Kubecost UI or Helm values. They support email, Slack, and Microsoft Teams. - -*What do saved reports include?* - -Kubecost saved reports are predefined views of cost and efficiency metrics. They include cost by cluster, namespace, label, and more. - -*What is cloud billing integration?* - -Integration with {aws} billing APIs allows Kubecost to display out-of-cluster costs (such as Amazon S3). Additionally, it allows Kubecost to reconcile Kubecost`'s in-cluster predictions with actual billing data to account for spot usage, savings plans, and enterprise discounts. - -*What do savings recommendations include?* - -Kubecost provides insights and automation to help users optimize their Kubernetes infrastructure and spend. - -*Is there a charge for this functionality?* - -No. You can use this version of Kubecost at no additional charge. If you want additional Kubecost capabilities that aren`'t included in this bundle, you can buy an enterprise license of Kubecost through the {aws} Marketplace, or from Kubecost directly. - -*Is support available?* - -Yes. You can open a support case with the {aws} Support team at link:contact-us/[Contact {aws},type="marketing"]. - -*Do I need a license to use Kubecost features provided by the Amazon EKS integration?* - -No. - -*Can I integrate Kubecost with {aws} Cost and Usage Report for more accurate reporting?* - -Yes. You can configure Kubecost to ingest data from {aws} Cost and Usage Report to get accurate cost visibility, including discounts, Spot pricing, reserved instance pricing, and others. For more information, see https://docs.kubecost.com/install-and-configure/install/cloud-integration/aws-cloud-integrations[{aws} Cloud Billing Integration] in the Kubecost documentation. - -*Does this version support cost management of self-managed Kubernetes clusters on Amazon EC2?* - -No. This version is only compatible with Amazon EKS clusters. - -*Can Kubecost track costs for Amazon EKS on {aws} Fargate?* - -Kubecost provides best effort to show cluster cost visibility for Amazon EKS on Fargate, but with lower accuracy than with Amazon EKS on Amazon EC2. This is primarily due to the difference in how you`'re billed for your usage. With Amazon EKS on Fargate, you`'re billed for consumed resources. With Amazon EKS on Amazon EC2 nodes, you`'re billed for provisioned resources. Kubecost calculates the cost of an Amazon EC2 node based on the node specification, which includes CPU, RAM, and ephemeral storage. With Fargate, costs are calculated based on the requested resources for the Fargate Pods. - -*How can I get updates and new versions of Kubecost?* - -You can upgrade your Kubecost version using standard Helm upgrade procedures. The latest versions are in the https://gallery.ecr.aws/kubecost/cost-analyzer[Amazon ECR Public Gallery]. - -*Is the `*kubectl-cost*` CLI supported? How do I install it?* - -Yes. `Kubectl-cost` is an open source tool by Kubecost (Apache 2.0 License) that provides CLI access to Kubernetes cost allocation metrics. To install `kubectl-cost`, see https://github.com/kubecost/kubectl-cost#installation[Installation] on GitHub. - -*Is the Kubecost user interface supported? How do I access it?* - -Kubecost provides a web dashboard that you can access through `kubectl` port forwarding, an ingress, or a load balancer. You can also use the {aws} Load Balancer Controller to expose [.noloc]`Kubecost` and use Amazon Cognito for authentication, authorization, and user management. For more information, see link:containers/how-to-use-application-load-balancer-and-amazon-cognito-to-authenticate-users-for-your-kubernetes-web-apps[How to use Application Load Balancer and Amazon Cognito to authenticate users for your Kubernetes web apps,type="blog"] on the {aws} blog. - -*Is Amazon EKS Anywhere supported?* - -No. - -[[kubecost-additional,kubecost-additional.title]] -== Additional [.noloc]`Kubecost` Features - -* The following features are available in both [.noloc]`Kubecost` v1 and v2. -* *Export cost metrics* – Amazon EKS optimized cost monitoring is deployed with [.noloc]`Kubecost` and [.noloc]`Prometheus`, which is an open-source monitoring system and time series database. [.noloc]`Kubecost` reads metric from [.noloc]`Prometheus` and then performs cost allocation calculations and writes the metrics back to [.noloc]`Prometheus`. The [.noloc]`Kubecost` front-end reads metrics from [.noloc]`Prometheus` and shows them on the [.noloc]`Kubecost` user interface. The architecture is illustrated in the following diagram. -+ -image::images/kubecost-architecture.png[Kubecost architecture,scaledwidth=100%] -+ -With https://prometheus.io/[Prometheus] pre-installed, you can write queries to ingest [.noloc]`Kubecost` data into your current business intelligence system for further analysis. You can also use it as a data source for your current https://grafana.com/[Grafana] dashboard to display Amazon EKS cluster costs that your internal teams are familiar with. To learn more about how to write [.noloc]`Prometheus` queries, see the https://github.com/opencost/opencost/blob/develop/PROMETHEUS.md[Prometheus Configuration]``readme`` file on GitHub or use the example [.noloc]`Grafana` JSON models in the https://github.com/kubecost/cost-analyzer-helm-chart/tree/develop/cost-analyzer[Kubecost Github repository] as references. -* *{aws} Cost and Usage Report integration* – To perform cost allocation calculations for your Amazon EKS cluster, [.noloc]`Kubecost` retrieves the public pricing information of {aws} services and {aws} resources from the {aws} Price List API. You can also integrate [.noloc]`Kubecost` with *{aws} Cost and Usage Report*:: - to enhance the accuracy of the pricing information specific to your {aws} account. This information includes enterprise discount programs, reserved instance usage, savings plans, and spot usage. To learn more about how the {aws} Cost and Usage Report integration works, see https://docs.kubecost.com/install-and-configure/install/cloud-integration/aws-cloud-integrations[{aws} Cloud Billing Integration] in the [.noloc]`Kubecost` documentation. diff --git a/latest/ug/clusters/management/cost-monitoring-kubecost.adoc b/latest/ug/clusters/management/cost-monitoring-kubecost.adoc deleted file mode 100644 index 5f8311846..000000000 --- a/latest/ug/clusters/management/cost-monitoring-kubecost.adoc +++ /dev/null @@ -1,115 +0,0 @@ - -[.topic] -[[cost-monitoring-kubecost,cost-monitoring-kubecost.title]] -= Install Kubecost and access dashboard -:info_doctype: section - -include::../../attributes.txt[] - -Amazon EKS supports [.noloc]`Kubecost`, which you can use to monitor your costs broken down by [.noloc]`Kubernetes` resources including [.noloc]`Pods`, nodes, namespaces, and labels. This topic covers installing [.noloc]`Kubecost`, and accessing the [.noloc]`Kubecost` dashboard. - -Amazon EKS provides an {aws} optimized bundle of [.noloc]`Kubecost` for cluster cost visibility. You can use your existing {aws} support agreements to obtain support. For more information about the available versions of [.noloc]`Kubecost`, see <>. - -As a [.noloc]`Kubernetes` platform administrator and finance leader, you can use [.noloc]`Kubecost` to visualize a breakdown of Amazon EKS charges, allocate costs, and charge back organizational units such as application teams. You can provide your internal teams and business units with transparent and accurate cost data based on their actual {aws} bill. Moreover, you can also get customized recommendations for cost optimization based on their infrastructure environment and usage patterns within their clusters. - -[NOTE] -==== - -Kubecost v2 introduces several major new features. <> - -==== - -For more information about [.noloc]`Kubecost`, see the https://guide.kubecost.com[Kubecost] documentation. - - -[[kubecost-addon,kubecost-addon.title]] -== Install Kubecost using Amazon EKS Add-ons - -[NOTE] -==== -Install Kubecost as an Amazon EKS Add-on and benefit from additional features at no additional cost with the Amazon EKS optimized Kubecost bundle. For more information, see <>. -==== - -Amazon EKS Add-ons reduce the complexity of upgrading Kubecost, and managing licenses. EKS Add-ons are integrated with the {aws} marketplace. - -. View link:marketplace/seller-profile?id=983de668-2731-4c99-a7e2-74f27d796173[Kubecost in the {aws} Marketplace console,type="marketing"] and subscribe. -. Determine the name of your cluster, and the region. Verify you are logged into the {aws} CLI with sufficient permissions to manage EKS. -. Create the Kubecost addon. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks create-addon --addon-name kubecost_kubecost --cluster-name $YOUR_CLUSTER_NAME --region $AWS_REGION ----- - -Learn how to <>, such as Kubecost. - -[[kubecost-helm,kubecost-helm.title]] -== Install Kubecost using Helm - -* An existing Amazon EKS cluster. To deploy one, see <>. The cluster must have Amazon EC2 nodes because you can't run [.noloc]`Kubecost` on Fargate nodes. -* The `kubectl` command line tool is installed on your device or {aws} CloudShell. The version can be the same as or up to one minor version earlier or later than the [.noloc]`Kubernetes` version of your cluster. For example, if your cluster version is `1.29`, you can use `kubectl` version `1.28`, `1.29`, or `1.30` with it. To install or upgrade `kubectl`, see <>. -* Helm version 3.9.0 or later configured on your device or {aws} CloudShell. To install or update Helm, see <>. -* If your cluster is version `1.23` or later, you must have the <> installed on your cluster. -. Determine the version of [.noloc]`Kubecost` to install. You can see the available versions at https://gallery.ecr.aws/kubecost/cost-analyzer[kubecost/cost-analyzer] in the Amazon ECR Public Gallery. For more information about the compatibility of [.noloc]`Kubecost` versions and Amazon EKS, see the https://docs.kubecost.com/install-and-configure/install/environment[Environment Requirements] in the Kubecost documentation. -. Install [.noloc]`Kubecost` with the following command. Replace [.replaceable]`kubecost-version` with the value retrieved from ECR, such as [.replaceable]`1.108.1`. -+ -[source,bash,subs="verbatim,attributes"] ----- -helm upgrade -i kubecost oci://public.ecr.aws/kubecost/cost-analyzer --version kubecost-version \ - --namespace kubecost --create-namespace \ - -f https://raw.githubusercontent.com/kubecost/cost-analyzer-helm-chart/develop/cost-analyzer/values-eks-cost-monitoring.yaml ----- -+ -[.noloc]`Kubecost` releases new versions regularly. You can update your version using https://helm.sh/docs/helm/helm_upgrade/[helm upgrade]. By default, the installation includes a local https://prometheus.io/[Prometheus] server and `kube-state-metrics`. You can customize your deployment to use link:mt/integrating-kubecost-with-amazon-managed-service-for-prometheus[Amazon Managed Service for Prometheus,type="blog"] by following the documentation in link:prometheus/latest/userguide/integrating-kubecost.html[Integrating with Amazon EKS cost monitoring,type="documentation"]. For a list of all other settings that you can configure, see the https://github.com/kubecost/cost-analyzer-helm-chart/blob/develop/cost-analyzer/values-eks-cost-monitoring.yaml[sample configuration file] on GitHub. -+ -You can remove [.noloc]`Kubecost` from your cluster with the following commands. -+ -[source,bash,subs="verbatim,attributes"] ----- -helm uninstall kubecost --namespace kubecost -kubectl delete ns kubecost ----- - - - -[[kubecost-dashboard,kubecost-dashboard.title]] -== Access Kubecost Dashboard -. Make sure the required [.noloc]`Pods` are running. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl get pods -n kubecost ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -NAME READY STATUS RESTARTS AGE -kubecost-cost-analyzer-b9788c99f-5vj5b 2/2 Running 0 3h27m -kubecost-kube-state-metrics-99bb8c55b-bn2br 1/1 Running 0 3h27m -kubecost-prometheus-server-7d9967bfc8-9c8p7 2/2 Running 0 3h27m ----- -. On your device, enable port-forwarding to expose the [.noloc]`Kubecost` dashboard. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl port-forward --namespace kubecost deployment/kubecost-cost-analyzer 9090 ----- -+ -Alternatively, you can use the <> to expose [.noloc]`Kubecost` and use Amazon Cognito for authentication, authorization, and user management. For more information, see link:containers/how-to-use-application-load-balancer-and-amazon-cognito-to-authenticate-users-for-your-kubernetes-web-apps[How to use Application Load Balancer and Amazon Cognito to authenticate users for your Kubernetes web apps,type="blog"]. -. On the same device that you completed the previous step on, open a web browser and enter the following address. -+ -[source,bash,subs="verbatim,attributes"] ----- -http://localhost:9090 ----- -+ -You see the [.noloc]`Kubecost` Overview page in your browser. It might take 5–10 minutes for [.noloc]`Kubecost` to gather metrics. You can see your Amazon EKS spend, including cumulative cluster costs, associated [.noloc]`Kubernetes` asset costs, and monthly aggregated spend. -+ -image::images/kubecost.png[Kubecost dashboard,scaledwidth=100%] -. To track costs at a cluster level, tag your Amazon EKS resources for billing. For more information, see <>. - - -* *Cost allocation* – View monthly Amazon EKS costs and cumulative costs for each of your namespaces and other dimensions over the past seven days. This is helpful for understanding which parts of your application are contributing to Amazon EKS spend. -* *Assets* – View the costs of the {aws} infrastructure assets that are associated with your Amazon EKS resources. diff --git a/latest/ug/clusters/management/cost-monitoring.adoc b/latest/ug/clusters/management/cost-monitoring.adoc deleted file mode 100644 index 6f92be955..000000000 --- a/latest/ug/clusters/management/cost-monitoring.adoc +++ /dev/null @@ -1,30 +0,0 @@ -//!!NODE_ROOT
- - -[.topic] -[[cost-monitoring,cost-monitoring.title]] -= Monitor and optimize Amazon EKS cluster costs -:info_doctype: section -:info_title: Monitor and optimize Amazon EKS cluster costs -:info_titleabbrev: Cost monitoring -:keywords: cost, monitoring, watch -:info_abstract: Learn how to monitor and optimize costs for your Amazon EKS clusters using {aws} Billing split cost allocation data or Kubecost, a Kubernetes-native cost monitoring tool integrated with {aws}. - -include::../../attributes.txt[] - -[abstract] --- -Learn how to monitor and optimize costs for your Amazon EKS clusters using {aws} Billing split cost allocation data or Kubecost, a Kubernetes-native cost monitoring tool integrated with {aws}. --- - -Cost monitoring is an essential aspect of managing your [.noloc]`Kubernetes` clusters on Amazon EKS. By gaining visibility into your cluster costs, you can optimize resource utilization, set budgets, and make data-driven decisions about your deployments. Amazon EKS provides two cost monitoring solutions, each with its own unique advantages, to help you track and allocate your costs effectively: - -*{aws} Billing split cost allocation data for Amazon EKS* -- This native feature integrates seamlessly with the {aws} Billing Console, allowing you to analyze and allocate costs using the same familiar interface and workflows you use for other {aws} services. With split cost allocation, you can gain insights into your [.noloc]`Kubernetes` costs directly alongside your other {aws} spend, making it easier to optimize costs holistically across your {aws} environment. You can also leverage existing {aws} Billing features like Cost Categories and Cost Anomaly Detection to further enhance your cost management capabilities. For more information, see link:cur/latest/userguide/split-cost-allocation-data.html[Understanding split cost allocation data,type="documentation"] in the {aws} Billing User Guide. - -*[.noloc]`Kubecost`* -- Amazon EKS supports Kubecost, a Kubernetes cost monitoring tool. Kubecost offers a feature-rich, Kubernetes-native approach to cost monitoring, providing granular cost breakdowns by Kubernetes resources, cost optimization recommendations, and out-of-the-box dashboards and reports. Kubecost also retrieves accurate pricing data by integrating with the {aws} Cost and Usage Report, ensuring you get a precise view of your Amazon EKS costs. Learn how to <>. - -include::cost-monitoring-aws.adoc[leveloffset=+1] - -include::cost-monitoring-kubecost.adoc[leveloffset=+1] - -include::cost-monitoring-kubecost-bundles.adoc[leveloffset=+1] diff --git a/latest/ug/clusters/management/eks-managing.adoc b/latest/ug/clusters/management/eks-managing.adoc deleted file mode 100644 index 42e6db90f..000000000 --- a/latest/ug/clusters/management/eks-managing.adoc +++ /dev/null @@ -1,40 +0,0 @@ -//!!NODE_ROOT -include::../../attributes.txt[] -[[eks-managing,eks-managing.title]] -= Organize and monitor cluster resources -:doctype: book -:sectnums: -:toc: left -:icons: font -:experimental: -:idprefix: -:idseparator: - -:sourcedir: . -:info_doctype: chapter -:info_title: Organize and monitor cluster resources -:info_titleabbrev: Cluster management - -This chapter includes the following topics to help you manage your cluster. You can also view information about your <> with the {aws-management-console}. - - - -* The [.noloc]`Kubernetes` Dashboard is a general purpose, web-based UI for [.noloc]`Kubernetes` clusters. It allows users to manage applications running in the cluster and troubleshoot them, as well as manage the cluster itself. For more information, see The https://github.com/kubernetes/dashboard[Kubernetes Dashboard] GitHub repository. -* <> – The [.noloc]`Kubernetes` Metrics Server is an aggregator of resource usage data in your cluster. It isn't deployed by default in your cluster, but is used by [.noloc]`Kubernetes` add-ons, such as the [.noloc]`Kubernetes` Dashboard and <>. In this topic you learn how to install the Metrics Server. -* <> – The Helm package manager for [.noloc]`Kubernetes` helps you install and manage applications on your [.noloc]`Kubernetes` cluster. This topic helps you install and run the Helm binaries so that you can install and manage charts using the Helm CLI on your local computer. -* <> – To help you manage your Amazon EKS resources, you can assign your own metadata to each resource in the form of _tags_. This topic describes tags and shows you how to create them. -* <> – Your {aws} account has default quotas, formerly referred to as limits, for each {aws} service. Learn about the quotas for Amazon EKS and how to increase them. - - -include::cost-monitoring.adoc[leveloffset=+1] - - -include::metrics-server.adoc[leveloffset=+1] - - -include::helm.adoc[leveloffset=+1] - - -include::eks-using-tags.adoc[leveloffset=+1] - - -include::service-quotas.adoc[leveloffset=+1] diff --git a/latest/ug/clusters/private-clusters.adoc b/latest/ug/clusters/private-clusters.adoc index 22477bb60..9018288b2 100644 --- a/latest/ug/clusters/private-clusters.adoc +++ b/latest/ug/clusters/private-clusters.adoc @@ -1,13 +1,9 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[private-clusters,private-clusters.title]] +[#private-clusters] = Deploy private clusters with limited internet access -:info_doctype: section -:info_title: Deploy private clusters with limited internet access :info_titleabbrev: Private clusters -:info_abstract: Learn how to deploy and operate an Amazon EKS cluster without outbound internet access, including requirements for private container registries, endpoint access control, and VPC interface endpoints for {aws} services. [abstract] -- @@ -18,11 +14,16 @@ This topic describes how to deploy an Amazon EKS cluster that is deployed on the If you're not familiar with Amazon EKS networking, see link:containers/de-mystifying-cluster-networking-for-amazon-eks-worker-nodes[De-mystifying cluster networking for Amazon EKS worker nodes,type="blog"]. If your cluster doesn't have outbound internet access, then it must meet the following requirements: - +[#private-clusters-architecture] +== Cluster architecture requirements * Your cluster must pull images from a container registry that's in your VPC. You can create an Amazon Elastic Container Registry in your VPC and copy container images to it for your nodes to pull from. For more information, see <>. * Your cluster must have endpoint private access enabled. This is required for nodes to register with the cluster endpoint. Endpoint public access is optional. For more information, see <>. -* Self-managed [.noloc]`Linux` and [.noloc]`Windows` nodes must include the following bootstrap arguments before they're launched. These arguments bypass Amazon EKS introspection and don't require access to the Amazon EKS API from within the VPC. + +[#private-clusters-node] +== Node requirements + +* Self-managed Linux and Windows nodes must include the following bootstrap arguments before they're launched. These arguments bypass Amazon EKS introspection and don't require access to the Amazon EKS API from within the VPC. + .. Determine the value of your cluster's endpoint with the following command. Replace [.replaceable]`my-cluster` with the name of your cluster. + @@ -45,19 +46,34 @@ aws eks describe-cluster --name my-cluster --query cluster.certificateAuthority ---- + The returned output is a long string. -.. Replace [.replaceable]`cluster-endpoint` and [.replaceable]`certificate-authority` in the following commands with the values returned in the output from the previous commands. For more information about specifying bootstrap arguments when launching self-managed nodes, see <> and <>. +.. Replace the values of `apiServerEndpoint` and `certificateAuthority` in the NodeConfig object with the values returned in the output from the previous commands. For more information about specifying bootstrap arguments when launching self-managed Amazon Linux 2023 nodes, see <> and <>. + -** For [.noloc]`Linux` nodes: +** For Linux nodes: + [source,bash,subs="verbatim,attributes"] ---- ---apiserver-endpoint cluster-endpoint --b64-cluster-ca certificate-authority +--- +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="BOUNDARY" + +--BOUNDARY +Content-Type: application/node.eks.aws + +--- +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: my-cluster + apiServerEndpoint: [.replaceable]https://EXAMPLE108C897D9B2F1B21D5EXAMPLE.sk1.region-code.eks.amazonaws.com + certificateAuthority: [.replaceable]Y2VydGlmaWNhdGVBdXRob3JpdHk= + ... ---- + -For additional arguments, see the https://github.com/awslabs/amazon-eks-ami/blob/main/templates/al2/runtime/bootstrap.sh[bootstrap script] on [.noloc]`GitHub`. -** For [.noloc]`Windows` nodes: +For additional arguments, see the https://github.com/awslabs/amazon-eks-ami/blob/main/templates/al2/runtime/bootstrap.sh[bootstrap script] on GitHub. +** For Windows nodes: + -NOTE: If you're using custom service CIDR, then you need to specify it using the `-ServiceCIDR` parameter. Otherwise, the DNS resolution for [.noloc]`Pods` in the cluster will fail. +NOTE: If you're using custom service CIDR, then you need to specify it using the `-ServiceCIDR` parameter. Otherwise, the DNS resolution for Pods in the cluster will fail. + [source,bash,subs="verbatim,attributes"] ---- @@ -66,12 +82,17 @@ NOTE: If you're using custom service CIDR, then you need to specify it using the + For additional arguments, see <>. * Your cluster's `aws-auth` `ConfigMap` must be created from within your VPC. For more information about creating and adding entries to the `aws-auth` `ConfigMap`, enter `eksctl create iamidentitymapping --help` in your terminal. If the `ConfigMap` doesn't exist on your server, `eksctl` will create it when you use the command to add an identity mapping. -* [.noloc]`Pods` configured with xref:iam-roles-for-service-accounts[IAM roles for service accounts,linkend=iam-roles-for-service-accounts] acquire credentials from an {aws} Security Token Service ({aws} STS) API call. If there is no outbound internet access, you must create and use an {aws} STS VPC endpoint in your VPC. Most {aws} `v1` SDKs use the global {aws} STS endpoint by default (`sts.amazonaws.com`), which doesn't use the {aws} STS VPC endpoint. To use the {aws} STS VPC endpoint, you might need to configure your SDK to use the regional {aws} STS endpoint (``sts.[.replaceable]`region-code`.amazonaws.com``). For more information, see <>. -* Your cluster's VPC subnets must have a VPC interface endpoint for any {aws} services that your [.noloc]`Pods` need access to. For more information, see link:vpc/latest/privatelink/create-interface-endpoint.html[Access an {aws} service using an interface VPC endpoint,type="documentation"]. Some commonly-used services and endpoints are listed in the following table. For a complete list of endpoints, see link:vpc/latest/privatelink/aws-services-privatelink-support.html[{aws} services that integrate with {aws} PrivateLink,type="documentation"] in the link:vpc/latest/privatelink/[{aws} PrivateLink Guide,type="documentation"]. + +[#private-clusters-pod] +== Pod requirements + +* *Pod Identity* - Pods configured with EKS Pod Identity acquire credentials from the EKS Auth API. If there is no outbound internet access, you must create and use a VPC endpoint for the EKS Auth API: `com.amazonaws.region-code.eks-auth`. For more information about the EKS and EKS Auth VPC endpoints, see <>. +* *IRSA* - Pods configured with <> acquire credentials from an {aws} Security Token Service ({aws} STS) API call. If there is no outbound internet access, you must create and use an {aws} STS VPC endpoint in your VPC. Most {aws} `v1` SDKs use the global {aws} STS endpoint by default (`sts.amazonaws.com`), which doesn't use the {aws} STS VPC endpoint. To use the {aws} STS VPC endpoint, you might need to configure your SDK to use the regional {aws} STS endpoint (``sts.[.replaceable]`region-code`.amazonaws.com``). For more information, see <>. +* Your cluster's VPC subnets must have a VPC interface endpoint for any {aws} services that your Pods need access to. For more information, see link:vpc/latest/privatelink/create-interface-endpoint.html[Access an {aws} service using an interface VPC endpoint,type="documentation"]. Some commonly-used services and endpoints are listed in the following table. For a complete list of endpoints, see link:vpc/latest/privatelink/aws-services-privatelink-support.html[{aws} services that integrate with {aws} PrivateLink,type="documentation"] in the link:vpc/latest/privatelink/[{aws} PrivateLink Guide,type="documentation"]. + We recommend that you link:vpc/latest/privatelink/interface-endpoints.html#enable-private-dns-names[enable private DNS names,type="documentation"] for your VPC endpoints, that way workloads can continue using public {aws} service endpoints without issues. + -[cols="1,1", options="header"] +[%header,cols="2"] |=== |Service |Endpoint @@ -83,24 +104,34 @@ We recommend that you link:vpc/latest/privatelink/interface-endpoints.html#enabl |Amazon Elastic Container Registry (for pulling container images) |com.amazonaws.[.replaceable]`region-code`.ecr.api, com.amazonaws.[.replaceable]`region-code`.ecr.dkr, and com.amazonaws.[.replaceable]`region-code`.s3 -|Application Load Balancers and Network Load Balancers +|Amazon Application Load Balancers and Network Load Balancers |com.amazonaws.[.replaceable]`region-code`.elasticloadbalancing -|{aws} X-Ray +|(Optional) {aws} X-Ray (required for tracing sent to {aws} X-Ray) |com.amazonaws.[.replaceable]`region-code`.xray -|Amazon CloudWatch Logs +|(Optional) Amazon SSM (required for the SSM Agent for node management tasks. Alternative to SSH) +|com.amazonaws.[.replaceable]`region-code`.ssm + +|Amazon CloudWatch Logs (required for node and pod logs sent to Amazon CloudWatch Logs) |com.amazonaws.[.replaceable]`region-code`.logs |{aws} Security Token Service (required when using IAM roles for service accounts) -|com.amazonaws.[.replaceable]`region-code`.sts -|=== +|com.amazonaws.[.replaceable]`region-code`.sts +|Amazon EKS Auth (required when using Pod Identity associations) +|com.amazonaws.[.replaceable]`region-code`.eks-auth +|Amazon EKS +|com.amazonaws.[.replaceable]`region-code`.eks + +|=== * Any self-managed nodes must be deployed to subnets that have the VPC interface endpoints that you require. If you create a managed node group, the VPC interface endpoint security group must allow the CIDR for the subnets, or you must add the created node security group to the VPC interface endpoint security group. -* If your [.noloc]`Pods` use Amazon EFS volumes, then before deploying the <>, the driver's https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/deploy/kubernetes/overlays/stable/kustomization.yaml[kustomization.yaml] file must be changed to set the container images to use the same {aws} Region as the Amazon EKS cluster. -* You can use the <> to deploy {aws} Application Load Balancers (ALB) and Network Load Balancers to your private cluster. When deploying it, you should use https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/deploy/configurations/#controller-command-line-flags[command line flags] to set `enable-shield`, `enable-waf`, and `enable-wafv2` to false. https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/ingress/cert_discovery/#discover-via-ingress-rule-host[Certificate discovery] with hostnames from Ingress objects isn't supported. This is because the controller needs to reach {aws} Certificate Manager, which doesn't have a VPC interface endpoint. +* *EFS storage* - If your Pods use Amazon EFS volumes, then before deploying the <>, the driver's https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/deploy/kubernetes/overlays/stable/kustomization.yaml[kustomization.yaml] file must be changed to set the container images to use the same {aws} Region as the Amazon EKS cluster. +* Route53 does not support {aws} PrivateLink. You cannot manage Route53 DNS records from a private Amazon EKS cluster. This impacts Kubernetes https://github.com/kubernetes-sigs/external-dns[external-dns]. +* If you use the EKS Optimized AMI, you should enable the `ec2` endpoint in the table above. Alternatively, you can manually set the Node DNS name. The optimized AMI uses EC2 APIs to set the node DNS name automatically. +* You can use the <> to deploy {aws} Application Load Balancers (ALB) and Network Load Balancers to your private cluster. When deploying it, you should use https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/deploy/configurations/#controller-command-line-flags[command line flags] to set `enable-shield`, `enable-waf`, and `enable-wafv2` to false. https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/guide/ingress/cert_discovery/#discover-via-ingress-rule-host[Certificate discovery] with hostnames from Ingress objects isn't supported. This is because the controller needs to reach {aws} Certificate Manager, which doesn't have a VPC interface endpoint. + The controller supports network load balancers with IP targets, which are required for use with Fargate. For more information, see <> and <>. -* https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md[Cluster Autoscaler] is supported. When deploying Cluster Autoscaler [.noloc]`Pods`, make sure that the command line includes `--aws-use-static-instance-list=true`. For more information, see https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#use-static-instance-list[Use Static Instance List] on [.noloc]`GitHub`. The worker node VPC must also include the {aws} STS VPC endpoint and autoscaling VPC endpoint. +* https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md[Cluster Autoscaler] is supported. When deploying Cluster Autoscaler Pods, make sure that the command line includes `--aws-use-static-instance-list=true`. For more information, see https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#use-static-instance-list[Use Static Instance List] on GitHub. The worker node VPC must also include the {aws} STS VPC endpoint and autoscaling VPC endpoint. * Some container software products use API calls that access the {aws} Marketplace Metering Service to monitor usage. Private clusters do not allow these calls, so you can't use these container types in private clusters. diff --git a/latest/ug/clusters/update-cluster.adoc b/latest/ug/clusters/update-cluster.adoc index 2a29b8ae8..b648d3747 100644 --- a/latest/ug/clusters/update-cluster.adoc +++ b/latest/ug/clusters/update-cluster.adoc @@ -1,123 +1,121 @@ -//!!NODE_ROOT
+include::../attributes.txt[] [.topic] -[[update-cluster,update-cluster.title]] +[#update-cluster] = Update existing cluster to new Kubernetes version -:info_doctype: section -:info_title: Update existing cluster to new Kubernetes version :info_titleabbrev: Update Kubernetes version -:info_abstract: Learn how to update your Amazon EKS cluster to the latest Kubernetes version, ensuring compatibility with nodes and add-ons, and maintaining high availability during the process. - -include::../attributes.txt[] [abstract] -- Learn how to update your Amazon EKS cluster to the latest Kubernetes version, ensuring compatibility with nodes and add-ons, and maintaining high availability during the process. -- -When a new [.noloc]`Kubernetes` version is available in Amazon EKS, you can update your Amazon EKS cluster to the latest version. +When a new Kubernetes version is available in Amazon EKS, you can update your Amazon EKS cluster to the latest version. [IMPORTANT] ==== -Once you upgrade a cluster, you can't downgrade to a previous version. We recommend that, before you update to a new [.noloc]`Kubernetes` version, you review the information in <> and also review in the update steps in this topic. +Once you upgrade a cluster, you can't downgrade to a previous version. Before you update to a new Kubernetes version, we recommend that you review the information in link:eks/latest/userguide/kubernetes-versions.html[Understand the Kubernetes version lifecycle on EKS,type="documentation"] and the update steps in this topic. ==== -New [.noloc]`Kubernetes` versions sometimes introduce significant changes. Therefore, we recommend that you test the behavior of your applications against a new [.noloc]`Kubernetes` version before you update your production clusters. You can do this by building a continuous integration workflow to test your application behavior before moving to a new [.noloc]`Kubernetes` version. +New Kubernetes versions sometimes introduce significant changes. Therefore, we recommend that you test the behavior of your applications against a new Kubernetes version before you update your production clusters. You can do this by building a continuous integration workflow to test your application behavior before moving to a new Kubernetes version. -The update process consists of Amazon EKS launching new API server nodes with the updated [.noloc]`Kubernetes` version to replace the existing ones. Amazon EKS performs standard infrastructure and readiness health checks for network traffic on these new nodes to verify that they're working as expected. However, once you've started the cluster upgrade, you can't pause or stop it. If any of these checks fail, Amazon EKS reverts the infrastructure deployment, and your cluster remains on the prior [.noloc]`Kubernetes` version. Running applications aren't affected, and your cluster is never left in a non-deterministic or unrecoverable state. Amazon EKS regularly backs up all managed clusters, and mechanisms exist to recover clusters if necessary. We're constantly evaluating and improving our [.noloc]`Kubernetes` infrastructure management processes. +The update process consists of Amazon EKS launching new API server nodes with the updated Kubernetes version to replace the existing ones. Amazon EKS performs standard infrastructure and readiness health checks for network traffic on these new nodes to verify that they're working as expected. However, once you've started the cluster upgrade, you can't pause or stop it. If any of these checks fail, Amazon EKS reverts the infrastructure deployment, and your cluster remains on the prior Kubernetes version. Running applications aren't affected, and your cluster is never left in a non-deterministic or unrecoverable state. Amazon EKS regularly backs up all managed clusters, and mechanisms exist to recover clusters if necessary. We're constantly evaluating and improving our Kubernetes infrastructure management processes. To update the cluster, Amazon EKS requires up to five available IP addresses from the subnets that you specified when you created your cluster. Amazon EKS creates new cluster elastic network interfaces (network interfaces) in any of the subnets that you specified. The network interfaces may be created in different subnets than your existing network interfaces are in, so make sure that your security group rules allow <> for any of the subnets that you specified when you created your cluster. If any of the subnets that you specified when you created the cluster don't exist, don't have enough available IP addresses, or don't have security group rules that allows necessary cluster communication, then the update can fail. -[NOTE] -==== - -To ensure that the API server endpoint for your cluster is always accessible, Amazon EKS provides a highly available [.noloc]`Kubernetes` control plane and performs rolling updates of API server instances during update operations. In order to account for changing IP addresses of API server instances supporting your [.noloc]`Kubernetes` API server endpoint, you must ensure that your API server clients manage reconnects effectively. Recent versions of `kubectl` and the [.noloc]`Kubernetes` client https://kubernetes.io/docs/tasks/administer-cluster/access-cluster-api/#programmatic-access-to-the-api[libraries] that are officially supported, perform this reconnect process transparently. +To ensure that the API server endpoint for your cluster is always accessible, Amazon EKS provides a highly available Kubernetes control plane and performs rolling updates of API server instances during update operations. In order to account for changing IP addresses of API server instances supporting your Kubernetes API server endpoint, you must ensure that your API server clients manage reconnects effectively. Recent versions of `kubectl` and the Kubernetes client https://kubernetes.io/docs/tasks/administer-cluster/access-cluster-api/#programmatic-access-to-the-api[libraries] that are officially supported, perform this reconnect process transparently. -==== +NOTE: To learn more about what goes into a cluster update, see link:eks/latest/best-practices/cluster-upgrades.html["Best Practices for Cluster Upgrades",type="documentation"] in the EKS Best Practices Guide. This resource helps you plan an upgrade, and understand the strategy of upgrading a cluster. == Considerations for Amazon EKS Auto Mode * The compute capability of Amazon EKS Auto Mode controls the Kubernetes version of nodes. After you upgrade the control plane, EKS Auto Mode will begin incrementally updating managed nodes. EKS Auto Mode respects pod disruption budgets. * You do not have to manually upgrade the capabilities of Amazon EKS Auto Mode, including the compute autoscaling, block storage, and load balancing capabilities. -[[update-existing-cluster,update-existing-cluster.title]] +[#update-cluster-summary] +== Summary +The high-level summary of the Amazon EKS cluster upgrade process is as follows: + +. Ensure your cluster is in a state that will support an upgrade. This includes checking the Kubernetes APIs used by resources deployed into the cluster, ensuring the cluster is free of any health issues. You should use Amazon EKS upgrade insights when evaluating your cluster's upgrade readiness. +. Upgrade the control plane to the next minor version (for example, from {k8s-n-1} to {k8s-n}). +. Upgrade the nodes in the data plane to match that of the control plane. +. Upgrade any additional applications that run on the cluster (for example, `cluster-autoscaler`). +. Upgrade the add-ons provided by Amazon EKS, such as those included by default: + * <> + * <> + * <> +. Upgrade any clients that communicate with the cluster (for example, `kubectl`). + +[#update-existing-cluster] == Step 1: Prepare for upgrade -. Compare the [.noloc]`Kubernetes` version of your cluster control plane to the [.noloc]`Kubernetes` version of your nodes. -+ -** Get the [.noloc]`Kubernetes` version of your cluster control plane. +Compare the Kubernetes version of your cluster control plane to the Kubernetes version of your nodes. + +* Get the Kubernetes version of your cluster control plane. + [source,bash,subs="verbatim,attributes"] ---- kubectl version ---- -** Get the [.noloc]`Kubernetes` version of your nodes. This command returns all self-managed and managed Amazon EC2, Fargate, and hybrid nodes. Each Fargate [.noloc]`Pod` is listed as its own node. +* Get the Kubernetes version of your nodes. This command returns all self-managed and managed Amazon EC2, Fargate, and hybrid nodes. Each Fargate Pod is listed as its own node. + [source,bash,subs="verbatim,attributes"] ---- kubectl get nodes ---- -+ -Before updating your control plane to a new [.noloc]`Kubernetes` version, make sure that the [.noloc]`Kubernetes` minor version of both the managed nodes and Fargate nodes in your cluster are the same as your control plane's version. For example, if your control plane is running version `1.29` and one of your nodes is running version `1.28`, then you must update your nodes to version `1.29` before updating your control plane to 1.30. We also recommend that you update your self-managed nodes and hybrid nodes to the same version as your control plane before updating the control plane. For more information, see <>, <>, and <>. If you have Fargate nodes with a minor version lower than the control plane version, first delete the [.noloc]`Pod` that's represented by the node. Then update your control plane. Any remaining [.noloc]`Pods` will update to the new version after you redeploy them. -. If the [.noloc]`Kubernetes` version that you originally deployed your cluster with was [.noloc]`Kubernetes` `1.25` or later, skip this step. -+ -By default, the [.noloc]`Pod` security policy admission controller is enabled on Amazon EKS clusters. Before updating your cluster, ensure that the proper [.noloc]`Pod` security policies are in place. This is to avoid potential security issues. You can check for the default policy with the `kubectl get psp eks.privileged` command. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl get psp eks.privileged ----- -+ -If you receive the following error, see <> before proceeding. -+ -[source,bash,subs="verbatim,attributes"] ----- -Error from server (NotFound): podsecuritypolicies.extensions "eks.privileged" not found ----- -. If the [.noloc]`Kubernetes` version that you originally deployed your cluster with was [.noloc]`Kubernetes` `1.18` or later, skip this step. -+ -You might need to remove a discontinued term from your [.noloc]`CoreDNS` manifest. -+ -.. Check to see if your [.noloc]`CoreDNS` manifest has a line that only has the word `upstream`. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl get configmap coredns -n kube-system -o jsonpath='{$.data.Corefile}' | grep upstream ----- -+ -If no output is returned, this means that your manifest doesn't have the line. If this is the case, skip to the next step. If the word `upstream` is returned, remove the line. -.. Remove the line near the top of the file that only has the word `upstream` in the configmap file. Don't change anything else in the file. After the line is removed, save the changes. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl edit configmap coredns -n kube-system -o yaml ----- +Before updating your control plane to a new Kubernetes version, make sure that the Kubernetes minor version of both the managed nodes and Fargate nodes in your cluster are the same as your control plane's version. For example, if your control plane is running version `1.29` and one of your nodes is running version `1.28`, then you must update your nodes to version `1.29` before updating your control plane to 1.30. We also recommend that you update your self-managed nodes and hybrid nodes to the same version as your control plane before updating the control plane. For more information, see <>, <>, and <>. If you have Fargate nodes with a minor version lower than the control plane version, first delete the Pod that's represented by the node. Then update your control plane. Any remaining Pods will update to the new version after you redeploy them. == Step 2: Review upgrade considerations -* If you're updating to version `1.23` and use Amazon EBS volumes in your cluster, then you must install the Amazon EBS CSI driver in your cluster before updating your cluster to version `1.23` to avoid workload disruptions. For more information, see <> and <>. -* Kubernetes `1.24` and later use `containerd` as the default container runtime. If you're switching to the `containerd` runtime and already have [.noloc]`Fluentd` configured for [.noloc]`Container Insights`, then you must migrate [.noloc]`Fluentd` to [.noloc]`Fluent Bit` before updating your cluster. The [.noloc]`Fluentd` parsers are configured to only parse log messages in JSON format. Unlike `dockerd`, the `containerd` container runtime has log messages that aren't in JSON format. If you don't migrate to [.noloc]`Fluent Bit`, some of the configured [.noloc]`Fluentd's` parsers will generate a massive amount of errors inside the [.noloc]`Fluentd` container. For more information on migrating, see link:AmazonCloudWatch/latest/monitoring/Container-Insights-setup-logs-FluentBit.html[Set up Fluent Bit as a DaemonSet to send logs to CloudWatch Logs,type="documentation"]. -** Because Amazon EKS runs a highly available control plane, you can update only one minor version at a time. For more information about this requirement, see https://kubernetes.io/docs/setup/version-skew-policy/#kube-apiserver[Kubernetes Version and Version Skew Support Policy]. Assume that your current cluster version is version `1.28` and you want to update it to version `1.30`. You must first update your version `1.28` cluster to version `1.29` and then update your version `1.29` cluster to version `1.30`. -* Review the version skew between the [.noloc]`Kubernetes` `kube-apiserver` and the `kubelet` on your nodes. +Amazon EKS cluster insights automatically scan clusters against a list of potential Kubernetes version upgrade impacting issues such as deprecated Kubernetes API usage. Amazon EKS periodically updates the list of insight checks to perform based on evaluations of changes in the Kubernetes project. Amazon EKS also updates the insight checks list as changes are introduced in the Amazon EKS service along with new versions. For more information, see <>. + +Review the https://kubernetes.io/docs/reference/using-api/deprecation-guide/[Deprecated API Migration Guide] in the Kubernetes docs. + +=== Review upgrade insights + +Use Amazon EKS upgrade insights to identify issues. For more information, see <>. + +=== Detailed considerations + +* Because Amazon EKS runs a highly available control plane, you can update only one minor version at a time. For more information about this requirement, see https://kubernetes.io/docs/setup/version-skew-policy/#kube-apiserver[Kubernetes Version and Version Skew Support Policy]. Assume that your current cluster version is version `1.28` and you want to update it to version `1.30`. You must first update your version `1.28` cluster to version `1.29` and then update your version `1.29` cluster to version `1.30`. +* Review the version skew between the Kubernetes `kube-apiserver` and the `kubelet` on your nodes. + -** Starting from [.noloc]`Kubernetes` version `1.28`, `kubelet` may be up to three minor versions older than `kube-apiserver`. See https://kubernetes.io/releases/version-skew-policy/#kubelet[Kubernetes upstream version skew policy]. -** If the `kubelet` on your managed and Fargate nodes is on [.noloc]`Kubernetes` version `1.25` or newer, you can update your cluster up to three versions ahead without updating the `kubelet` version. For example, if the `kubelet` is on version `1.25`, you can update your Amazon EKS cluster version from `1.25` to `1.26`, to `1.27`, and to `1.28` while the `kubelet` remains on version `1.25`. -** If the `kubelet` on your managed and Fargate nodes is on [.noloc]`Kubernetes` version `1.24` or older, it may only be up to two minor versions older than the `kube-apiserver`. In other words, if the `kubelet` is version `1.24` or older, you can only update your cluster up to two versions ahead. For example, if the `kubelet` is on version `1.21`, you can update your Amazon EKS cluster version from `1.21` to `1.22`, and to `1.23`, but you will not be able to update the cluster to `1.24` while the `kubelet` remains on `1.21`. -* As a best practice before starting an update, make sure that the `kubelet` on your nodes is at the same [.noloc]`Kubernetes` version as your control plane. -* If your cluster is configured with a version of the [.noloc]`Amazon VPC CNI plugin for Kubernetes` that is earlier than `1.8.0`, then we recommend that you update the plugin to the latest version before updating your cluster. To update the plugin, see <>. -* If you're updating your cluster to version `1.25` or later and have the [.noloc]`{aws} Load Balancer Controller` deployed in your cluster, then update the controller to version `2.4.7` or later _before_ updating your cluster version to `1.25`. For more information, see the xref:kubernetes-1.25[Kubernetes 1.25,linkend=kubernetes-1.25] release notes. +** Starting from Kubernetes version `1.28`, `kubelet` may be up to three minor versions older than `kube-apiserver`. See https://kubernetes.io/releases/version-skew-policy/#kubelet[Kubernetes upstream version skew policy]. +** If the `kubelet` on your managed and Fargate nodes is on Kubernetes version `1.25` or newer, you can update your cluster up to three versions ahead without updating the `kubelet` version. For example, if the `kubelet` is on version `1.25`, you can update your Amazon EKS cluster version from `1.25` to `1.26`, to `1.27`, and to `1.28` while the `kubelet` remains on version `1.25`. +* As a best practice before starting an update, make sure that the `kubelet` on your nodes is at the same Kubernetes version as your control plane. +* If your cluster is configured with a version of the Amazon VPC CNI plugin for Kubernetes that is earlier than `1.8.0`, then we recommend that you update the plugin to the latest version before updating your cluster. To update the plugin, see <>. +[#update-cluster-control-plane] == Step 3: Update cluster control plane +//Whenever you try to upgrade the Kubernetes version of your cluster's control plane, cluster insights will check for certain upgrade impacting issues. Cluster insights will proactively prevent you from upgrading your cluster if it detects any such issues. You will need to review and resolve the issues to continue with the cluster upgrade. + +//Once the upgrade impacting issues are resolved, you will be able to upgrade the Kubernetes version of your cluster. When needed, you can use the `--force` flag for update cluster version commands. Passing this flag will force the upgrade even if their are upgrade impacting issues detected by cluster insights. + +[IMPORTANT] +==== +Amazon EKS has temporarily rolled back a feature that would +require you to use a `--force` flag to upgrade your cluster when there were certain cluster insight issues. For more information, see link:https://github.com/aws/containers-roadmap/issues/2570[Temporary rollback of enforcing upgrade insights on update cluster version] on GitHub. + +Amazon EKS refreshes a cluster insight 24 hours after the "last refresh time". You can compare the time you addressed an issue to the "last refresh time" of the cluster insight. + +Additionally, it can take up to 30 days for the insight status to update after addressing deprecated API usage. Upgrade insights always looks for deprecated API usage over a rolling 30 day window. + +//Amazon EKS refreshes a cluster insight 24 hours after the "last refresh time". Once the next insight check is run and is determined to be resolved, you can upgrade your cluster normally. Additionally, it can take up to 30 days for the insight status to update after addressing deprecated API usage. Upgrade insights always looks for deprecated API usage over a rolling 30 day window. + +//You can compare the time you addressed an issue to the "last refresh time" of the cluster insight. If you determine that the cluster insight is no longer accurate, you can add the `--force` flag without waiting for the next insight check refresh. + +==== + You can submit the request to upgrade your EKS control plane version using: -* xref:step3-eksctl[eksctl] -* xref:step3-console[the {aws} console] -* xref:step3-cli[the {aws} cli] +* <> +* <> +* <> -[[step3-eksctl,step3-eksctl.title]] +[#step3-eksctl] === Update cluster - eksctl This procedure requires `eksctl` version `{eksctl-min-version}` or later. You can check your version with the following command: @@ -129,36 +127,36 @@ eksctl version For instructions on how to install and update `eksctl`, see https://eksctl.io/installation[Installation] in the `eksctl` documentation. -Update the [.noloc]`Kubernetes` version of your Amazon EKS control plane. Replace [.replaceable]`my-cluster` with your cluster name. Replace [.replaceable]`1.30` with the Amazon EKS supported version number that you want to update your cluster to. For a list of supported version numbers, see <>. +Update the Kubernetes version of your Amazon EKS control plane. Replace `` with your cluster name. Replace `` with the Amazon EKS supported version number that you want to update your cluster to. For a list of supported version numbers, see link:eks/latest/userguide/kubernetes-versions.html[Amazon EKS supported versions,type="documentation"]. [source,bash,subs="verbatim,attributes"] ---- -eksctl upgrade cluster --name my-cluster --version 1.30 --approve +eksctl upgrade cluster --name --version --approve ---- The update takes several minutes to complete. -Continue to <> +Continue to <>. -[[step3-console,step3-console.title]] +[#step3-console] === Update cluster - {aws} console . Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. -. Choose the name of the Amazon EKS cluster to update and choose *Update cluster version*. -. For *[.noloc]`Kubernetes` version*, select the version to update your cluster to and choose *Update*. -. For *Cluster name*, enter the name of your cluster and choose *Confirm*. -+ -The update takes several minutes to complete. -. Continue to <> +. Choose *Upgrade now* for a cluster you wish to upgrade. +. Select the version to update your cluster to and choose *Upgrade*. +//. If cluster insights has identified any upgrade impacting issues, Amazon EKS will prevent you from upgrading your cluster. You can force the upgrade by typing `confirm` in the *confirm upgrade* field and choosing *upgrade*. +. The update takes several minutes to complete. Continue to <>. -[[step3-cli,step3-cli.title]] +[#step3-cli] === Update cluster - {aws} CLI -. Update your Amazon EKS cluster with the following {aws} CLI command. Replace the [.replaceable]`example values` with your own. Replace [.replaceable]`1.30` with the Amazon EKS supported version number that you want to update your cluster to. For a list of supported version numbers, see <>. +. Verify that the {aws} CLI is installed and that you are logged in. For more information, see link:cli/latest/userguide/getting-started-install.html[Installing or updating to the latest version of the {aws} CLI,type="documentation"]. +. Update your Amazon EKS cluster with the following {aws} CLI command. Replace `` and `` of the cluster you want to upgrade. Replace `` with the Amazon EKS supported version number that you want to update your cluster to. For a list of supported version numbers, see link:eks/latest/userguide/kubernetes-versions.html[Amazon EKS supported versions,type="documentation"]. + [source,bash,subs="verbatim,attributes"] ---- -aws eks update-cluster-version --region region-code --name my-cluster --kubernetes-version 1.30 +aws eks update-cluster-version --name \ + --kubernetes-version --region ---- + An example output is as follows. @@ -167,13 +165,13 @@ An example output is as follows. ---- { "update": { - "id": "b5f0ba18-9a87-4450-b5a0-825e6e84496f", + "id": "", "status": "InProgress", "type": "VersionUpdate", "params": [ { "type": "Version", - "value": "1.30" + "value": "" }, { "type": "PlatformVersion", @@ -186,71 +184,46 @@ An example output is as follows. ---- -. Monitor the status of your cluster update with the following command. Use the cluster name and update ID that the previous command returned. When a `Successful` status is displayed, the update is complete. The update takes several minutes to complete. +. The update takes several minutes to complete. Monitor the status of your cluster update with the following command. In addition to using the same `` and ``, use the `` that the previous command returned. + [source,bash,subs="verbatim,attributes"] ---- -aws eks describe-update --region region-code --name my-cluster --update-id b5f0ba18-9a87-4450-b5a0-825e6e84496f +aws eks describe-update --name \ + --region --update-id ---- +//+ +//If needed, resolve any upgrade impacting issues and repeat this procedure. If you need to override `ERROR` status upgrade insights checks that you believe are no longer applicable, add the `--force` flag to the `update-cluster-version` command. + -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -{ - "update": { - "id": "b5f0ba18-9a87-4450-b5a0-825e6e84496f", - "status": "Successful", - "type": "VersionUpdate", - "params": [ - { - "type": "Version", - "value": "1.30" - }, - { - "type": "PlatformVersion", - "value": "eks.1" - } - ], -[...] - "errors": [] - } - ----- -. Continue to <> +When a `Successful` status is displayed, the update is complete. +. Continue to <>. -[[step4,step4.title]] +[#step4] == Step 4: Update cluster components -. After your cluster update is complete, update your nodes to the same [.noloc]`Kubernetes` minor version as your updated cluster. For more information, see <>, <>, and <>. Any new [.noloc]`Pods` that are launched on Fargate have a `kubelet` version that matches your cluster version. Existing Fargate [.noloc]`Pods` aren't changed. -. (Optional) If you deployed the [.noloc]`Kubernetes` Cluster Autoscaler to your cluster before updating the cluster, update the Cluster Autoscaler to the latest version that matches the [.noloc]`Kubernetes` major and minor version that you updated to. +. After your cluster update is complete, update your nodes to the same Kubernetes minor version as your updated cluster. For more information, see <>, <>, and <>. Any new Pods that are launched on Fargate have a `kubelet` version that matches your cluster version. Existing Fargate Pods aren't changed. +. (Optional) If you deployed the Kubernetes Cluster Autoscaler to your cluster before updating the cluster, update the Cluster Autoscaler to the latest version that matches the Kubernetes major and minor version that you updated to. + -.. Open the Cluster Autoscaler https://github.com/kubernetes/autoscaler/releases[releases] page in a web browser and find the latest Cluster Autoscaler version that matches your cluster's [.noloc]`Kubernetes` major and minor version. For example, if your cluster's [.noloc]`Kubernetes` version is `1.30` find the latest Cluster Autoscaler release that begins with `1.30`. Record the semantic version number (``1.30.n``, for example) for that release to use in the next step. -.. Set the Cluster Autoscaler image tag to the version that you recorded in the previous step with the following command. If necessary, replace [.replaceable]`1.30`.[.replaceable]`n`` with your own value. +.. Open the Cluster Autoscaler https://github.com/kubernetes/autoscaler/releases[releases] page in a web browser and find the latest Cluster Autoscaler version that matches your cluster's Kubernetes major and minor version. For example, if your cluster's Kubernetes version is `1.30` find the latest Cluster Autoscaler release that begins with `1.30`. Record the semantic version number (``1.30.n``, for example) for that release to use in the next step. +.. Set the Cluster Autoscaler image tag to the version that you recorded in the previous step with the following command. If necessary, replace `X.XX.X` with your own value. + [source,bash,subs="verbatim,attributes"] ---- -kubectl -n kube-system set image deployment.apps/cluster-autoscaler cluster-autoscaler=registry.k8s.io/autoscaling/cluster-autoscaler:v1.30.n +kubectl -n kube-system set image deployment.apps/cluster-autoscaler cluster-autoscaler=registry.k8s.io/autoscaling/cluster-autoscaler:vX.XX.X ---- -. (Clusters with GPU nodes only) If your cluster has node groups with GPU support (for example, `p3.2xlarge`), you must update the https://github.com/NVIDIA/k8s-device-plugin[NVIDIA device plugin for Kubernetes][.noloc]`DaemonSet` on your cluster. Replace [.replaceable]`vX.X.X` with your desired https://github.com/NVIDIA/k8s-device-plugin/releases[NVIDIA/k8s-device-plugin] version before running the following command. +. (Clusters with GPU nodes only) If your cluster has node groups with GPU support (for example, `p3.2xlarge`), you must update the https://github.com/NVIDIA/k8s-device-plugin[NVIDIA device plugin for Kubernetes]DaemonSet on your cluster. Replace `` with your desired https://github.com/NVIDIA/k8s-device-plugin/releases[NVIDIA/k8s-device-plugin] version before running the following command. + [source,bash,subs="verbatim,attributes"] ---- -kubectl apply -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/vX.X.X/deployments/static/nvidia-device-plugin.yml +kubectl apply -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin//deployments/static/nvidia-device-plugin.yml ---- -. Update the [.noloc]`Amazon VPC CNI plugin for Kubernetes`, [.noloc]`CoreDNS`, and `kube-proxy` add-ons. We recommend updating the add-ons to the minimum versions listed in <>. +. Update the Amazon VPC CNI plugin for Kubernetes, CoreDNS, and `kube-proxy` add-ons. We recommend updating the add-ons to the minimum versions listed in <>. + -** If you are using Amazon EKS add-ons, select *Clusters* in the Amazon EKS console, then select the name of the cluster that you updated in the left navigation pane. Notifications appear in the console. They inform you that a new version is available for each add-on that has an available update. To update an add-on, select the *Add-ons* tab. In one of the boxes for an add-on that has an update available, select *Update now*, select an available version, and then select *Update*. +** If you are using Amazon EKS add-ons, select *Clusters* in the Amazon EKS console, then select the name of the cluster that you updated in the left navigation pane. Notifications appear in the console. They inform you that a new version is available for each add-on that has an available update. To update an add-on, select the *Add-ons* tab. In one of the boxes for an add-on that has an update available, select *Update now*, select an available version, and then select *Update*. ** Alternately, you can use the {aws} CLI or `eksctl` to update add-ons. For more information, see <>. -. If necessary, update your version of `kubectl`. You must use a `kubectl` version that is within one minor version difference of your Amazon EKS cluster control plane. For example, a `1.29` `kubectl` client works with [.noloc]`Kubernetes` `1.28`, `1.29`, and `1.30` clusters. You can check your currently installed version with the following command. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl version --client ----- +. If necessary, update your version of `kubectl`. You must use a `kubectl` version that is within one minor version difference of your Amazon EKS cluster control plane. -[[downgrade-cluster,downgrade-cluster.title]] -== Downgrade the [.noloc]`Kubernetes` version for an Amazon EKS cluster +[#downgrade-cluster] +== Downgrade the Kubernetes version for an Amazon EKS cluster -You cannot downgrade the [.noloc]`Kubernetes` of an Amazon EKS cluster. Instead, create a new cluster on a previous Amazon EKS version and migrate the workloads. +You cannot downgrade the Kubernetes of an Amazon EKS cluster. Instead, create a new cluster on a previous Amazon EKS version and migrate the workloads. diff --git a/latest/ug/clusters/view-cluster-insights.adoc b/latest/ug/clusters/view-cluster-insights.adoc new file mode 100644 index 000000000..d3b097df1 --- /dev/null +++ b/latest/ug/clusters/view-cluster-insights.adoc @@ -0,0 +1,232 @@ +include::../attributes.txt[] + +[.topic] +[#view-cluster-insights] += View cluster insights +Amazon EKS cluster insights provide recommendations to help you follow Amazon EKS and Kubernetes best practices. Every Amazon EKS cluster undergoes automatic, recurring checks against an Amazon EKS curated list of insights. These __insight checks__ are fully managed by Amazon EKS and offer recommendations on how to address any findings. + +//In addition to insight checks for best practices, there are upgrade insights that check for issues before you upgrade your cluster to a new Kubernetes version. +Amazon EKS provides two types of insights: *Configuration insights* and *Upgrade insights*. *Configuration insights* identify misconfigurations in your EKS Hybrid Nodes setup that could impair functionality of your cluster or workloads. *Upgrade insights* identify issues that could impact your ability to upgrade to new versions of Kubernetes. + +To see the list of insight checks performed and any relevant issues that Amazon EKS has identified, you can call the look in the {aws-management-console}, the {aws} CLI, {aws} SDKs, and Amazon EKS `ListInsights` API operation. + + +[#view-config-insights-console] +== View configuration insights (Console) +. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. +. From the cluster list, choose the name of the Amazon EKS cluster for which you want to see the insights. +. Choose *Monitor cluster*. +. Choose the *Cluster health* tab. +. In the *Configuration insights* table, you will see the following columns: ++ +** *Name* – The check that was performed by Amazon EKS against the cluster. +** *Insight status* – An insight with a status of `Error` means that there is a misconfiguration that is likely impacting cluster functionality. An insight with a status of `Warning` means that the configuration doesn't match the documented approach, but that cluster functionality might work if you configured it intentionally. An insight with status of `Passing` means Amazon EKS has not found any issues associated with this insight check in your cluster. +** *Version* – The applicable version. +** *Last refresh time* – The time the status of the insight was last refreshed for this cluster. +** *Description* – Information from the insight check, which includes the alert and recommended actions for remediation. + + +[#view-upgrade-insights-console] +== View upgrade insights (Console) + +. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. +. From the cluster list, choose the name of the Amazon EKS cluster for which you want to see the insights. +. Choose *Monitor cluster*. +. Choose the *Upgrade insights* tab. +. To view the latest data, choose the *Refresh insights* button and wait for the refresh operation to complete. +. In the *Upgrade insights* table, you will see the following columns: ++ +** *Name* – The check that was performed by Amazon EKS against the cluster. +** *Insight status* – An insight with a status of "Error" typically means the impacted Kubernetes version is N+1 of the current cluster version, while a status of "Warning" means the insight applies to a future Kubernetes version N+2 or more. An insight with status of "Passing" means Amazon EKS has not found any issues associated with this insight check in your cluster. An insight status of "Unknown" means Amazon EKS is unable to determine if your cluster is impacted by this insight check. +** *Version* – The Kubernetes version that the insight checked for possible issues. +** *Last refresh time* – The time the status of the insight was last refreshed for this cluster. +** *Last transition time* – The time the status of this insight last changed. +** *Description* – Information from the insight check, which includes the alert and recommended actions for remediation. + + +[#cluster-insights-cli] +== View cluster insights ({aws} CLI) + +. To view the latest data, refresh the insights for a specified cluster. Make the following modifications to the command as needed and then run the modified command. ++ +** Replace [.replaceable]`region-code` with the code for your {aws} Region. +** Replace [.replaceable]`my-cluster` with the name of your cluster. ++ +[source,bash,subs="verbatim,attributes,quotes"] +---- +aws eks start-insight-refresh --region [.replaceable]`region-code` --cluster-name [.replaceable]`my-cluster` +---- +. To track the status of an insights refresh, run the following command. Replace [.replaceable]`my-cluster` with the name of your cluster. ++ +[source,bash,subs="verbatim,attributes,quotes"] +---- +aws eks describe-insights-refresh --cluster-name [.replaceable]`my-cluster` +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes",role="nocopy"] +---- +{ + "message": "Insights refresh is in progress", + "status": "IN_PROGRESS", + "startedAt": "2025-07-30T13:36:09-07:00" +} +---- +. List the insights for a specified cluster. Make the following modifications to the command as needed and then run the modified command. ++ +** Replace [.replaceable]`region-code` with the code for your {aws} Region. +** Replace [.replaceable]`my-cluster` with the name of your cluster. ++ +[source,bash,subs="verbatim,attributes,quotes"] +---- +aws eks list-insights --region [.replaceable]`region-code` --cluster-name [.replaceable]`my-cluster` +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes",role="nocopy"] +---- +{ +"insights": + [ + { + "id": "a1b2c3d4-5678-90ab-cdef-EXAMPLE11111", + "name": "Deprecated APIs removed in Kubernetes vX.XX", + "category": "UPGRADE_READINESS", + "kubernetesVersion": "X.XX", + "lastRefreshTime": 1734557315.000, + "lastTransitionTime": 1734557309.000, + "description": "Checks for usage of deprecated APIs that are scheduled for removal in Kubernetes vX.XX. Upgrading your cluster before migrating to the updated APIs supported by vX.XX could cause application impact.", + "insightStatus": + { + "status": "PASSING", + "reason": "No deprecated API usage detected within the last 30 days.", + }, + }, + { + "id": "a1b2c3d4-5678-90ab-cdef-EXAMPLE22222", + "name": "Kubelet version skew", + "category": "UPGRADE_READINESS", + "kubernetesVersion": "X.XX", + "lastRefreshTime": 1734557309.000, + "lastTransitionTime": 1734557309.000, + "description": "Checks for kubelet versions of worker nodes in the cluster to see if upgrade would cause non compliance with supported Kubernetes kubelet version skew policy.", + "insightStatus": + { + "status": "UNKNOWN", + "reason": "Unable to determine status of node kubelet versions.", + }, + }, + { + "id": "a1b2c3d4-5678-90ab-cdef-EXAMPLE33333", + "name": "Deprecated APIs removed in Kubernetes vX.XX", + "category": "UPGRADE_READINESS", + "kubernetesVersion": "X.XX", + "lastRefreshTime": 1734557315.000, + "lastTransitionTime": 1734557309.000, + "description": "Checks for usage of deprecated APIs that are scheduled for removal in Kubernetes vX.XX. Upgrading your cluster before migrating to the updated APIs supported by vX.XX could cause application impact.", + "insightStatus": + { + "status": "PASSING", + "reason": "No deprecated API usage detected within the last 30 days.", + }, + }, + { + "id": "a1b2c3d4-5678-90ab-cdef-EXAMPLEaaaaa", + "name": "Cluster health issues", + "category": "UPGRADE_READINESS", + "kubernetesVersion": "X.XX", + "lastRefreshTime": 1734557314.000, + "lastTransitionTime": 1734557309.000, + "description": "Checks for any cluster health issues that prevent successful upgrade to the next Kubernetes version on EKS.", + "insightStatus": + { + "status": "PASSING", + "reason": "No cluster health issues detected.", + }, + }, + { + "id": "a1b2c3d4-5678-90ab-cdef-EXAMPLEbbbbb", + "name": "EKS add-on version compatibility", + "category": "UPGRADE_READINESS", + "kubernetesVersion": "X.XX", + "lastRefreshTime": 1734557314.000, + "lastTransitionTime": 1734557309.000, + "description": "Checks version of installed EKS add-ons to ensure they are compatible with the next version of Kubernetes. ", + "insightStatus": { "status": "PASSING", "reason": "All installed EKS add-on versions are compatible with next Kubernetes version."}, + }, + { + "id": "a1b2c3d4-5678-90ab-cdef-EXAMPLEccccc", + "name": "kube-proxy version skew", + "category": "UPGRADE_READINESS", + "kubernetesVersion": "X.XX", + "lastRefreshTime": 1734557314.000, + "lastTransitionTime": 1734557309.000, + "description": "Checks version of kube-proxy in cluster to see if upgrade would cause non compliance with supported Kubernetes kube-proxy version skew policy.", + "insightStatus": + { + "status": "PASSING", + "reason": "kube-proxy versions match the cluster control plane version.", + }, + }, + { + "id": "a1b2c3d4-5678-90ab-cdef-EXAMPLEddddd", + "name": "Deprecated APIs removed in Kubernetes vX.XX", + "category": "UPGRADE_READINESS", + "kubernetesVersion": "X.XX", + "lastRefreshTime": 1734557315.000, + "lastTransitionTime": 1734557309.000, + "description": "Checks for usage of deprecated APIs that are scheduled for removal in Kubernetes vX.XX. Upgrading your cluster before migrating to the updated APIs supported by vX.XX could cause application impact.", + "insightStatus": + { + "status": "PASSING", + "reason": "No deprecated API usage detected within the last 30 days.", + }, + }, + ], +"nextToken": null, +} +---- +. For descriptive information about an insight, run the following command. Make the following modifications to the command as needed and then run the modified command. ++ +** Replace [.replaceable]`region-code` with the code for your {aws} Region. +** Replace [.replaceable]`a1b2c3d4-5678-90ab-cdef-EXAMPLE22222` with an insight ID retrieved from listing the cluster insights. +** Replace [.replaceable]`my-cluster` with the name of your cluster. ++ +[source,bash,subs="verbatim,attributes,quotes"] +---- +aws eks describe-insight --region region-code --id [.replaceable]`a1b2c3d4-5678-90ab-cdef-EXAMPLE22222` --cluster-name my-cluster +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes",role="nocopy"] +---- +{ + "insight": + { + "id": "a1b2c3d4-5678-90ab-cdef-EXAMPLE22222", + "name": "Kubelet version skew", + "category": "UPGRADE_READINESS", + "kubernetesVersion": "1.27", + "lastRefreshTime": 1734557309.000, + "lastTransitionTime": 1734557309.000, + "description": "Checks for kubelet versions of worker nodes in the cluster to see if upgrade would cause non compliance with supported Kubernetes kubelet version skew policy.", + "insightStatus": + { + "status": "UNKNOWN", + "reason": "Unable to determine status of node kubelet versions.", + }, + "recommendation": "Upgrade your worker nodes to match the Kubernetes version of your cluster control plane.", + "additionalInfo": + { + "Kubelet version skew policy": "https://kubernetes.io/releases/version-skew-policy/#kubelet", + "Updating a managed node group": "https://docs.aws.amazon.com/eks/latest/userguide/update-managed-node-group.html", + }, + "resources": [], + "categorySpecificSummary": + { "deprecationDetails": [], "addonCompatibilityDetails": [] }, + }, +} +---- \ No newline at end of file diff --git a/latest/ug/clusters/view-support-status.adoc b/latest/ug/clusters/view-support-status.adoc deleted file mode 100644 index d6e089802..000000000 --- a/latest/ug/clusters/view-support-status.adoc +++ /dev/null @@ -1,20 +0,0 @@ -include::../attributes.txt[] -[.topic] -[[view-support-status,view-support-status.title]] -= View current cluster support period -:info_titleabbrev: View support period - -The *cluster support period* section of the {aws} console indicates if your cluster is _currently_ on standard or extended support. If your cluster support period is *Extended support*, you are being charged for EKS extended support. - -For more information about standard and extended support, see <>. - -. Navigate to the *Clusters* page in the EKS section of the {aws} Console. Confirm the console is set to the same {aws} region as the cluster you want to review. -. Review the *Support Period* column. If the value is *Standard support until...*, you are not currently being charged for extended support. You are within the standard support period. If the value is *Extended support...* this cluster is currently being charged for extended support. - - -[NOTE] -==== - -The *Support Period* cannot be retrieved with the {aws} API or CLI. - -==== diff --git a/latest/ug/clusters/windows-support.adoc b/latest/ug/clusters/windows-support.adoc index 6826baeff..e261da5ca 100644 --- a/latest/ug/clusters/windows-support.adoc +++ b/latest/ug/clusters/windows-support.adoc @@ -1,81 +1,57 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[windows-support,windows-support.title]] -= Deploy [.noloc]`Windows` nodes on EKS clusters -:info_doctype: section -:info_title: Deploy Windows nodes on EKS \ - clusters +[#windows-support] += Deploy Windows nodes on EKS clusters :info_titleabbrev: Enable Windows support -:info_abstract: Learn how to enable and manage Windows support for your Amazon EKS cluster to run \ - Windows containers alongside Linux containers. [abstract] -- Learn how to enable and manage Windows support for your Amazon EKS cluster to run Windows containers alongside Linux containers. -- -Before deploying [.noloc]`Windows` nodes, be aware of the following considerations. +Learn how to enable and manage Windows support for your Amazon EKS cluster to run Windows containers alongside Linux containers. + + +== Considerations +Before deploying Windows nodes, be aware of the following considerations. * EKS Auto Mode does not support Windows nodes -* You can use host networking on Windows nodes using `HostProcess` Pods. For more information, see https://kubernetes.io/docs/tasks/configure-pod-container/create-hostprocess-pod/[Create a Windows HostProcessPod] in the [.noloc]`Kubernetes` documentation. -* Amazon EKS clusters must contain one or more [.noloc]`Linux` or Fargate nodes to run core system [.noloc]`Pods` that only run on [.noloc]`Linux`, such as [.noloc]`CoreDNS`. +* You can use host networking on Windows nodes using `HostProcess` Pods. For more information, see https://kubernetes.io/docs/tasks/configure-pod-container/create-hostprocess-pod/[Create a Windows HostProcessPod] in the Kubernetes documentation. +* Amazon EKS clusters must contain one or more Linux or Fargate nodes to run core system Pods that only run on Linux, such as CoreDNS. * The `kubelet` and `kube-proxy` event logs are redirected to the `EKS Windows` Event Log and are set to a 200 MB limit. -* You can't use xref:security-groups-for-pods[Assign security groups to individual pods,linkend=security-groups-for-pods] with [.noloc]`Pods` running on [.noloc]`Windows` nodes. -* You can't use xref:cni-custom-network[custom networking,linkend=cni-custom-network] with [.noloc]`Windows` nodes. -* You can't use `IPv6` with [.noloc]`Windows` nodes. -* [.noloc]`Windows` nodes support one elastic network interface per node. By default, the number of [.noloc]`Pods` that you can run per [.noloc]`Windows` node is equal to the number of IP addresses available per elastic network interface for the node's instance type, minus one. For more information, see link:AWSEC2/latest/WindowsGuide/using-eni.html#AvailableIpPerENI[IP addresses per network interface per instance type,type="documentation"] in the _Amazon EC2 User Guide_. -* In an Amazon EKS cluster, a single service with a load balancer can support up to 1024 back-end [.noloc]`Pods`. Each [.noloc]`Pod` has its own unique IP address. The previous limit of 64 [.noloc]`Pods` is no longer the case, after https://github.com/microsoft/Windows-Containers/issues/93[a Windows Server update] starting with https://support.microsoft.com/en-us/topic/march-22-2022-kb5011551-os-build-17763-2746-preview-690a59cd-059e-40f4-87e8-e9139cc65de4[OS Build 17763.2746]. -* Windows containers aren't supported for Amazon EKS [.noloc]`Pods` on Fargate. +* You can't use <> with Pods running on Windows nodes. +* You can't use <> with Windows nodes. +* You can't use `IPv6` with Windows nodes. +* Windows nodes support one elastic network interface per node. By default, the number of Pods that you can run per Windows node is equal to the number of IP addresses available per elastic network interface for the node's instance type, minus one. For more information, see link:AWSEC2/latest/WindowsGuide/using-eni.html#AvailableIpPerENI[IP addresses per network interface per instance type,type="documentation"] in the _Amazon EC2 User Guide_. +* In an Amazon EKS cluster, a single service with a load balancer can support up to 1024 back-end Pods. Each Pod has its own unique IP address. The previous limit of 64 Pods is no longer the case, after https://github.com/microsoft/Windows-Containers/issues/93[a Windows Server update] starting with https://support.microsoft.com/en-us/topic/march-22-2022-kb5011551-os-build-17763-2746-preview-690a59cd-059e-40f4-87e8-e9139cc65de4[OS Build 17763.2746]. +* Windows containers aren't supported for Amazon EKS Pods on Fargate. * You can't use Amazon EKS Hybrid Nodes with Windows as the operating system for the host. * You can't retrieve logs from the `vpc-resource-controller` Pod. You previously could when you deployed the controller to the data plane. * There is a cool down period before an `IPv4` address is assigned to a new Pod. This prevents traffic from flowing to an older Pod with the same `IPv4` address due to stale `kube-proxy` rules. -* The source for the controller is managed on [.noloc]`GitHub`. To contribute to, or file issues against the controller, visit the https://github.com/aws/amazon-vpc-resource-controller-k8s[project] on [.noloc]`GitHub`. -* When specifying a custom AMI ID for [.noloc]`Windows` managed node groups, add `eks:kube-proxy-windows` to your {aws} IAM Authenticator configuration map. For more information, see <>. +* The source for the controller is managed on GitHub. To contribute to, or file issues against the controller, visit the https://github.com/aws/amazon-vpc-resource-controller-k8s[project] on GitHub. +* When specifying a custom AMI ID for Windows managed node groups, add `eks:kube-proxy-windows` to your {aws} IAM Authenticator configuration map. For more information, see <>. * If preserving your available IPv4 addresses is crucial for your subnet, refer to https://aws.github.io/aws-eks-best-practices/windows/docs/networking/#ip-address-management[EKS Best Practices Guide - Windows Networking IP Address Management] for guidance. - - -* An existing cluster. The cluster must be running one of the [.noloc]`Kubernetes` versions and platform versions listed in the following table. Any [.noloc]`Kubernetes` and platform versions later than those listed are also supported. +* Considerations for EKS Access Entries +** Access Entries for use with Windows nodes need the type of `EC2_WINDOWS`. For more information, see <>. + -[[windows-support-platform-versions]] -[cols="1,1", options="header"] -|=== -|Kubernetes version -|Platform version - -|1.31 -|eks.4 - -|1.30 -|eks.2 - -|1.29 -|eks.1 - -|1.28 -|eks.1 - -|1.27 -|eks.1 - -|1.26 -|eks.1 +To create an access entry for a Windows node: ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks create-access-entry --cluster-name my-cluster --principal-arn {arn-aws}iam::111122223333:role/ --type EC2_Windows +---- -|1.25 -|eks.1 +== Prerequisites -|1.24 -|eks.2 -|=== -* Your cluster must have at least one (we recommend at least two) [.noloc]`Linux` node or Fargate [.noloc]`Pod` to run [.noloc]`CoreDNS`. If you enable legacy [.noloc]`Windows` support, you must use a [.noloc]`Linux` node (you can't use a Fargate [.noloc]`Pod`) to run [.noloc]`CoreDNS`. +* An existing cluster. +* Your cluster must have at least one (we recommend at least two) Linux node or Fargate Pod to run CoreDNS. If you enable legacy Windows support, you must use a Linux node (you can't use a Fargate Pod) to run CoreDNS. * An existing <>. - -[[enable-windows-support,enable-windows-support.title]] -== Enable [.noloc]`Windows` support -. If you don't have Amazon Linux nodes in your cluster and use security groups for [.noloc]`Pods`, skip to the next step. Otherwise, confirm that the `AmazonEKSVPCResourceController` managed policy is attached to your <>. Replace [.replaceable]`eksClusterRole` with your cluster role name. +[#enable-windows-support] +== Enable Windows support +. If you don't have Amazon Linux nodes in your cluster and use security groups for Pods, skip to the next step. Otherwise, confirm that the `AmazonEKSVPCResourceController` managed policy is attached to your <>. Replace [.replaceable]`eksClusterRole` with your cluster role name. + [source,bash,subs="verbatim,attributes"] ---- @@ -101,7 +77,7 @@ An example output is as follows. ---- + If the policy is attached, as it is in the previous output, skip the next step. -. Attach the *link:aws-managed-policy/latest/reference/AmazonEKSVPCResourceController.html[AmazonEKSVPCResourceController,type="documentation"]* managed policy to your <>. Replace [.replaceable]`eksClusterRole` with your cluster role name. +. Attach the *link:aws-managed-policy/latest/reference/AmazonEKSVPCResourceController.html[AmazonEKSVPCResourceController,type="documentation"]* managed policy to your <>. Replace [.replaceable]`eksClusterRole` with your cluster role name. + [source,bash,subs="verbatim,attributes"] ---- @@ -109,7 +85,8 @@ aws iam attach-role-policy \ --role-name eksClusterRole \ --policy-arn {arn-aws}iam::aws:policy/AmazonEKSVPCResourceController ---- -. Create a file named [.replaceable]`vpc-resource-controller-configmap.yaml` with the following contents. +. Update the VPC CNI ConfigMap to enable Windows IPAM. Note, if the VPC CNI is installed on your cluster using a Helm chart or as an Amazon EKS Add-on you may not be able to directly modify the ConfigMap. For information on configuring Amazon EKS Add-ons, see <>. +.. Create a file named [.replaceable]`vpc-resource-controller-configmap.yaml` with the following contents. + [source,yaml,subs="verbatim,attributes"] ---- @@ -121,13 +98,14 @@ metadata: data: enable-windows-ipam: "true" ---- -. Apply the `ConfigMap` to your cluster. +.. Apply the `ConfigMap` to your cluster. + [source,bash,subs="verbatim,attributes"] ---- kubectl apply -f vpc-resource-controller-configmap.yaml ---- -. Verify that your `aws-auth` `ConfigMap` contains a mapping for the instance role of the [.noloc]`Windows` node to include the `eks:kube-proxy-windows` RBAC permission group. You can verify by running the following command. +. If your cluster has the authentication mode set to enable the `aws-auth` configmap: +** Verify that your `aws-auth` `ConfigMap` contains a mapping for the instance role of the Windows node to include the `eks:kube-proxy-windows` RBAC permission group. You can verify by running the following command. + [source,bash,subs="verbatim,attributes"] ---- @@ -155,14 +133,14 @@ data: ---- + You should see `eks:kube-proxy-windows` listed under groups. If the group isn't specified, you need to update your `ConfigMap` or create it to include the required group. For more information about the `aws-auth` `ConfigMap`, see <>. +. If your cluster has the authentication mode set to disable the `aws-auth` configmap, then you can use EKS Access Entries. Create a new node role for use with Windows instances, and EKS will automatically create an access entry of type `EC2_WINDOWS`. - -[[windows-support-pod-deployment,windows-support-pod-deployment.title]] +[#windows-support-pod-deployment] == Deploy Windows Pods When you deploy Pods to your cluster, you need to specify the operating system that they use if you're running a mixture of node types. -For [.noloc]`Linux` [.noloc]`Pods`, use the following node selector text in your manifests. +For Linux Pods, use the following node selector text in your manifests. [source,yaml,subs="verbatim,attributes"] ---- @@ -171,7 +149,7 @@ nodeSelector: kubernetes.io/arch: amd64 ---- -For [.noloc]`Windows` [.noloc]`Pods`, use the following node selector text in your manifests. +For Windows Pods, use the following node selector text in your manifests. [source,yaml,subs="verbatim,attributes"] ---- @@ -180,25 +158,25 @@ nodeSelector: kubernetes.io/arch: amd64 ---- -You can deploy a <> to see the node selectors in use. +You can deploy a <> to see the node selectors in use. -[[windows-support-pod-density,windows-support-pod-density.title]] -== Support higher [.noloc]`Pod` density on Windows nodes +[#windows-support-pod-density] +== Support higher Pod density on Windows nodes -In Amazon EKS, each [.noloc]`Pod` is allocated an `IPv4` address from your VPC. Due to this, the number of [.noloc]`Pods` that you can deploy to a node is constrained by the available IP addresses, even if there are sufficient resources to run more [.noloc]`Pods` on the node. Since only one elastic network interface is supported by a Windows node, by default, the maximum number of available IP addresses on a Windows node is equal to: +In Amazon EKS, each Pod is allocated an `IPv4` address from your VPC. Due to this, the number of Pods that you can deploy to a node is constrained by the available IP addresses, even if there are sufficient resources to run more Pods on the node. Since only one elastic network interface is supported by a Windows node, by default, the maximum number of available IP addresses on a Windows node is equal to: [source,bash,subs="verbatim,attributes"] ---- Number of private IPv4 addresses for each interface on the node - 1 ---- -One IP address is used as the primary IP address of the network interface, so it can't be allocated to [.noloc]`Pods`. +One IP address is used as the primary IP address of the network interface, so it can't be allocated to Pods. -You can enable higher [.noloc]`Pod` density on Windows nodes by enabling IP prefix delegation. This feature enables you to assign a `/28` `IPv4` prefix to the primary network interface, instead of assigning secondary `IPv4` addresses. Assigning an IP prefix increases the maximum available `IPv4` addresses on the node to: +You can enable higher Pod density on Windows nodes by enabling IP prefix delegation. This feature enables you to assign a `/28` `IPv4` prefix to the primary network interface, instead of assigning secondary `IPv4` addresses. Assigning an IP prefix increases the maximum available `IPv4` addresses on the node to: [source,bash,subs="verbatim,attributes"] ---- (Number of private IPv4 addresses assigned to the interface attached to the node - 1) * 16 ---- -With this significantly larger number of available IP addresses, available IP addresses shouldn't limit your ability to scale the number of [.noloc]`Pods` on your nodes. For more information, see <>. +With this significantly larger number of available IP addresses, available IP addresses shouldn't limit your ability to scale the number of Pods on your nodes. For more information, see <>. diff --git a/latest/ug/clusters/zone-shift-enable.adoc b/latest/ug/clusters/zone-shift-enable.adoc index 78107e9b6..d95b9a144 100644 --- a/latest/ug/clusters/zone-shift-enable.adoc +++ b/latest/ug/clusters/zone-shift-enable.adoc @@ -1,9 +1,7 @@ -//!!NODE_ROOT
[.topic] -[[zone-shift-enable,zone-shift-enable.title]] -= Enable EKS Zonal Shift to avoid impaired Availability Zones -:info_doctype: section -:info_titleabbrev: Enable Zonal Shift +[#zone-shift-enable] += Enable EKS zonal shift to avoid impaired Availability Zones +:info_titleabbrev: Enable zonal shift :aws: pass:q[[.shared]``AWS``] @@ -11,18 +9,23 @@ Amazon Application Recovery Controller (ARC) helps you manage and coordinate recovery for your applications across Availability Zones (AZs) and works with many services, including Amazon EKS. With EKS support for ARC zonal shift, you can shift in-cluster network traffic away from an impaired AZ. You can also authorize {aws} to monitor the health of your AZs and temporarily shift network traffic away from an unhealthy AZ on your behalf. -*How to use EKS Zonal Shift:* +*How to use EKS zonal shift:* . Enable your EKS cluster with Amazon Application Recovery Controller (ARC). This is done at the cluster level using the Amazon EKS Console, the {aws} CLI, CloudFormation, or eksctl. -. Once enabled, you can manage zonal shifts or zonal autoshifts using the ARC Console, the {aws} CLI, or the Zonal Shift and Zonal Autoshift APIs. +. Once enabled, you can manage zonal shifts or zonal autoshifts using the ARC Console, the {aws} CLI, or the zonal shift and zonal autoshift APIs. -Note that after you register an EKS cluster with ARC, you still need to configure ARC. For example, you can use the ARC console to configure Zonal Autoshift. +Note that after you register an EKS cluster with ARC, you still need to configure ARC. For example, you can use the ARC console to configure zonal autoshift. -For more detailed information about how EKS Zonal Shift works, and how to design your workloads to handle impaired availability zones, see <>. +For more detailed information about how EKS zonal shift works, and how to design your workloads to handle impaired availability zones, see <>. -*Considerations:* +[#zone-shift-enable-considerations] +== Considerations -* EKS Auto Mode does not support Amazon Application Recovery Controller, Zonal Shift, and Zonal Autoshift +* EKS Auto Mode does not support Amazon Application Recovery Controller, zonal shift, and zonal autoshift. +// Ref: 46657 +* We recommend waiting at least 60 seconds between zonal shift operations to ensure proper processing of each request. ++ +When attempting to perform zonal shifts in quick succession (within 60 seconds of each other), the Amazon EKS service may not properly process all shift requests. This is due to the current polling mechanism that updates the cluster's zonal state. If you need to perform multiple zonal shifts, ensure there is adequate time between operations for the system to process each change. == What is Amazon Application Recovery Controller? @@ -34,7 +37,7 @@ link:r53recovery/latest/dg/what-is-route53-recovery.html["Learn more about Amazo Zonal shift is a capability in ARC that allows you to move traffic for a resource like an EKS cluster or an Elastic Load Balancer away from an Availability Zone in an {aws} Region to quickly mitigate an issue and quickly recover your application. You might choose to shift traffic, for example, because a bad deployment is causing latency issues, or because the Availability Zone is impaired. A zonal shift requires no advance configuration steps. -link:r53recovery/latest/dg/arc-zonal-shift.how-it-works.html["Learn more about ARC Zonal Shift", type="documentation"] +link:r53recovery/latest/dg/arc-zonal-shift.how-it-works.html["Learn more about ARC zonal shift", type="documentation"] == What is zonal autoshift? @@ -42,28 +45,28 @@ Zonal autoshift is a capability in ARC that you can enable to authorize {aws} to {aws} ends autoshifts when indicators show that there is no longer an issue or potential issue. -link:r53recovery/latest/dg/arc-zonal-autoshift.how-it-works.html["Learn more about ARC Zonal Autoshift", type="documentation"] +link:r53recovery/latest/dg/arc-zonal-autoshift.how-it-works.html["Learn more about ARC zonal autoshift", type="documentation"] == What does EKS do during an autoshift? EKS updates networking configurations to avoid directing traffic to impaired AZs. Additionally, if you are using Managed Node Groups, EKS will only launch new nodes in the healthy AZs during a zonal shift. When the shift expires or gets cancelled, the networking configurations will be restored to include the AZ that was previously detected as unhealthy. -xref:zone-shift[Learn more about EKS Zonal Shift]. +<>. -[[zone-shift-enable-steps,zone-shift-enable-steps.title]] +[#zone-shift-enable-steps] == Register EKS cluster with Amazon Application Recovery Controller (ARC) ({aws} console) . Find the name and region of the EKS cluster you want to register with ARC. . Navigate to the link:eks[EKS console,type="console"] in that region, and select your cluster. . On the *Cluster info* page, select the *Overview* tab. . Under the *Zonal shift* heading, select the *Manage* button. -. Select *enable* or *disable* for _EKS Zonal Shift_. +. Select *enable* or *disable* for _EKS zonal shift_. Now your EKS cluster is registered with ARC. -If you want {aws} to detect and avoid impaired availability zones, you need to configure ARC Zonal Autoshift. For example, you can do this in the ARC console. +If you want {aws} to detect and avoid impaired availability zones, you need to configure ARC zonal autoshift. For example, you can do this in the ARC console. == Next Steps -* Learn how to link:r53recovery/latest/dg/arc-zonal-autoshift.start-cancel.html["enable zonal autoshift",type="documentation"] -* Learn how to manually link:r53recovery/latest/dg/arc-zonal-shift.start-cancel.html["start a zonal shift",type="documentation"] +* Learn how to link:r53recovery/latest/dg/arc-zonal-autoshift.start-cancel.html["enable zonal autoshift",type="documentation"]. +* Learn how to manually link:r53recovery/latest/dg/arc-zonal-shift.start-cancel.html["start a zonal shift",type="documentation"]. diff --git a/latest/ug/clusters/zone-shift.adoc b/latest/ug/clusters/zone-shift.adoc index 4153ff050..e75b75df4 100644 --- a/latest/ug/clusters/zone-shift.adoc +++ b/latest/ug/clusters/zone-shift.adoc @@ -1,102 +1,99 @@ -//!!NODE_ROOT
[.topic] -[[zone-shift,zone-shift.title]] -= Learn about Amazon Application Recovery Controller's (ARC) Zonal Shift in Amazon EKS -:info_doctype: section -:info_titleabbrev: Learn about Zonal Shift +[#zone-shift] += Learn about Amazon Application Recovery Controller (ARC) zonal shift in Amazon EKS +:info_titleabbrev: Learn about zonal shift :aws: pass:q[[.shared]``AWS``] -:imagesdir: images/ //GDC: remove use of "failure" -Kubernetes has native features that enable you to make your applications more resilient to events such as the degraded health or impairment of an Availability Zone (AZ). When running your workloads in an Amazon EKS cluster, you can further improve your application environment's fault tolerance and application recovery using link:r53recovery/latest/dg/arc-zonal-shift.html["Amazon Application Recovery Controller's (ARC) zonal shift",type="documentation"] or link:r53recovery/latest/dg/arc-zonal-autoshift.html["zonal autoshift",type="documentation"]. ARC zonal shift is designed to be a temporary measure that allows you to move traffic for a resource away from an impaired AZ until the zonal shift expires or you cancel it. You can extend the zonal shift if necessary. +Kubernetes has native features that enable you to make your applications more resilient to events such as the degraded health or impairment of an Availability Zone (AZ). When you run your workloads in an Amazon EKS cluster, you can further improve your application environment's fault tolerance and application recovery by using link:r53recovery/latest/dg/arc-zonal-shift.html["Amazon Application Recovery Controller (ARC) zonal shift",type="documentation"] or link:r53recovery/latest/dg/arc-zonal-autoshift.html["zonal autoshift",type="documentation"]. ARC zonal shift is designed to be a temporary measure that enables you to move traffic for a resource away from an impaired AZ until the zonal shift expires or you cancel it. You can extend the zonal shift, if necessary. -You can start a zonal shift for an EKS cluster, or you can allow {aws} to do it for you by enabling zonal autoshift. This shift updates the flow of east-to-west network traffic in your cluster to only consider network endpoints for Pods running on worker nodes in healthy AZs. Additionally, any ALB or NLB handling ingress traffic for applications in your EKS cluster will automatically route traffic to targets in the healthy AZs. For those customers seeking the highest availability goals, in the case that an AZ becomes impaired, it can be important to be able to steer all traffic away from the impaired AZ until it recovers. For this, you can also link:r53recovery/latest/dg/arc-zonal-shift.resource-types.html["_enable an ALB or NLB with ARC zonal shift_",type="documentation"]. +You can start a zonal shift for an EKS cluster, or you can allow {aws} to shift traffic for you by enabling zonal autoshift. This shift updates the flow of east-to-west network traffic in your cluster to only consider network endpoints for Pods running on worker nodes in healthy AZs. Additionally, any ALB or NLB handling ingress traffic for applications in your EKS cluster will automatically route traffic to targets in the healthy AZs. For those customers seeking the highest availability goals, in the case that an AZ becomes impaired, it can be important to be able to steer all traffic away from the impaired AZ until it recovers. For this, you can also link:r53recovery/latest/dg/arc-zonal-shift.resource-types.html["_enable an ALB or NLB with ARC zonal shift_",type="documentation"]. -== Understanding East-West Network Traffic Flow Between Pods +== Understanding east-west network traffic flow between Pods The following diagram illustrates two example workloads, Orders, and Products. The purpose of this example is to show how workloads and Pods in different AZs communicate. -image::zs-traffic-flow-before-1.png[Illustration of network traffic] +image::images/zs-traffic-flow-before-1.png[Illustration of network traffic] -image::zs-traffic-flow-before-2.png[Illustration of network traffic] +image::images/zs-traffic-flow-before-2.png[Illustration of network traffic] -. For Orders to communicate with Products, it must first resolve the DNS name of the destination service. Orders will communicate with CoreDNS to fetch the virtual IP address (Cluster IP) for that Service. Once Orders resolves the Products service name, it sends traffic to that target IP. -. The kube-proxy runs on every node in the cluster and continuously watches the https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/[EndpointSlices] for Services. When a Service is created, an EndpointSlice is created and managed in the background by the EndpointSlice controller. Each EndpointSlice has a list or table of endpoints containing a subset of Pod addresses along with the nodes that they're running on. The kube-proxy sets up routing rules for each of these Pod endpoints using `iptables` on the nodes. The kube-proxy is also responsible for a basic form of load balancing by redirecting traffic destined to a service's Cluster IP to instead be sent to a Pod's IP address directly. The kube-proxy does this by rewriting the destination IP on the outgoing connection. -. The network packets are then sent to the Products Pod in AZ 2 via the ENIs on the respective nodes (as depicted in the diagram above). +. For Orders to communicate with Products, Orders must first resolve the DNS name of the destination service. Orders communicates with CoreDNS to fetch the virtual IP address (Cluster IP) for that service. After Orders resolves the Products service name, it sends traffic to that target IP address. +. The kube-proxy runs on every node in the cluster and continuously watches https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/[EndpointSlices] for services. When a service is created, an EndpointSlice is created and managed in the background by the EndpointSlice controller. Each EndpointSlice has a list or table of endpoints that contains a subset of Pod addresses, along with the nodes that they're running on. The kube-proxy sets up routing rules for each of these Pod endpoints using `iptables` on the nodes. The kube-proxy is also responsible for a basic form of load balancing, redirecting traffic destined to a service's Cluster IP address to instead be sent to a Pod's IP address directly. The kube-proxy does this by rewriting the destination IP address on the outgoing connection. +. The network packets are then sent to the Products Pod in AZ 2 by using the ENIs on the respective nodes, as shown in the earlier diagram. -=== Understanding ARC Zonal Shift in EKS +=== Understanding ARC zonal shift in Amazon EKS -In the case that there is an AZ impairment in your environment, you can initiate a zonal shift for your EKS cluster environment. Alternatively, you can allow {aws} to manage this for you with zonal autoshift. With zonal autoshift, {aws} will monitor the overall AZ health and respond to a potential AZ impairment by automatically shifting traffic away from the impaired AZ in your cluster environment. +If there is an AZ impairment in your environment, you can initiate a zonal shift for your EKS cluster environment. Alternatively, you can allow {aws} to manage shifting traffic for you with zonal autoshift. With zonal autoshift, {aws} monitors overall AZ health and responds to a potential AZ impairment by automatically shifting traffic away from the impaired AZ in your cluster environment. -Once your EKS cluster zonal shift enabled with ARC, you can trigger a zonal shift or enable zonal autoshift using the ARC Console, the {aws} CLI, or the zonal shift and zonal autoshift APIs. -During an EKS zonal shift, the following will automatically take place: +After your Amazon EKS cluster has zonal shift enabled with ARC, you can start a zonal shift or enable zonal autoshift by using the ARC Console, the {aws} CLI, or the zonal shift and zonal autoshift APIs. +During an EKS zonal shift, the following is performed automatically: -* All the nodes in the impacted AZ will be cordoned. This will prevent the Kubernetes Scheduler from scheduling new Pods onto the nodes in the unhealthy AZ. -* If you're using link:eks/latest/userguide/managed-node-groups.html["Managed Node Groups",type="documentation"], link:autoscaling/ec2/userguide/auto-scaling-benefits.html#AutoScalingBehavior.InstanceUsage["_Availability Zone rebalancing_",type="documentation"] will be suspended, and your Auto Scaling Group (ASG) will be updated to ensure that new EKS Data Plane nodes are only launched in the healthy AZs. -* The nodes in the unhealthy AZ will not be terminated and the Pods will not be evicted from these nodes. This is to ensure that when a zonal shift expires or gets cancelled, your traffic can be safely returned to the AZ which still has full capacity -* The EndpointSlice controller will find all the Pod endpoints in the impaired AZ and remove them from the relevant EndpointSlices. This will ensure that only Pod endpoints in healthy AZs are targeted to receive network traffic. When a zonal shift is cancelled or expires, the EndpointSlice controller will update the EndpointSlices to include the endpoints in the restored AZ. +* All the nodes in the impacted AZ are cordoned. This prevents the Kubernetes Scheduler from scheduling new Pods onto nodes in the unhealthy AZ. +* If you're using <>, link:autoscaling/ec2/userguide/auto-scaling-benefits.html#AutoScalingBehavior.InstanceUsage["_Availability Zone rebalancing_",type="documentation"] is suspended, and your Auto Scaling group is updated to ensure that new EKS data plane nodes are only launched in healthy AZs. +* The nodes in the unhealthy AZ are not terminated, and Pods are not evicted from the nodes. This ensures that when a zonal shift expires or is canceled, your traffic can be safely returned to the AZ for full capacity. +* The EndpointSlice controller finds all Pod endpoints in the impaired AZ, and removes them from the relevant EndpointSlices. This ensures that only Pod endpoints in healthy AZs are targeted to receive network traffic. When a zonal shift is canceled or expires, the EndpointSlice controller updates the EndpointSlices to include the endpoints in the restored AZ. -The diagrams below depicts a high level flow of how EKS zonal shift ensures that only healthy Pod endpoints are targeted in your cluster environment. +The following diagrams provide a high level overview of how EKS zonal shift ensures that only healthy Pod endpoints are targeted in your cluster environment. -image::zs-traffic-flow-after-1.png[Illustration of network traffic] +image::images/zs-traffic-flow-after-1.png[Illustration of network traffic] -image::zs-traffic-flow-after-2.png[Illustration of network traffic] +image::images/zs-traffic-flow-after-2.png[Illustration of network traffic] -== EKS Zonal Shift Requirements +== EKS zonal shift requirements -For zonal shift to work successfully in EKS, you need to setup your cluster environment to be resilient to an AZ impairment beforehand. Below is a list of the steps that you have to follow. +For zonal shift to work successfully with EKS, you must set up your cluster environment ahead of time to be resilient to an AZ impairment. The following is a list of configuration options that help to ensure resilience. * Provision your cluster's worker nodes across multiple AZs -* Provision enough compute capacity to withstand removal of a single AZ -* Pre-scale your Pods (including CoreDNS) in every AZ -* Spread multiple Pod replicas across all AZs to ensure that shifting away from a single AZ will leave you with sufficient capacity -* Co-locate interdependent or related Pods in the same AZ -* Test that your cluster environment would work as expected with on less AZ by manually starting a zonal shift. Alternatively, you can enable zonal autoshift and reply on the autoshift practice runs. This is not required for zonal shift to work in EKS but it's strongly recommended. +* Provision enough compute capacity to accommodate removal of a single AZ +* Pre-scale your Pods, including CoreDNS, in every AZ +* Spread multiple Pod replicas across all AZs, to help ensure that when you shift away from a single AZ, you'll still have sufficient capacity +* Colocate interdependent or related Pods in the same AZ +* Test that your cluster environment works as expected without one AZ by manually starting a zonal shift away from an AZ. Alternatively, you can enable zonal autoshift and rely on autoshift practice runs. Testing with manual or practice zonal shifts is not required for zonal shift to work in EKS but it's strongly recommended. -=== Provision Your EKS Worker Nodes Across Multiple AZs +=== Provision your EKS worker nodes across multiple Availability Zones -{aws} Regions have multiple, separate locations with physical data centers known as Availability Zones (AZs). AZs are designed to be physically isolated from one another to avoid simultaneous impact that could affect an entire Region. When provisioning an EKS cluster, you should deploy your worker nodes across multiple AZs in a Region. This will make your cluster environment more resilient to the impairment of a single AZ, and allow you to maintain high availability (HA) of your applications running in the other AZs. When you start a zonal shift away from the impacted AZ, your EKS environment's in-cluster network will automatically update to only use healthy AZs, while maintaining a highly available posture for your cluster. +{aws} Regions have multiple, separate locations with physical data centers, known as Availability Zones (AZs). AZs are designed to be physically isolated from one another to avoid simultaneous impact that could affect an entire Region. When you provision an EKS cluster, we recommend that you deploy your worker nodes across multiple AZs in a Region. This helps to make your cluster environment more resilient to the impairment of a single AZ, and allows you to maintain high availability for your applications that run in the other AZs. When you start a zonal shift away from the impacted AZ, your EKS environment's in-cluster network automatically updates to only use healthy AZs, to help maintaining high availability for your cluster. -Ensuring that you have such a multi-AZ setup for your EKS environment will enhance the overall reliability of your system. However, multi-AZ environments can play a significant role in how application data is transferred and processed, which will in turn have an impact on your environment's network charges. In particular, frequent egress cross-zone traffic (traffic distributed between AZs) can have a major impact on your network-related costs. You can apply different strategies to control the amount of cross-zone traffic between Pods in your EKS cluster and drive down the associated costs. Please refer to https://aws.github.io/aws-eks-best-practices/cost_optimization/cost_opt_networking/[_this best practice guide_] for more details on how to optimize network costs when running highly available EKS environments. +Ensuring that you have a multi-AZ setup for your EKS environment enhances the overall reliability of your system. However, multi-AZ environments influence how application data is transferred and processed, which in turn has an impact on your environment's network charges. Specifically, frequent egress cross-zone traffic (traffic distributed between AZs) can have a major impact on your network-related costs. You can apply different strategies to control the amount of cross-zone traffic between Pods in your EKS cluster and drive down the associated costs. For more information on how to optimize network costs when running highly available EKS environments, see https://aws.github.io/aws-eks-best-practices/cost_optimization/cost_opt_networking/[_these best practices_]. -The diagram below depicts a highly available EKS environment with 3 healthy AZs. +The following diagram illustrates a highly-available EKS environment with three healthy AZs. -image::zs-ha-before-failure.png[Illustration of network] +image::images/zs-ha-before-failure.png[Illustration of network] -The diagram below depicts how an EKS environment with 3 AZs is resilient to an AZ impairment and remains highly available because of the 2 other healthy AZs. +The following diagram illustrates how an EKS environment with three AZs is resilient to an AZ impairment and remains highly available because there are two remaining healthy AZs. -image::zs-ha-after-failure.png[Illustration of network] +image::images/zs-ha-after-failure.png[Illustration of network] -=== Provision Enough Compute Capacity to Withstand Removal of a Single AZ +=== Provision enough compute capacity to withstand removal of a single Availability Zone -To optimize resource utilization and costs for your compute infrastructure in the EKS Data Plane, it's a best practice to align compute capacity with your workload requirements. However, *if all your worker nodes are at full capacity*, then this makes you reliant on having new worker nodes added to the EKS Data Plane before new Pods can be scheduled. When running critical workloads, it is generally always a good practice to run with redundant capacity online to handle eventualities such as sudden increases in load, node health issues, etc. If you plan to use zonal shift, you are planning to remove an entire AZ of capacity so you need to adjust your redundant compute capacity so that it's sufficient to handle the load even with an AZ offline. +To optimize resource utilization and costs for your compute infrastructure in the EKS data plane, it's a best practice to align compute capacity with your workload requirements. However, *if all your worker nodes are at full capacity*, you are reliant on having new worker nodes added to the EKS data plane before new Pods can be scheduled. When you run critical workloads, it is generally a good practice to run with redundant capacity online to handle scenarios such as sudden increases in load and node health issues. If you plan to use zonal shift, you are planning to remove an entire AZ of capacity when there's an impairment. This means that you must adjust your redundant compute capacity so that it's sufficient to handle the load even with one of the AZs offline. -When scaling your compute, the process of adding new nodes to the EKS Data Plane will take some time which can have implications on the real-time performance and availability of your applications, especially in the event of a zonal impairment. Your EKS environment should be resilient to absorb the load of losing an AZ to avoid a degraded experience for your end users or clients. This means minimizing or eliminating any lag between the time at which a new Pod is needed and when it's actually scheduled on a worker node. +When you scale your compute resources, the process of adding new nodes to the EKS data plane takes some time. This can have implications on the real-time performance and availability of your applications, especially in the event of a zonal impairment. Your EKS environment should be able to absorb the load of losing one AZ without resulting in a degraded experience for your end users or clients. This means minimizing or eliminating lag between the time when a new Pod is needed and when it's actually scheduled on a worker node. -Additionally, in the event of a zonal impairment, you should mitigate the risk of a potential compute capacity constraint which would prevent newly required nodes from being added to your EKS Data Plane in the healthy AZs. +Additionally, when there's a zonal impairment, you should aim to mitigate the risk of running into a compute capacity constraint that would prevent newly-required nodes from being added to your EKS data plane in the healthy AZs. -To accomplish this, you should over-provision compute capacity in some of the worker nodes in each of the AZs so that the Kubernetes Scheduler has pre-existing capacity available for new Pod placements, especially when you have one less AZ in your environment. +To accomplish reduce the risk of these potential negative impacts, we recommend that you over-provision compute capacity in some of the worker nodes in each of the AZs. By doing this, the Kubernetes Scheduler has pre-existing capacity available for new Pod placements, which is especially important when you lose one of the AZs in your environment. -=== Run & Spread Multiple Pod Replicas Across AZs +=== Run and spread multiple Pod replicas across Availability Zones -Kubernetes allows you to pre-scale your workloads by running multiple instances (Pod replicas) of a single application. Running multiple Pod replicas for an application eliminates a single point of failure and increases its overall performance by reducing the resource strain on a single replica. However, to have both high availability and better fault tolerance for your applications, you should run and spread multiple replicas of an application across different failure domains (also referred to as topology domains) in this case AZs. With https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/[topology spread constraints], you can setup your applications to have pre-existing, static stability so that, in the case of an AZ impairment, you'll have enough replicas in the healthy AZs to immediately handle any additional spike or surge in traffic that they may experience. +Kubernetes allows you to pre-scale your workloads by running multiple instances (Pod replicas) of a single application. Running multiple Pod replicas for an application eliminates single points of failure and increases overall performance by reducing the resource strain on a single replica. However, to have both high availability and better fault tolerance for your applications, we recommend that you run multiple replicas of your application and spread the replicas across different failure domains, also referred to as topology domains. The failure domains in this scenario are the Availability Zones. By using https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/[topology spread constraints], you can set up your applications to have pre-existing, static stability. Then, when there's an AZ impairment, your environment will have enough replicas in healthy AZs to immediately handle any spikes or surges in traffic. -The diagram below depicts an EKS environment with east-to-west traffic flow when all AZs are healthy. +The following diagram illustrates an EKS environment that has east-to-west traffic flow when all AZs are healthy. -image::zs-spread-constraints.png[Illustration of network] +image::images/zs-spread-constraints.png[Illustration of network] -The diagram below depicts an EKS environment with east-to-west traffic flow when a single AZ fails, and you initiate a zonal shift. +The following diagram illustrates an EKS environment that has east-to-west traffic flow where a single AZ has failed and you have started a zonal shift. -image::zs-spread-constraints-2.png[Illustration of network] +image::images/zs-spread-constraints-2.png[Illustration of network] -The code snippet below is an example of how to setup your workload with this Kubernetes feature. +The following code snippet is an example of how to set up your workload with multiple replicas in Kubernetes. [source,yaml] ---- @@ -125,11 +122,11 @@ spec: ---- -Most important, you should run multiple replicas of your DNS server software (CoreDNS/kube-dns) and apply similar topology spread constraints if they are not already configured by default. This will help ensure that you have enough DNS Pods in healthy AZs to continue handling service discovery requests for other communicating Pods in the cluster if there's a single AZ impairment. The link:eks/latest/userguide/managing-coredns.html["CoreDNS EKS add-on",type="documentation"] has default settings for the CoreDNS Pods to be spread across your cluster's Availability Zones if there are nodes in multiple AZs available. You can also replace these default settings with your own custom configurations. +Most importantly, you should run multiple replicas of your DNS server software (CoreDNS/kube-dns) and apply similar topology spread constraints, if they are not configured by default. This helps to ensure that, if there's a single AZ impairment, you have enough DNS Pods in healthy AZs to continue handling service discovery requests for other communicating Pods in the cluster. The <> has default settings for the CoreDNS Pods that ensure that, if there are nodes in multiple AZs available, they are spread across your cluster's Availability Zones. If you like, you can replace these default settings with your own custom configurations. -When installing https://github.com/coredns/helm/tree/master[CoreDNS with Helm], you can update the `replicaCount` in the https://github.com/coredns/helm/blob/master/charts/coredns/values.yaml[values.yaml file] to ensure that you have a sufficient number of replicas in each AZ. In addition, to ensure that these replicas are spread across the different AZs in your cluster environment, you should update the `topologySpreadConstraints` property in the same values.yaml file. The code snippet below demonstrates how to configure CoreDNS for this. +When you install https://github.com/coredns/helm/tree/master[CoreDNS with Helm], you can update the `replicaCount` in the https://github.com/coredns/helm/blob/master/charts/coredns/values.yaml[values.yaml file] to ensure that you have sufficient replicas in each AZ. In addition, to ensure that these replicas are spread across the different AZs in your cluster environment, make sure that you update the `topologySpreadConstraints` property in the same `values.yaml` file. The following code snippet illustrates how you can configure CoreDNS to do this. -**CoreDNS Helm values.yaml** +*CoreDNS Helm values.yaml* [source,yaml] ---- @@ -144,7 +141,7 @@ topologySpreadConstraints: ---- -In the event of an AZ impairment, you can absorb the increased load on the CoreDNS Pods by using an autoscaling system for CoreDNS. The number of DNS instances you require will depend on the number of workloads running in your cluster. CoreDNS is CPU bound which allows it to scale based on CPU using the https://aws.github.io/aws-eks-best-practices/reliability/docs/application/#horizontal-pod-autoscaler-hpa[Horizontal Pod Autoscaler (HPA)]. Below is an example that you can modify to suit your needs. +If there's an AZ impairment, you can absorb the increased load on the CoreDNS Pods by using an autoscaling system for CoreDNS. The number of DNS instances that you will require depends on the number of workloads that are running in your cluster. CoreDNS is CPU bound, which allows it to scale based on CPU by using the https://aws.github.io/aws-eks-best-practices/reliability/docs/application/#horizontal-pod-autoscaler-hpa[Horizontal Pod Autoscaler (HPA)]. The following is an example that you can modify to suit your needs. [source,yaml] @@ -164,9 +161,9 @@ spec: targetCPUUtilizationPercentage: 50 ---- -Alternatively, EKS can manage the autoscaling of the CoreDNS Deployment in the EKS add-on version of CoreDNS. This CoreDNS autoscaler continuously monitors the cluster state, including the number of nodes and CPU cores. Based on that information, the controller will dynamically adapt the number of replicas of the CoreDNS deployment in an EKS cluster. +Alternatively, EKS can manage autoscaling of the CoreDNS deployment in the EKS add-on version of CoreDNS. This CoreDNS autoscaler continuously monitors the cluster state, including the number of nodes and CPU cores. Based on that information, the controller dynamically adjusts the number of replicas of the CoreDNS deployment in an EKS cluster. -To enable the link:eks/latest/userguide/coredns-autoscaling.html["autoscaling configuration in the CoreDNS EKS add-on",type="documentation"], you should add the following optional configuration settings: +To enable the <>, use the following configuration setting: [source,json] @@ -179,13 +176,13 @@ To enable the link:eks/latest/userguide/coredns-autoscaling.html["autoscaling co ---- -You can also use https://kubernetes.io/docs/tasks/administer-cluster/nodelocaldns/[NodeLocal DNS] or the https://github.com/kubernetes-sigs/cluster-proportional-autoscaler[cluster proportional autoscaler] to scale CoreDNS. You can read further about https://aws.github.io/aws-eks-best-practices/scalability/docs/cluster-services/#scale-coredns-horizontally[scaling CoreDNS horizontally here]. +You can also use https://kubernetes.io/docs/tasks/administer-cluster/nodelocaldns/[NodeLocal DNS] or the https://github.com/kubernetes-sigs/cluster-proportional-autoscaler[cluster proportional autoscaler] to scale CoreDNS. For more information, see https://aws.github.io/aws-eks-best-practices/scalability/docs/cluster-services/#scale-coredns-horizontally[Scaling CoreDNS horizontally]. -=== Colocate Interdependent Pods in the Same AZ +=== Colocate interdependent Pods in the same Availability Zone -In most cases, you may be running distinct workloads that have to communicate with each other for successful execution of an end-to-end process. If the distinct applications are spread across different AZs but are not colocated in the same AZ, then a single AZ impairment may impact the underlying end-to-end process. For example, if *Application A* has multiple replicas in AZ 1 and AZ 2, but *Application B* has all its replicas in AZ 3, then the loss of AZ 3 will affect any end-to-end processes between these two workloads (*Application A and B*). Combining topology spread constraints with pod affinity can enhance your application's resiliency by spreading Pods across all AZs, as well as configuring a relationship between certain Pods to ensure that they're colocated together. +Typically, applications have distinct workloads that need to communicate with each other to successfully complete an end-to-end process. If these distinct applications are spread across different AZs and are not colocated in the same AZ, then a single AZ impairment can impact the end-to-end process. For example, if *Application A* has multiple replicas in AZ 1 and AZ 2, but *Application B* has all its replicas in AZ 3, then the loss of AZ 3 will affect end-to-end processes between the two workloads, *Application A* and *Application B*. If you combine topology spread constraints with pod affinity, you can enhance your application's resiliency by spreading Pods across all AZs. In addition, this configures a relationship between certain Pods to ensure that they're colocated. -With https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/[pod affinity rules], you can define relationships between workloads to influence the behavior of the Kubernetes Scheduler so that it colocates Pods on the same worker node or in the same AZ. You can also configure how strict these scheduling constraints should be. +With https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/[pod affinity rules], you can define relationships between workloads to influence the behavior of the Kubernetes Scheduler so that it colocates Pods on the same worker node or in the same AZ. You can also configure how strict the scheduling constraints should be. [source,yaml] @@ -213,66 +210,66 @@ metadata: ---- -The diagram below depicts pods that have been co-located on the same node using +The following diagram shows several pods that have been colocated on the same node by using pod affinity rules. -image::zs-pod-affinity-rule.png[Illustration of network] +image::images/zs-pod-affinity-rule.png[Illustration of network] -=== Test That Your Cluster Environment Can Handle The Loss of an AZ +=== Test that your cluster environment can handle the loss of an AZ -After completing the above requirements, the next important step is to test that you have sufficient compute and workload capacity to handle the loss of an AZ. You can do this by manually triggering a zonal shift in EKS. Alternatively, you can enable zonal autoshift and configure practice runs to test that your applications function as expected with one less AZ in your cluster environment. +After you complete the requirements described in the previous sections, the next step is to test that you have sufficient compute and workload capacity to handle the loss of an AZ. You can do this by manually starting a zonal shift in EKS. Alternatively, you can enable zonal autoshift and configure practice runs, which also test that your applications function as expected with one less AZ in your cluster environment. -## Frequently Asked Questions +== Frequently asked questions -**Why should I use this feature?** +*Why should I use this feature?* -By using ARC zonal shift or zonal autoshift in your EKS cluster, you can better maintain Kubernetes application availability by automating the quick recovery process of shifting in-cluster network traffic away from an impaired AZ. With ARC, you can avoid long and complicated steps which often lead to an extended recovery period during impaired AZ events. +By using ARC zonal shift or zonal autoshift in your EKS cluster, you can better maintain Kubernetes application availability by automating the quick recovery process of shifting in-cluster network traffic away from an impaired AZ. With ARC, you can avoid long, complicated steps that can lead to an extended recovery period during impaired AZ events. -**How does this feature work with other {aws} services?** +*How does this feature work with other {aws} services?* -EKS integrates with ARC which provides the primary interface for you to accomplish recovery operations in {aws}. To ensure that in-cluster traffic is appropriately routed away from an impaired AZ, modifications are made to the list of network endpoints for Pods running in the Kubernetes data plane. If you're using {aws} Load Balancers for routing external traffic into the cluster, you can already register your load balancers with ARC and trigger a zonal shift on them to prevent traffic flowing into the degraded zone. This feature also interacts with Amazon EC2 Auto Scaling Groups (ASG) that are created by EKS Managed Node Groups (MNG). To prevent an impaired AZ from being used for new Kubernetes Pods or node launches, EKS removes the impaired AZ from the ASG. +EKS integrates with ARC, which provides the primary interface for you to accomplish recovery operations in {aws}. To ensure that in-cluster traffic is appropriately routed away from an impaired AZ, EKS makes modifications to the list of network endpoints for Pods running in the Kubernetes data plane. If you're using Elastic Load Balancing to route external traffic into the cluster, you can register your load balancers with ARC and start a zonal shift on them to prevent traffic from flowing into the degraded AZ. Zonal shift also works with Amazon EC2 Auto Scaling groups that are created by EKS managed node groups. To prevent an impaired AZ from being used for new Kubernetes Pods or node launches, EKS removes the impaired AZ from the Auto Scaling groups. -**How is this feature different from default Kubernetes protections?** +*How is this feature different from default Kubernetes protections?* -This feature works in tandem with several Kubernetes native built-in protections that help customers stay resilient. You can configure Pod readiness and liveness probes that decide when a Pod should take traffic. When these probes fail, Kubernetes removes these Pods as targets for a Service and traffic is no longer sent to the Pod. While this is useful, it's non-trivial for customers to configure these health checks so that they are guaranteed to fail when a zone is degraded. The ARC zonal shift feature provides you with an additional safety net that helps them isolate a degraded AZ entirely when Kubernetes' native protections have not sufficed. It also provides you with an easy way to test the operational readiness and resilience of your architecture. +This feature works in tandem with several Kubernetes built-in protections that help customer applications' resiliency. You can configure Pod readiness and liveness probes that decide when a Pod should take traffic. When these probes fail, Kubernetes removes these Pods as targets for a service, and traffic is no longer sent to the Pod. While this is useful, it's not simple for customers to configure these health checks so that they are guaranteed to fail when an AZ is degraded. The ARC zonal shift feature provides an additional safety net that helps you to isolate a degraded AZ entirely when Kubernetes' native protections were not enough. Zonal shift also gives you an easy way to test the operational readiness and resilience of your architecture. -**Can {aws} trigger a zonal shift on my behalf?** +*Can {aws} start a zonal shift on my behalf?* -Yes, if you want a fully automated way of using ARC zonal shift, you can enable ARC zonal autoshift. With zonal autoshift, you can rely on {aws} to monitor the health of the AZs for your EKS cluster, and to automatically trigger a shift when an AZ impairment is detected. +Yes, if you want a fully automated way of using ARC zonal shift, you can enable ARC zonal autoshift. With zonal autoshift, you can rely on {aws} to monitor the health of the AZs for your EKS cluster, and to automatically start a zonal shift when an AZ impairment is detected. -**What happens if I use this feature and my worker nodes and workloads are not pre-scaled?** +*What happens if I use this feature and my worker nodes and workloads are not pre-scaled?* -If you are not pre-scaled and rely on provisioning additional nodes or Pods during a zonal shift, then you risk experiencing a delayed recovery. The process of adding new nodes to the Kubernetes data plane will take some time which can have implications on the real-time performance and availability of your applications, especially in the event of a zonal impairment. Additionally, in the event of a zonal impairment, you may encounter a potential compute capacity constraint which would prevent newly required nodes from being added to the healthy AZs. +If you are not pre-scaled and rely on provisioning additional nodes or Pods during a zonal shift, you risk a delayed recovery. The process of adding new nodes to the Kubernetes data plane takes some time, which can impact the real-time performance and availability of your applications, especially when there's a zonal impairment. Additionally, in the event of a zonal impairment, you may encounter a potential compute capacity constraint that could prevent newly required nodes from being added to the healthy AZs. -If your workloads are not pre-scaled and spread across all AZs in your cluster, a zonal impairment may impact the availability of an application that is only running on worker nodes in an impacted AZ. To mitigate the risk of a complete availability outage for your application, EKS has a fail safe for traffic to be sent to Pod endpoints in an impaired zone if that workload has all of its endpoints in the unhealthy AZ. However, it's strongly recommended that you rather pre-scale and spread your applications across all AZs to maintain availability in the event of a zonal issue. +If your workloads are not pre-scaled and spread across all AZs in your cluster, a zonal impairment might impact the availability of an application that is only running on worker nodes in an impacted AZ. To mitigate the risk of a complete availability outage for your application, EKS has a fail safe for traffic to be sent to Pod endpoints in an impaired zone if that workload has all of its endpoints in the unhealthy AZ. However, we strongly recommend that you pre-scale and spread your applications across all AZs to maintain availability in the event of a zonal issue. -**What happens if I'm running a stateful application?** +*How does this work if I'm running a stateful application?* -If you are running a stateful application, you will need to assess its fault tolerance depending on the use case and the architecture. If you have an active/standby architecture or pattern, there may be instances where the active is in an impaired AZ. At the application level, if the standby is not activated, you may run into issues with your application. You may also run into issues when new Kubernetes Pods are launched in healthy AZs since they will not be able to attach to the persistent volumes bounded to the impaired AZ. +If you are running a stateful application, you must assess its fault tolerance, based on your use case and architecture. If you have an active/standby architecture or pattern, there might be instances where the active is in an impaired AZ. At the application level, if the standby is not activated, you might run into issues with your application. You might also run into issues when new Kubernetes Pods are launched in healthy AZs, since they won't be able to attach to the persistent volumes bounded to the impaired AZ. -**Does this feature work with Karpenter?** +*Does this feature work with Karpenter?* -Karpenter support is currently not available with ARC zonal shift and zonal autoshift in EKS. If an AZ is impaired, you can adjust the relevant Karpenter NodePool configuration by removing the unhealthy AZ so that new worker nodes are only launched in the healthy AZs. +Karpenter support is currently not available with ARC zonal shift and zonal autoshift in EKS. If an AZ is impaired, you can adjust the relevant Karpenter NodePool configuration by removing the unhealthy AZ so that new worker nodes are only launched in the other AZs. -**Does this feature work with EKS Fargate?** +*Does this feature work with EKS Fargate?* This feature does not work with EKS Fargate. By default, when EKS Fargate recognizes a zonal health event, Pods will prefer to run in the other AZs. -**Will the EKS managed Kubernetes control plane be impacted?** +*Will the EKS managed Kubernetes control plane be impacted?* -No, by default Amazon EKS runs and scales the Kubernetes control plane across multiple AZs to ensure high availability. ARC zonal shift and zonal autoshift will only act on the Kubernetes data plane. +No, by default Amazon EKS runs and scales the Kubernetes control plane across multiple AZs to ensure high availability. ARC zonal shift and zonal autoshift only act on the Kubernetes data plane. -**Are there any costs associated with this new feature?** +*Are there any costs associated with this new feature?* -You can use ARC zonal shift and zonal autoshift in your EKS cluster at no additional charge. However, you will continue to pay for provisioned instances and it is strongly recommended that you pre-scale your Kubernetes data plane before using this feature. You should consider the right balance between cost and application availability. +You can use ARC zonal shift and zonal autoshift in your EKS cluster at no additional charge. However, you will continue to pay for provisioned instances and we strongly recommended that you pre-scale your Kubernetes data plane before using this feature. You should consider a balance between cost and application availability. -== Additional Resources +== Additional resources * link:r53recovery/latest/dg/arc-zonal-shift.how-it-works.html["How a zonal shift works",type="documentation"] * link:r53recovery/latest/dg/route53-arc-best-practices.zonal-shifts.html#zonalshift.route53-arc-best-practices.zonal-shifts["Best practices for zonal shifts in ARC",type="documentation"] * link:r53recovery/latest/dg/arc-zonal-shift.resource-types.html["Resources and scenarios supported for zonal shift and zonal autoshift",type="documentation"] -* link:blogs/containers/operating-resilient-workloads-on-amazon-eks/["Operating resilient workloads on Amazon EKS",type="marketing"] -* link:blogs/containers/eliminate-kubernetes-node-scaling-lag-with-pod-priority-and-over-provisioning/["Eliminate Kubernetes node scaling lag with pod priority and over-provisioning",type="marketing"] -* link:eks/latest/userguide/coredns-autoscaling.html["Scale CoreDNS Pods for high DNS traffic",type="documentation"] +* link:containers/operating-resilient-workloads-on-amazon-eks["Operating resilient workloads on Amazon EKS",type="blog"] +* link:containers/eliminate-kubernetes-node-scaling-lag-with-pod-priority-and-over-provisioning["Eliminate Kubernetes node scaling lag with pod priority and over-provisioning",type="blog"] +* <> diff --git a/latest/ug/connector/connecting-cluster.adoc b/latest/ug/connector/connecting-cluster.adoc index 5f3cb087e..dc05a6a0d 100644 --- a/latest/ug/connector/connecting-cluster.adoc +++ b/latest/ug/connector/connecting-cluster.adoc @@ -1,20 +1,16 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[connecting-cluster,connecting-cluster.title]] -= Connect an external [.noloc]`Kubernetes` cluster to the Amazon EKS Management Console -:info_doctype: section -:info_title: Connect an external Kubernetes cluster to the Amazon EKS Management Console +[#connecting-cluster] += Connect an external Kubernetes cluster to the Amazon EKS Management Console :info_titleabbrev: Connect a cluster -:info_abstract: Learn to connect an external Kubernetes cluster to an Amazon EKS Management Console and install the eks-connector agent via Helm or YAML manifests to enable visibility and management of the external cluster. [abstract] -- -Learn to connect an external [.noloc]`Kubernetes` cluster to an Amazon EKS Management Console and install the eks-connector agent via Helm or YAML manifests to enable visibility and management of the external cluster. +Learn to connect an external Kubernetes cluster to an Amazon EKS Management Console and install the eks-connector agent via Helm or YAML manifests to enable visibility and management of the external cluster. -- -You can connect an external [.noloc]`Kubernetes` cluster to Amazon EKS by using multiple methods in the following process. This process involves two steps: Registering the cluster with Amazon EKS and installing the `eks-connector` agent in the cluster. +You can connect an external Kubernetes cluster to Amazon EKS by using multiple methods in the following process. This process involves two steps: Registering the cluster with Amazon EKS and installing the `eks-connector` agent in the cluster. [IMPORTANT] ==== @@ -23,12 +19,12 @@ You must complete the second step within 3 days of completing the first step, be ==== -[[connecting-cluster-considerations,connecting-cluster-considerations.title]] +[#connecting-cluster-considerations] == Considerations You can use YAML manifests when installing the agent. Alternatively, you can use Helm if you register the cluster with the {aws-management-console} or {aws} Command Line Interface. However, you cannot use Helm to install the agent if you register the cluster with `eksctl`. -[[connector-prereqs,connector-prereqs.title]] +[#connector-prereqs] == Prerequisites * Ensure the Amazon EKS Connector agent role was created. Follow the steps in <>. @@ -40,7 +36,7 @@ You can use YAML manifests when installing the agent. Alternatively, you can use ** `iam:PassRole` -[[connector-connecting,connector-connecting.title]] +[#connector-connecting] == Step 1: Registering the cluster To register a cluster to Amazon EKS connector, you can use one of these tools: @@ -63,7 +59,7 @@ aws eks register-cluster \ + An example output is as follows. + -[source,json,subs="verbatim,attributes"] +[source,json,subs="verbatim,attributes",role="nocopy"] ---- { "cluster": { @@ -86,11 +82,11 @@ You use the `aws-region`, `activationId`, and `activationCode` values in the nex === {aws-management-console} [[console_register_cluster_connect]] . Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. -. Choose *Add cluster* and select *Register* to bring up the configuration page. +. Choose *Add cluster* and select *Register* to bring up the configuration page. . On the *Configure cluster* section, fill in the following fields: + *** *Name* – A unique name for your cluster. -*** *Provider* – Choose to display the dropdown list of [.noloc]`Kubernetes` cluster providers. If you don't know the specific provider, select *Other*. +*** *Provider* – Choose to display the dropdown list of Kubernetes cluster providers. If you don't know the specific provider, select *Other*. *** *EKS Connector role* – Select the role to use for connecting the cluster. . Select *Register cluster*. . The Cluster overview page displays. If you want to use the Helm chart, copy the `helm install` command and continue to the next step. If you want to use the YAML manifest, choose *Download YAML file* to download the manifest file to your local drive. @@ -100,9 +96,9 @@ You use the `aws-region`, `activationId`, and `activationCode` values in the nex This is your only opportunity to copy the `helm install` command or download this file. Don't navigate away from this page, as the link will not be accessible and you must deregister the cluster and start the steps from the beginning. ==== + -The command or manifest file can be used only once for the registered cluster. If you delete resources from the [.noloc]`Kubernetes` cluster, you must re-register the cluster and obtain a new manifest file. +The command or manifest file can be used only once for the registered cluster. If you delete resources from the Kubernetes cluster, you must re-register the cluster and obtain a new manifest file. -Continue to the next step to apply the manifest file to your [.noloc]`Kubernetes` cluster. +Continue to the next step to apply the manifest file to your Kubernetes cluster. === `eksctl` [[eksctl_register_cluster_connect]] . `eksctl` version `0.68` or later must be installed. To install or upgrade it, see <>. @@ -137,7 +133,7 @@ kubectl apply -f eks-connector-binding.yaml ---- -[[eks-connector-apply,eks-connector-apply.title]] +[#eks-connector-apply] == Step 2: Installing the `eks-connector` agent To install the `eks-connector` agent, use one of the following tools: @@ -145,7 +141,7 @@ To install the `eks-connector` agent, use one of the following tools: * <> * <> -=== helm [[helm_agent_cluster_connect]] +=== Helm [[helm_agent_cluster_connect]] [NOTE] ==== @@ -168,7 +164,7 @@ If you used the {aws-management-console} in the previous step, use the command t . Check the healthiness of the installed `eks-connector` deployment and wait for the status of the registered cluster in Amazon EKS to be `ACTIVE`. === yaml [[yaml_agent_cluster_connect]] -Complete the connection by applying the Amazon EKS Connector manifest file to your [.noloc]`Kubernetes` cluster. To do this, you must use the methods described previously. If the manifest isn't applied within three days, the Amazon EKS Connector registration expires. If the cluster connection expires, the cluster must be deregistered before connecting the cluster again. +Complete the connection by applying the Amazon EKS Connector manifest file to your Kubernetes cluster. To do this, you must use the methods described previously. If the manifest isn't applied within three days, the Amazon EKS Connector registration expires. If the cluster connection expires, the cluster must be deregistered before connecting the cluster again. . Download the Amazon EKS Connector YAML file. + @@ -195,7 +191,7 @@ Ensure that your activation code is in the base64 format. ---- kubectl apply -f eks-connector.yaml ---- -. After the Amazon EKS Connector manifest and role binding YAML files are applied to your [.noloc]`Kubernetes` cluster, confirm that the cluster is now connected. +. After the Amazon EKS Connector manifest and role binding YAML files are applied to your Kubernetes cluster, confirm that the cluster is now connected. + [source,bash,subs="verbatim,attributes"] ---- @@ -208,9 +204,9 @@ The output should include `status=ACTIVE`. . (Optional) Add tags to your cluster. For more information, see <>. -[[eks-connector-next,eks-connector-next.title]] +[#eks-connector-next] == Next steps If you have any issues with these steps, see <>. -To grant additional link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principals,type="documentation"] access to the Amazon EKS console to view [.noloc]`Kubernetes` resources in a connected cluster, see <>. +To grant additional link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principals,type="documentation"] access to the Amazon EKS console to view Kubernetes resources in a connected cluster, see <>. \ No newline at end of file diff --git a/latest/ug/connector/connector-grant-access.adoc b/latest/ug/connector/connector-grant-access.adoc index aaa58c363..e6d423962 100644 --- a/latest/ug/connector/connector-grant-access.adoc +++ b/latest/ug/connector/connector-grant-access.adoc @@ -1,29 +1,24 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[connector-grant-access,connector-grant-access.title]] -= Grant access to view [.noloc]`Kubernetes` cluster resources on an Amazon EKS console -:info_doctype: section -:info_title: Grant access to view Kubernetes cluster resources on an \ - Amazon EKS console -:info_titleabbrev: Grant access to Kubernetes clusters from {aws} console -:info_abstract: Learn to grant IAM principals access to view Kubernetes cluster resources on an Amazon EKS Management Console. +[#connector-grant-access] += Grant access to view Kubernetes cluster resources on an Amazon EKS console +:info_titleabbrev: Grant access to clusters [abstract] -- Learn to grant IAM principals access to view Kubernetes cluster resources on an Amazon EKS Management Console. -- -Grant link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principals,type="documentation"] access to the Amazon EKS console to view information about [.noloc]`Kubernetes` resources running on your connected cluster. +Grant link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principals,type="documentation"] access to the Amazon EKS console to view information about Kubernetes resources running on your connected cluster. -[[connector-grant-access-prereqs,connector-grant-access-prereqs.title]] +[#connector-grant-access-prereqs] == Prerequisites -The link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] that you use to access the {aws-management-console} must meet the following requirements: +The link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] that you use to access the {aws-management-console} must meet the following requirements: * It must have the `eks:AccessKubernetesApi` IAM permission. -* The Amazon EKS Connector service account can impersonate the IAM principal in the cluster. This allows the Amazon EKS Connector to map the IAM principal to a [.noloc]`Kubernetes` user. +* The Amazon EKS Connector service account can impersonate the IAM principal in the cluster. This allows the Amazon EKS Connector to map the IAM principal to a Kubernetes user. *To create and apply the Amazon EKS Connector cluster role* @@ -34,21 +29,21 @@ The link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,t curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/eks-connector/manifests/eks-connector-console-roles/eks-connector-clusterrole.yaml ---- . Edit the cluster role template YAML file. Replace references of `%IAM_ARN%` with the Amazon Resource Name (ARN) of your IAM principal. -. Apply the Amazon EKS Connector cluster role YAML to your [.noloc]`Kubernetes` cluster. +. Apply the Amazon EKS Connector cluster role YAML to your Kubernetes cluster. + [source,bash,subs="verbatim,attributes"] ---- kubectl apply -f eks-connector-clusterrole.yaml ---- -For an IAM principal to view [.noloc]`Kubernetes` resources in Amazon EKS console, the principal must be associated with a [.noloc]`Kubernetes` `role` or `clusterrole` with necessary permissions to read the resources. For more information, see https://kubernetes.io/docs/reference/access-authn-authz/rbac/[Using RBAC Authorization] in the [.noloc]`Kubernetes` documentation. +For an IAM principal to view Kubernetes resources in Amazon EKS console, the principal must be associated with a Kubernetes `role` or `clusterrole` with necessary permissions to read the resources. For more information, see https://kubernetes.io/docs/reference/access-authn-authz/rbac/[Using RBAC Authorization] in the Kubernetes documentation. *To configure an IAM principal to access the connected cluster* . You can download either of these example manifest files to create a `clusterrole` and `clusterrolebinding` or a `role` and `rolebinding`, respectively: + -*View [.noloc]`Kubernetes` resources in all namespaces*::: +*View Kubernetes resources in all namespaces*::: ** The `eks-connector-console-dashboard-full-access-clusterrole` cluster role gives access to all namespaces and resources that can be visualized in the console. You can change the name of the `role`, `clusterrole` and their corresponding binding before applying it to your cluster. Use the following command to download a sample file. + [source,bash,subs="verbatim,attributes"] @@ -57,7 +52,7 @@ curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/eks-connector/manifests/ek ---- -*View [.noloc]`Kubernetes` resources in a specific namespace*::: +*View Kubernetes resources in a specific namespace*::: ** The namespace in this file is `default`, so if you want to specify a different namespace, edit the file before applying it to your cluster. Use the following command to download a sample file. + [source,bash,subs="verbatim,attributes"] @@ -65,11 +60,11 @@ curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/eks-connector/manifests/ek curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/eks-connector/manifests/eks-connector-console-roles/eks-connector-console-dashboard-restricted-access-group.yaml ---- . Edit the full access or restricted access YAML file to replace references of `%IAM_ARN%` with the Amazon Resource Name (ARN) of your IAM principal. -. Apply the full access or restricted access YAML files to your [.noloc]`Kubernetes` cluster. Replace the YAML file value with your own. +. Apply the full access or restricted access YAML files to your Kubernetes cluster. Replace the YAML file value with your own. + [source,bash,subs="verbatim,attributes"] ---- kubectl apply -f eks-connector-console-dashboard-full-access-group.yaml ---- -To view [.noloc]`Kubernetes` resources in your connected cluster, see <>. Data for some resource types on the *Resources* tab isn't available for connected clusters. +To view Kubernetes resources in your connected cluster, see <>. Data for some resource types on the *Resources* tab isn't available for connected clusters. \ No newline at end of file diff --git a/latest/ug/connector/deregister-connected-cluster.adoc b/latest/ug/connector/deregister-connected-cluster.adoc index 090a4358a..addbc3429 100644 --- a/latest/ug/connector/deregister-connected-cluster.adoc +++ b/latest/ug/connector/deregister-connected-cluster.adoc @@ -1,17 +1,13 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[deregister-connected-cluster,deregister-connected-cluster.title]] +[#deregister-connected-cluster] = Deregister a Kubernetes cluster from the Amazon EKS console -:info_doctype: section -:info_title: Deregister a Kubernetes cluster from the Amazon EKS console :info_titleabbrev: Deregister a cluster -:info_abstract: Learn to deregister a Kubernetes cluster from Amazon EKS and uninstall the eks-connector agent to stop managing the cluster from the Amazon EKS Management Console. [abstract] -- -Learn to deregister a [.noloc]`Kubernetes` cluster from Amazon EKS and uninstall the eks-connector agent to stop managing the cluster from the Amazon EKS Management Console. +Learn to deregister a Kubernetes cluster from Amazon EKS and uninstall the eks-connector agent to stop managing the cluster from the Amazon EKS Management Console. -- If you are finished using a connected cluster, you can deregister it. After it's deregistered, the cluster is no longer visible in the Amazon EKS console. @@ -26,8 +22,8 @@ You must have the following permissions to call the deregisterCluster API: This process involves two steps: Deregistering the cluster with Amazon EKS and uninstalling the eks-connector agent in the cluster. -[[deregister-connected-cluster-eks,deregister-connected-cluster-eks.title]] -== Deregister the [.noloc]`Kubernetes` cluster +[#deregister-connected-cluster-eks] +== Deregister the Kubernetes cluster To deregister a cluster from Amazon EKS connector, you can use one of these tools: * <> @@ -51,7 +47,7 @@ aws eks deregister-cluster \ . Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. . Choose *Clusters*. -. On the *Clusters* page, select the connected cluster and select *Deregister*. +. On the *Clusters* page, select the connected cluster and select *Deregister*. . Confirm that you want to deregister the cluster. === `eksctl` [[eksctl_deregister_cluster_connect]] @@ -66,8 +62,8 @@ eksctl deregister cluster --name my-cluster ---- -[[deregister-connected-cluster-k8s,deregister-connected-cluster-k8s.title]] -== Clean up the resources in your [.noloc]`Kubernetes` cluster +[#deregister-connected-cluster-k8s] +== Clean up the resources in your Kubernetes cluster To uninstall the `eks-connector` agent, use one of the following tools: * <> @@ -84,10 +80,10 @@ helm -n eks-connector uninstall eks-connector === yaml [[yaml_agent_cluster_deregister]] -. Delete the Amazon EKS Connector YAML file from your [.noloc]`Kubernetes` cluster. +. Delete the Amazon EKS Connector YAML file from your Kubernetes cluster. + [source,bash,subs="verbatim,attributes"] ---- kubectl delete -f eks-connector.yaml ---- -. If you created `clusterrole` or `clusterrolebindings` for additional link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principals,type="documentation"] to access the cluster, delete them from your [.noloc]`Kubernetes` cluster. +. If you created `clusterrole` or `clusterrolebindings` for additional link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principals,type="documentation"] to access the cluster, delete them from your Kubernetes cluster. \ No newline at end of file diff --git a/latest/ug/connector/eks-connector.adoc b/latest/ug/connector/eks-connector.adoc index c0d61be84..adc4f842e 100644 --- a/latest/ug/connector/eks-connector.adoc +++ b/latest/ug/connector/eks-connector.adoc @@ -1,49 +1,38 @@ -//!!NODE_ROOT include::../attributes.txt[] -[[eks-connector,eks-connector.title]] -= Connect a [.noloc]`Kubernetes` cluster to an Amazon EKS Management Console with Amazon EKS Connector -:doctype: book -:sectnums: -:toc: left -:icons: font -:experimental: -:idprefix: -:idseparator: - -:sourcedir: . -:info_doctype: chapter -:info_title: Connect a Kubernetes cluster to an Amazon EKS Management Console with Amazon EKS Connector + +[#eks-connector] += Connect a Kubernetes cluster to an Amazon EKS Management Console with Amazon EKS Connector :info_titleabbrev: Amazon EKS Connector -:info_abstract: Discover how to connect conformant Kubernetes clusters to {aws} and visualize them in the Amazon EKS console using the Amazon EKS Connector agent and required IAM roles. [abstract] -- -Discover how to connect conformant [.noloc]`Kubernetes` clusters to {aws} and visualize them in the Amazon EKS console using the Amazon EKS Connector agent and required IAM roles. +Discover how to connect conformant Kubernetes clusters to {aws} and visualize them in the Amazon EKS console using the Amazon EKS Connector agent and required IAM roles. -- -You can use Amazon EKS Connector to register and connect any conformant [.noloc]`Kubernetes` cluster to {aws} and visualize it in the Amazon EKS console. After a cluster is connected, you can see the status, configuration, and workloads for that cluster in the Amazon EKS console. You can use this feature to view connected clusters in Amazon EKS console, but you can't manage them. The Amazon EKS Connector requires an agent that is an https://github.com/aws/amazon-eks-connector[open source project on Github]. For additional technical content, including frequently asked questions and troubleshooting, see <>. +You can use Amazon EKS Connector to register and connect any conformant Kubernetes cluster to {aws} and visualize it in the Amazon EKS console. After a cluster is connected, you can see the status, configuration, and workloads for that cluster in the Amazon EKS console. You can use this feature to view connected clusters in Amazon EKS console, but you can't manage them. The Amazon EKS Connector requires an agent that is an https://github.com/aws/amazon-eks-connector[open source project on Github]. For additional technical content, including frequently asked questions and troubleshooting, see <>. -The Amazon EKS Connector can connect the following types of [.noloc]`Kubernetes` clusters to Amazon EKS. +The Amazon EKS Connector can connect the following types of Kubernetes clusters to Amazon EKS. -* On-premises [.noloc]`Kubernetes` clusters +* On-premises Kubernetes clusters * Self-managed clusters that are running on Amazon EC2 * Managed clusters from other cloud providers -[[connect-cluster-reqts,connect-cluster-reqts.title]] +[#connect-cluster-reqts] == Amazon EKS Connector considerations Before you use Amazon EKS Connector, understand the following: -* You must have administrative privileges to the [.noloc]`Kubernetes` cluster to connect the cluster to Amazon EKS. -* The [.noloc]`Kubernetes` cluster must have [.noloc]`Linux` 64-bit (x86) worker nodes present before connecting. ARM worker nodes aren't supported. -* You must have worker nodes in your [.noloc]`Kubernetes` cluster that have outbound access to the `ssm.` and `ssmmessages.` Systems Manager endpoints. For more information, see link:general/latest/gr/ssm.html[Systems Manager endpoints,type="documentation"] in the _{aws} General Reference_. +* You must have administrative privileges to the Kubernetes cluster to connect the cluster to Amazon EKS. +* The Kubernetes cluster must have Linux 64-bit (x86) worker nodes present before connecting. ARM worker nodes aren't supported. +* You must have worker nodes in your Kubernetes cluster that have outbound access to the `ssm.` and `ssmmessages.` Systems Manager endpoints. For more information, see link:general/latest/gr/ssm.html[Systems Manager endpoints,type="documentation"] in the _{aws} General Reference_. * By default, you can connect up to 10 clusters in a Region. You can request an increase through the link:servicequotas/latest/userguide/request-quota-increase.html[service quota console,type="documentation"]. See link:servicequotas/latest/userguide/request-quota-increase.html[Requesting a quota increase,type="documentation"] for more information. -* Only the Amazon EKS `RegisterCluster`, `ListClusters`, `DescribeCluster`, and `DeregisterCluster` APIs are supported for external [.noloc]`Kubernetes` clusters. +* Only the Amazon EKS `RegisterCluster`, `ListClusters`, `DescribeCluster`, and `DeregisterCluster` APIs are supported for external Kubernetes clusters. * You must have the following permissions to register a cluster: + ** eks:RegisterCluster @@ -57,7 +46,7 @@ Before you use Amazon EKS Connector, understand the following: ** ssm:DeregisterManagedInstance -[[connector-iam-permissions,connector-iam-permissions.title]] +[#connector-iam-permissions] == Required IAM roles for Amazon EKS Connector Using the Amazon EKS Connector requires the following two IAM roles: @@ -67,7 +56,7 @@ Using the Amazon EKS Connector requires the following two IAM roles: * The <> service-linked role is created when you register a cluster for the first time. * You must create the Amazon EKS Connector agent IAM role. See <> for details. -To enable cluster and workload view permission for link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principals,type="documentation"], apply the `eks-connector` and Amazon EKS Connector cluster roles to your cluster. Follow the steps in <>. +To enable cluster and workload view permission for link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principals,type="documentation"], apply the `eks-connector` and Amazon EKS Connector cluster roles to your cluster. Follow the steps in <>. include::connecting-cluster.adoc[leveloffset=+1] @@ -84,4 +73,4 @@ include::troubleshooting-connector.adoc[leveloffset=+1] include::tsc-faq.adoc[leveloffset=+1] -include::security-connector.adoc[leveloffset=+1] +include::security-connector.adoc[leveloffset=+1] \ No newline at end of file diff --git a/latest/ug/connector/security-connector.adoc b/latest/ug/connector/security-connector.adoc index e7dfa1cf5..63614312c 100644 --- a/latest/ug/connector/security-connector.adoc +++ b/latest/ug/connector/security-connector.adoc @@ -1,22 +1,16 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[security-connector,security-connector.title]] +[#security-connector] = Understand security in Amazon EKS Connector -:info_doctype: section -:info_title: Understand security in Amazon EKS Connector :info_titleabbrev: Security considerations -:info_abstract: Learn how the open-source EKS Connector affects security, and understand {aws} and \ - customer security responsibilities for connectivity, cluster management, and IAM \ - access control. [abstract] -- Learn how the open-source EKS Connector affects security, and understand {aws} and customer security responsibilities for connectivity, cluster management, and IAM access control. -- -The Amazon EKS Connector is an open source component that runs on your [.noloc]`Kubernetes` cluster. This cluster can be located outside of the {aws} environment. This creates additional considerations for security responsibilities. This configuration can be illustrated by the following diagram. Orange represents {aws} responsibilities, and blue represents customer responsibilities: +The Amazon EKS Connector is an open source component that runs on your Kubernetes cluster. This cluster can be located outside of the {aws} environment. This creates additional considerations for security responsibilities. This configuration can be illustrated by the following diagram. Orange represents {aws} responsibilities, and blue represents customer responsibilities: @@ -24,21 +18,21 @@ image::images/connector-model.png[EKS Connector Responsibilities,scaledwidth=100 This topic describes the differences in the responsibility model if the connected cluster is outside of {aws}. -[[connect-aws-resp,connect-aws-resp.title]] +[#connect-aws-resp] == {aws} responsibilities -* Maintaining, building, and delivering Amazon EKS Connector, which is an https://github.com/aws/amazon-eks-connector[open source component] that runs on a customer's [.noloc]`Kubernetes` cluster and communicates with {aws}. -* Maintaining transport and application layer communication security between the connected [.noloc]`Kubernetes` cluster and {aws} services. +* Maintaining, building, and delivering Amazon EKS Connector, which is an https://github.com/aws/amazon-eks-connector[open source component] that runs on a customer's Kubernetes cluster and communicates with {aws}. +* Maintaining transport and application layer communication security between the connected Kubernetes cluster and {aws} services. -[[connect-cust-resp,connect-cust-resp.title]] +[#connect-cust-resp] == Customer responsibilities -* [.noloc]`Kubernetes` cluster specific security, specifically along the following lines: +* Kubernetes cluster specific security, specifically along the following lines: + -** [.noloc]`Kubernetes` secrets must be properly encrypted and protected. +** Kubernetes secrets must be properly encrypted and protected. ** Lock down access to the `eks-connector` namespace. -* Configuring role-based access control (RBAC) permissions to manage link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] access from {aws}. For instructions, see <>. +* Configuring role-based access control (RBAC) permissions to manage link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] access from {aws}. For instructions, see <>. * Installing and upgrading Amazon EKS Connector. -* Maintaining the hardware, software, and infrastructure that supports the connected [.noloc]`Kubernetes` cluster. -* Securing their {aws} accounts (for example, through safeguarding your link:IAM/latest/UserGuide/best-practices.html#lock-away-credentials[root user credentials,type="documentation"]). +* Maintaining the hardware, software, and infrastructure that supports the connected Kubernetes cluster. +* Securing their {aws} accounts (for example, through safeguarding your link:IAM/latest/UserGuide/best-practices.html#lock-away-credentials[root user credentials,type="documentation"]). \ No newline at end of file diff --git a/latest/ug/connector/troubleshooting-connector.adoc b/latest/ug/connector/troubleshooting-connector.adoc index 3440d66ea..7dd90ca24 100644 --- a/latest/ug/connector/troubleshooting-connector.adoc +++ b/latest/ug/connector/troubleshooting-connector.adoc @@ -1,27 +1,23 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[troubleshooting-connector,troubleshooting-connector.title]] +[#troubleshooting-connector] = Troubleshoot Amazon EKS Connector issues -:info_doctype: section -:info_title: Troubleshoot Amazon EKS Connector issues -:info_titleabbrev: Troubleshoot Amazon EKS Connector -:info_abstract: Troubleshoot and resolve common issues when using Amazon EKS Connector to connect your Kubernetes clusters to Amazon EKS. - -include::../attributes.txt[] +:info_titleabbrev: Troubleshoot EKS Connector [abstract] -- -Troubleshoot and resolve common issues when using Amazon EKS Connector to connect your [.noloc]`Kubernetes` clusters to Amazon EKS. +Troubleshoot and resolve common issues when using Amazon EKS Connector to connect your Kubernetes clusters to Amazon EKS. -- This topic covers some of the common errors that you might encounter while using the Amazon EKS Connector, including instructions on how to resolve them and workarounds. -[[tsc-steps,tsc-steps.title]] +[#tsc-steps] == Basic troubleshooting This section describes steps to diagnose Amazon EKS Connector issues. -[[tsc-check,tsc-check.title]] +[#tsc-check] === Check Amazon EKS Connector status To check the Amazon EKS Connector status, type: @@ -32,10 +28,10 @@ kubectl get pods -n eks-connector ---- -[[tsc-logs,tsc-logs.title]] +[#tsc-logs] === Inspect Amazon EKS Connector logs -The Amazon EKS Connector [.noloc]`Pod` consists of three containers. To retrieve full logs for all of these containers so that you can inspect them, run the following commands: +The Amazon EKS Connector Pod consists of three containers. To retrieve full logs for all of these containers so that you can inspect them, run the following commands: @@ -62,10 +58,10 @@ kubectl exec eks-connector-1 --container connector-agent -n eks-connector -- cat ---- -[[tsc-name,tsc-name.title]] +[#tsc-name] === Get the effective cluster name -Amazon EKS clusters are uniquely identified by `clusterName` within a single {aws} account and {aws} Region. If you have multiple connected clusters in Amazon EKS, you can confirm which Amazon EKS cluster that the current [.noloc]`Kubernetes` cluster is registered to. To do this, enter the following to find out the `clusterName` of the current cluster. +Amazon EKS clusters are uniquely identified by `clusterName` within a single {aws} account and {aws} Region. If you have multiple connected clusters in Amazon EKS, you can confirm which Amazon EKS cluster that the current Kubernetes cluster is registered to. To do this, enter the following to find out the `clusterName` of the current cluster. // Not using subs="quotes" here with [.replaceable]`region-code` because the * characters get dropped. [source,bash,subs="verbatim,attributes"] @@ -77,12 +73,12 @@ kubectl exec eks-connector-1 --container connector-agent -n eks-connector \ ---- -[[tsc-misc,tsc-misc.title]] +[#tsc-misc] === Miscellaneous commands The following commands are useful to retrieve information that you need to troubleshoot issues. -* Use the following command to gather images that's used by [.noloc]`Pods` in Amazon EKS Connector. +* Use the following command to gather images that's used by Pods in Amazon EKS Connector. // Not using subs="quotes" here with [.replaceable]`region-code` because the * characters get dropped. + [source,bash,subs="verbatim,attributes"] @@ -95,7 +91,7 @@ kubectl get pods -n eks-connector -o jsonpath="{.items[*].spec.containers[*].ima ---- kubectl get pods -n eks-connector -o jsonpath="{.items[*].spec.nodeName}" | tr -s '[[:space:]]' '\n' ---- -* Run the following command to get your [.noloc]`Kubernetes` client and server versions. +* Run the following command to get your Kubernetes client and server versions. + [source,bash,subs="verbatim,attributes"] ---- @@ -127,24 +123,24 @@ docker logout public.ecr.aws ---- -[[symp-pending,symp-pending.title]] +[#symp-pending] == Console error: the cluster is stuck in the Pending state -If the cluster gets stuck in the `Pending` state on the Amazon EKS console after you're registered it, it might be because the Amazon EKS Connector didn't successfully connect the cluster to {aws} yet. For a registered cluster, the `Pending` state means that the connection isn't successfully established. To resolve this issue, make sure that you have applied the manifest to the target [.noloc]`Kubernetes` cluster. If you applied it to the cluster, but the cluster is still in the `Pending` state, then the `eks-connector` statefulset might be unhealthy. To troubleshoot this issue, see <>in this topic. +If the cluster gets stuck in the `Pending` state on the Amazon EKS console after you're registered it, it might be because the Amazon EKS Connector didn't successfully connect the cluster to {aws} yet. For a registered cluster, the `Pending` state means that the connection isn't successfully established. To resolve this issue, make sure that you have applied the manifest to the target Kubernetes cluster. If you applied it to the cluster, but the cluster is still in the `Pending` state, then the `eks-connector` statefulset might be unhealthy. To troubleshoot this issue, see <>in this topic. -[[symp-imp,symp-imp.title]] +[#symp-imp] == Console error: User system:serviceaccount:eks-connector:eks-connector can't impersonate resource users in API group at cluster scope -The Amazon EKS Connector uses [.noloc]`Kubernetes` https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation[user impersonation] to act on behalf of link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principals,type="documentation"] from the {aws-management-console}. Each principal that accesses the [.noloc]`Kubernetes` API from the {aws} `eks-connector` service account must be granted permission to impersonate the corresponding [.noloc]`Kubernetes` user with an IAM ARN as its [.noloc]`Kubernetes` user name. In the following examples, the IAM ARN is mapped to a [.noloc]`Kubernetes` user. +The Amazon EKS Connector uses Kubernetes https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation[user impersonation] to act on behalf of link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principals,type="documentation"] from the {aws-management-console}. Each principal that accesses the Kubernetes API from the {aws} `eks-connector` service account must be granted permission to impersonate the corresponding Kubernetes user with an IAM ARN as its Kubernetes user name. In the following examples, the IAM ARN is mapped to a Kubernetes user. -* IAM user [.replaceable]`john` from {aws} account [.replaceable]`111122223333` is mapped to a [.noloc]`Kubernetes` user. link:IAM/latest/UserGuide/id_users.html[IAM best practices,type="documentation"] recommend that you grant permissions to roles instead of users. +* IAM user [.replaceable]`john` from {aws} account [.replaceable]`111122223333` is mapped to a Kubernetes user. link:IAM/latest/UserGuide/id_users.html[IAM best practices,type="documentation"] recommend that you grant permissions to roles instead of users. + [source,bash,subs="verbatim,attributes"] ---- {arn-aws}iam::111122223333:user/john ---- -* IAM role [.replaceable]`admin` from {aws} account [.replaceable]`111122223333` is mapped to a [.noloc]`Kubernetes` user: +* IAM role [.replaceable]`admin` from {aws} account [.replaceable]`111122223333` is mapped to a Kubernetes user: + [source,bash,subs="verbatim,attributes"] ---- @@ -155,12 +151,12 @@ The result is an IAM role ARN, instead of the {aws} STS session ARN. For instructions on how to configure the `ClusterRole` and `ClusterRoleBinding` to grant the `eks-connector` service account privilege to impersonate the mapped user, see <>. Make sure that in the template, `%IAM_ARN%` is replaced with the IAM ARN of the {aws-management-console} IAM principal. -[[symp-rbac,symp-rbac.title]] +[#symp-rbac] == Console error: [...] is forbidden: User [...] cannot list resource [...] in API group at the cluster scope -Consider the following problem. The Amazon EKS Connector has successfully impersonated the requesting {aws-management-console} IAM principal in the target [.noloc]`Kubernetes` cluster. However, the impersonated principal doesn't have RBAC permission for [.noloc]`Kubernetes` API operations. +Consider the following problem. The Amazon EKS Connector has successfully impersonated the requesting {aws-management-console} IAM principal in the target Kubernetes cluster. However, the impersonated principal doesn't have RBAC permission for Kubernetes API operations. -To resolve this issue, there are two methods to give permissions to additional users. If you previously installed eks-connector via helm chart, you can easily grant users access by running the following command. Replace the `userARN1` and `userARN2` with a list of the ARNs of the IAM roles to give access to view the [.noloc]`Kubernetes` resources: +To resolve this issue, there are two methods to give permissions to additional users. If you previously installed eks-connector via helm chart, you can easily grant users access by running the following command. Replace the `userARN1` and `userARN2` with a list of the ARNs of the IAM roles to give access to view the Kubernetes resources: [source,shell,subs="verbatim,attributes"] ---- @@ -169,22 +165,22 @@ helm upgrade eks-connector oci://public.ecr.aws/eks-connector/eks-connector-char --set 'authentication.allowedUserARNs={userARN1,userARN2}' ---- -Or, as the cluster administrator, grant the appropriate level of RBAC privileges to individual [.noloc]`Kubernetes` users. For more information and examples, see <>. +Or, as the cluster administrator, grant the appropriate level of RBAC privileges to individual Kubernetes users. For more information and examples, see <>. -[[symp-con,symp-con.title]] -== Console error: Amazon EKS can't communicate with your [.noloc]`Kubernetes` cluster API server. The cluster must be in an ACTIVE state for successful connection. Try again in few minutes. +[#symp-con] +== Console error: Amazon EKS can't communicate with your Kubernetes cluster API server. The cluster must be in an ACTIVE state for successful connection. Try again in few minutes. If the Amazon EKS service can't communicate with the Amazon EKS connector in the target cluster, it might be because of one of the following reasons: * The Amazon EKS Connector in the target cluster is unhealthy. * Poor connectivity or an interrupted connection between the target cluster and the {aws} Region. -To resolve this problem, check the <>. If you don't see an error for the Amazon EKS Connector, retry the connection after a few minutes. If you regularly experience high latency or intermittent connectivity for the target cluster, consider re-registering the cluster to an {aws} Region that's located closer to you. +To resolve this problem, check the <>. If you don't see an error for the Amazon EKS Connector, retry the connection after a few minutes. If you regularly experience high latency or intermittent connectivity for the target cluster, consider re-registering the cluster to an {aws} Region that's located closer to you. -[[symp-loop,symp-loop.title]] -== Amazon EKS connector [.noloc]`Pods` are crash looping +[#symp-loop] +== Amazon EKS connector Pods are crash looping -There are many reasons that can cause an Amazon EKS connector [.noloc]`Pod` to enter the `CrashLoopBackOff` status. This issue likely involves the `connector-init` container. Check the status of the Amazon EKS connector [.noloc]`Pod`. +There are many reasons that can cause an Amazon EKS connector Pod to enter the `CrashLoopBackOff` status. This issue likely involves the `connector-init` container. Check the status of the Amazon EKS connector Pod. [source,bash,subs="verbatim,attributes"] ---- @@ -201,7 +197,7 @@ eks-connector-0 0/2 Init:CrashLoopBackOff 1 7s If your output is similar to the previous output, see <> to troubleshoot the issue. -[[symp-regis,symp-regis.title]] +[#symp-regis] == Failed to initiate eks-connector: InvalidActivation When you start the Amazon EKS Connector for the first time, it registers an `activationId` and `activationCode` with Amazon Web Services. The registration might fail, which can cause the `connector-init` container to crash with an error similar to the following error. @@ -215,8 +211,8 @@ To troubleshoot this issue, consider the following causes and recommended fixes: -* Registration might have failed because the `activationId` and `activationCode` aren't in your manifest file. If this is the case, make sure that they are the correct values that were returned from the `RegisterCluster` API operation, and that the `activationCode` is in the manifest file. The `activationCode` is added to [.noloc]`Kubernetes` secrets, so it must be `base64` encoded. For more information, see <>. -* Registration might have failed because your activation expired. This is because, for security reasons, you must activate the Amazon EKS Connector within three days after registering the cluster. To resolve this issue, make sure that the Amazon EKS Connector manifest is applied to the target [.noloc]`Kubernetes` cluster before the expiry date and time. To confirm your activation expiry date, call the `DescribeCluster` API operation. +* Registration might have failed because the `activationId` and `activationCode` aren't in your manifest file. If this is the case, make sure that they are the correct values that were returned from the `RegisterCluster` API operation, and that the `activationCode` is in the manifest file. The `activationCode` is added to Kubernetes secrets, so it must be `base64` encoded. For more information, see <>. +* Registration might have failed because your activation expired. This is because, for security reasons, you must activate the Amazon EKS Connector within three days after registering the cluster. To resolve this issue, make sure that the Amazon EKS Connector manifest is applied to the target Kubernetes cluster before the expiry date and time. To confirm your activation expiry date, call the `DescribeCluster` API operation. + [source,bash,subs="verbatim,attributes"] ---- @@ -249,15 +245,15 @@ In the following example response, the expiry date and time is recorded as `2021 If the `activationExpiry` passed, deregister the cluster and register it again. Doing this generates a new activation. -[[symp-out,symp-out.title]] +[#symp-out] == Cluster node is missing outbound connectivity To work properly, the Amazon EKS Connector requires outbound connectivity to several {aws} endpoints. You can't connect a private cluster without outbound connectivity to a target {aws} Region. To resolve this issue, you must add the necessary outbound connectivity. For information about connector requirements, see <>. -[[symp-img,symp-img.title]] -== Amazon EKS connector [.noloc]`Pods` are in `ImagePullBackOff` state +[#symp-img] +== Amazon EKS connector Pods are in `ImagePullBackOff` state -If you run the `get pods` command and [.noloc]`Pods` are in the `ImagePullBackOff` state, they can't work properly. If the Amazon EKS Connector [.noloc]`Pods` are in the `ImagePullBackOff` state, they can't work properly. Check the status of your Amazon EKS Connector [.noloc]`Pods`. +If you run the `get pods` command and Pods are in the `ImagePullBackOff` state, they can't work properly. If the Amazon EKS Connector Pods are in the `ImagePullBackOff` state, they can't work properly. Check the status of your Amazon EKS Connector Pods. [source,bash,subs="verbatim,attributes"] ---- @@ -272,4 +268,4 @@ NAME READY STATUS RESTARTS AGE eks-connector-0 0/2 Init:ImagePullBackOff 0 4s ---- -The default Amazon EKS Connector manifest file references images from the https://gallery.ecr.aws/[Amazon ECR Public Gallery]. It's possible that the target [.noloc]`Kubernetes` cluster can't pull images from the Amazon ECR Public Gallery. Either resolve the Amazon ECR Public Gallery image pull issue, or consider mirroring the images in the private container registry of your choice. +The default Amazon EKS Connector manifest file references images from the https://gallery.ecr.aws/[Amazon ECR Public Gallery]. It's possible that the target Kubernetes cluster can't pull images from the Amazon ECR Public Gallery. Either resolve the Amazon ECR Public Gallery image pull issue, or consider mirroring the images in the private container registry of your choice. \ No newline at end of file diff --git a/latest/ug/connector/tsc-faq.adoc b/latest/ug/connector/tsc-faq.adoc index 82c65f309..388f1afc1 100644 --- a/latest/ug/connector/tsc-faq.adoc +++ b/latest/ug/connector/tsc-faq.adoc @@ -1,27 +1,23 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[tsc-faq,tsc-faq.title]] +[#tsc-faq] = {aws} Connector frequently asked questions -:info_doctype: section -:info_title: {aws} Connector frequently asked questions :info_titleabbrev: Frequently asked questions -:info_abstract: Learn to connect and manage Kubernetes clusters outside {aws} with Amazon EKS Connector, enabling unified cluster visibility and management across environments using a secure, outbound-only connection. [abstract] -- -Learn to connect and manage [.noloc]`Kubernetes` clusters outside {aws} with Amazon EKS Connector, enabling unified cluster visibility and management across environments using a secure, outbound-only connection. +Learn to connect and manage Kubernetes clusters outside {aws} with Amazon EKS Connector, enabling unified cluster visibility and management across environments using a secure, outbound-only connection. -- .Q: How does the underlying technology behind the Amazon EKS Connector work? -A: The Amazon EKS Connector is based on the {aws} Systems Manager (Systems Manager) agent. The Amazon EKS Connector runs as a `StatefulSet` on your [.noloc]`Kubernetes` cluster. It establishes a connection and proxies the communication between the API server of your cluster and Amazon Web Services. It does this to display cluster data in the Amazon EKS console until you disconnect the cluster from {aws}. The Systems Manager agent is an open source project. For more information about this project, see the https://github.com/aws/amazon-ssm-agent[GitHub project page]. +A: The Amazon EKS Connector is based on the {aws} Systems Manager (Systems Manager) agent. The Amazon EKS Connector runs as a `StatefulSet` on your Kubernetes cluster. It establishes a connection and proxies the communication between the API server of your cluster and Amazon Web Services. It does this to display cluster data in the Amazon EKS console until you disconnect the cluster from {aws}. The Systems Manager agent is an open source project. For more information about this project, see the https://github.com/aws/amazon-ssm-agent[GitHub project page]. -.Q: I have an on-premises [.noloc]`Kubernetes` cluster that I want to connect. Do I need to open firewall ports to connect it? -A: No, you don't need to open any firewall ports. The [.noloc]`Kubernetes` cluster only requires outbound connection to {aws} Regions. {aws} services never access resources in your on-premises network. The Amazon EKS Connector runs on your cluster and initiates the connection to {aws}. When the cluster registration completes, {aws} only issues commands to the Amazon EKS Connector after you start an action from the Amazon EKS console that requires information from the [.noloc]`Kubernetes` API server on your cluster. +.Q: I have an on-premises Kubernetes cluster that I want to connect. Do I need to open firewall ports to connect it? +A: No, you don't need to open any firewall ports. The Kubernetes cluster only requires outbound connection to {aws} Regions. {aws} services never access resources in your on-premises network. The Amazon EKS Connector runs on your cluster and initiates the connection to {aws}. When the cluster registration completes, {aws} only issues commands to the Amazon EKS Connector after you start an action from the Amazon EKS console that requires information from the Kubernetes API server on your cluster. .Q: What data is sent from my cluster to {aws} by the Amazon EKS Connector? -A: The Amazon EKS Connector sends technical information that's necessary for your cluster to be registered on {aws}. It also sends cluster and workload metadata for the Amazon EKS console features that customers request. The Amazon EKS Connector only gathers or sends this data if you start an action from the Amazon EKS console or the Amazon EKS API that necessitates the data to be sent to {aws}. Other than the [.noloc]`Kubernetes` version number, {aws} doesn't store any data by default. It stores data only if you authorize it to. +A: The Amazon EKS Connector sends technical information that's necessary for your cluster to be registered on {aws}. It also sends cluster and workload metadata for the Amazon EKS console features that customers request. The Amazon EKS Connector only gathers or sends this data if you start an action from the Amazon EKS console or the Amazon EKS API that necessitates the data to be sent to {aws}. Other than the Kubernetes version number, {aws} doesn't store any data by default. It stores data only if you authorize it to. .Q: Can I connect a cluster outside of an {aws} Region? -A: Yes, you can connect a cluster from any location to Amazon EKS. Moreover, your Amazon EKS service can be located in any {aws} public commercial {aws} Region. This works with a valid network connection from your cluster to the target {aws} Region. We recommend that you pick an {aws} Region that is closest to your cluster location for UI performance optimization. For example, if you have a cluster running in Tokyo, connect your cluster to the {aws} Region in Tokyo (that is, the `ap-northeast-1` {aws} Region) for low latency. You can connect a cluster from any location to Amazon EKS in any of the public commercial {aws} Regions, except the China or GovCloud {aws} Regions. +A: Yes, you can connect a cluster from any location to Amazon EKS. Moreover, your Amazon EKS service can be located in any {aws} public commercial {aws} Region. This works with a valid network connection from your cluster to the target {aws} Region. We recommend that you pick an {aws} Region that is closest to your cluster location for UI performance optimization. For example, if you have a cluster running in Tokyo, connect your cluster to the {aws} Region in Tokyo (that is, the `ap-northeast-1` {aws} Region) for low latency. You can connect a cluster from any location to Amazon EKS in any of the public commercial {aws} Regions, except the China or GovCloud {aws} Regions. \ No newline at end of file diff --git a/latest/ug/contribute/asciidoc-syntax.adoc b/latest/ug/contribute/asciidoc-syntax.adoc new file mode 100644 index 000000000..9a6a79384 --- /dev/null +++ b/latest/ug/contribute/asciidoc-syntax.adoc @@ -0,0 +1,197 @@ +include::../attributes.txt[] + +[.topic] +[#asciidoc-syntax] += AsciiDoc syntax reference +:info_titleabbrev: AsciiDoc syntax + +[abstract] +-- +This page provides a quick reference guide to AsciiDoc syntax for contributing to Amazon EKS documentation. +-- + +AsciiDoc is the preferred markup language for {aws} documentation. While the tooling has partial support for Markdown syntax (such as headings and lists), we recommend using AsciiDoc syntax for all content to ensure consistency and proper rendering. + +This guide covers common syntax elements you'll need when contributing to Amazon EKS documentation. For more advanced syntax and detailed information, refer to the https://docs.asciidoctor.org/asciidoc/latest/[AsciiDoc documentation]. + +== New page + +Every new AsciiDoc document should begin with the structure defined in <>. + +== Headings + +Use headings to organize your content hierarchically: + +[source,asciidoc] +---- += Page/topic title (level 1) +== Section title (level 2) +=== Level 3 heading +==== Level 4 heading +===== Level 5 heading +====== Level 6 heading +---- + +Note: Always use sentence case for headings in {aws} documentation. + +== Text formatting + +Format text to emphasize important information: + +[source,asciidoc] +---- +Use *bold text* for UI labels. +Use _italic text_ for introducing terms or light emphasis. +Use `monospace text` for code, file names, and commands. +---- + +== Lists + +=== Unordered lists + +Create bulleted lists for items without a specific sequence: + +[source,asciidoc] +---- +* First item +* Second item +** Nested item +** Another nested item +* Third item +---- + +=== Ordered lists + +Create numbered lists for sequential steps or prioritized items: + +[source,asciidoc] +---- +. First step +. Second step +.. Substep 1 +.. Substep 2 +. Third step +---- + +== Links + +See <> for details on how to properly format links. Markdown-style links are not supported. + +== Code examples + +=== Inline code + +Use backticks for inline code: + +[source,asciidoc] +---- +Use the `kubectl get pods` command to list all pods. +---- + +=== Code blocks + +Create code blocks with syntax highlighting and support for attributes (similar to entities): + +[source,asciidoc] +---- + [source,python,subs="verbatim,attributes"] + ---- + def hello_world(): + print("Hello, World!") + ---- +---- + +== Images + +Insert images with alt text for accessibility: + +[source,asciidoc] +---- +image::images/image-file.png[Alt text description] +---- + +== Tables + +Create tables to organize information: + +[source,asciidoc] +---- +[%header,cols="2"] +|=== +|Header 1 +|Header 2 + +|Cell 1,1 +|Cell 1,2 + +|Cell 2,1 +|Cell 2,2 +|=== +---- + +For more complex tables, see the https://docs.asciidoctor.org/asciidoc/latest/tables/build-a-basic-table/[AsciiDoc table documentation]. + +== Callouts + +Use callouts to highlight important information and admonitions: + +[source,asciidoc] +---- +NOTE: This is a note callout for general information. + +TIP: This is a tip callout for helpful advice. + +IMPORTANT: This is an important callout for critical information. + +---- + +Preview: + +NOTE: This is a note callout. + + +== Including other files + +Include content from other files: + +[source,asciidoc] +---- + include::filename.adoc[] +---- + +== Attributes (similar to entities) + +Use predefined attributes to maintain consistency. In particular, you MUST use attributes for {aws} and `{arn-aws}`. + +[source,asciidoc] +---- +{aws} provides Amazon EKS as a managed Kubernetes service. +---- + +[source,asciidoc] +---- + [source,bash,subs="verbatim,attributes"] + ---- + aws iam attach-role-policy \ + --role-name AmazonEKSAutoClusterRole \ + --policy-arn {arn-aws}iam::aws:policy/AmazonEKSClusterPolicy + ---- +---- + +For a list of attributes, look in the `../attributes.txt` file. + +== Procedures + +Format step-by-step procedures: + +[source,asciidoc] +---- +To create an Amaozon EKS cluster. do the following steps. + +. Sign in to the {aws} Management Console. +. Open the Amazon EKS console. +. Choose *Create cluster*. + +... + +---- diff --git a/latest/ug/contribute/contribute.adoc b/latest/ug/contribute/contribute.adoc index fa868a907..851f8308c 100644 --- a/latest/ug/contribute/contribute.adoc +++ b/latest/ug/contribute/contribute.adoc @@ -1,20 +1,40 @@ -[[contribute,contribute.title]] -# Contribute to the EKS User Guide -:info_titleabbrev: Contribute - include::../attributes.txt[] +[#contribute] += Contribute to the EKS User Guide +:info_titleabbrev: Contribute + -{aws} is building an improved contribution experience for the EKS User Guide. +{aws} has launched an improved contribution experience for the EKS User Guide. -The previous GitHub repository at `awsdocs/amazon-eks-user-guide` is temporarily unavailable while we prepare the new contribution system. +You can now edit the EKS User Guide source directly on GitHub. -The updated experience will use AsciiDoc, a powerful authoring language similar to markdown. AsciiDoc combines simple syntax with enterprise documentation features like advanced formatting, cross-referencing, and security controls. +The docs now use AsciiDoc, a powerful authoring language similar to markdown. AsciiDoc combines simple syntax with enterprise documentation features like advanced formatting, cross-referencing, and security controls. -When the EKS User Guide returns to GitHub in mid-November, you'll be able to edit the documentation source files directly. Our streamlined process includes: +You can now edit the EKS Docs directly on GitHub. Our streamlined process includes: * Faster pull request processing * Reduced manual steps * Automated content quality checks We look forward to your contributions. + +include::edit-single-web.adoc[leveloffset=+1] + +include::edit-web.adoc[leveloffset=+1] + +//include::vale-github.adoc[leveloffset=+1] + +include::vale-local.adoc[leveloffset=+1] + +include::create-page.adoc[leveloffset=+1] + +include::insert-link.adoc[leveloffset=+1] + +include::create-content-q.adoc[leveloffset=+1] + +include::pr-preview.adoc[leveloffset=+1] + +include::asciidoc-syntax.adoc[leveloffset=+1] + +//include::pr-status.adoc[leveloffset=+1] \ No newline at end of file diff --git a/latest/ug/contribute/create-content-q.adoc b/latest/ug/contribute/create-content-q.adoc new file mode 100644 index 000000000..523f0f496 --- /dev/null +++ b/latest/ug/contribute/create-content-q.adoc @@ -0,0 +1,51 @@ +include::../attributes.txt[] + +[.topic] +[#create-content-q] += Create docs content with Amazon Q +:info_titleabbrev: Create with Amazon Q + + +You can use Amazon Q to create and revise docs content. This is an easy way to get started on a new page. Amazon Q is available as an extension to Visual Studio (VS) Code. + +In the following image, Amazon Q generated the lines marked with green. + +image::images/contribute-q.png["Amazon Q in VS Code"] + +== Install Amazon Q with VS Code + +1. Open VS Code +2. Go to the Extensions view (Ctrl+Shift+X or Cmd+Shift+X) +3. Search for "Amazon Q" +4. Click Install on the Amazon Q extension +5. Wait for installation to complete +6. Restart VS Code when prompted + +== Login to Amazon Q + +1. After installing, choose the Amazon Q icon in the VS Code activity bar. +2. Choose *Sign in to Amazon Q*. +3. Enter your {aws} credentials when prompted. +4. Once authenticated, you'll see the Amazon Q chat interface. + +== Use Amazon Q to create content + +1. Open the file you want to edit in VS Code. +2. Select the text you want to revise or the location for new content. +3. Press *Ctrl+I* or *Cmd+I*. +4. In the prompt, be specific about: + * The type of content you need. + * The target audience. + * Key points to cover. + * Desired tone and style. +5. Review the generated content in the inline preview. +6. Use *enter* to accept the changes, or *esc* to reject them. +7. Edit further as needed. + +== Tips + +* Start with a simple request and iterate to get the content you want. +* Create a first draft of the page headings, then ask Q to fill them in. +* Amazon Q might output Markdown. This is fine. The AsciiDoc tooling can understand most markdown syntax. + +To learn more about Amazon Q Developer, see link:amazonq/latest/qdeveloper-ug/q-in-IDE.html["Using Amazon Q Developer in the IDE",type="documentation"]. \ No newline at end of file diff --git a/latest/ug/contribute/create-page.adoc b/latest/ug/contribute/create-page.adoc new file mode 100644 index 000000000..bfcf8e343 --- /dev/null +++ b/latest/ug/contribute/create-page.adoc @@ -0,0 +1,50 @@ +include::../attributes.txt[] + +[.topic] +[#create-page] += Create a new page +:info_titleabbrev: Create page + +[abstract] +-- +This topic includes instructions for creating the initial page metadata and adding the page to the guide table of contents. +-- + +Learn how to create a new documentation page. This topic includes instructions for creating the initial page metadata and adding the page to the guide table of contents. + +== Create page + +. Navigate to the chapter directory. For example, if you want to create a new page in the "Security" section, navigate to the `latest/ug/security` directory. +. Determine the page ID. By convention, the page ID is all lowercase and segmented with `-`. The ID of this page is `create-page`. +. Create a new file with the page ID and the `adoc` extension. For example, `create-page.adoc`. +. Insert the page metadata using this template: + + +[source,asciidoc] +---- + include::../attributes.txt[] + + [.topic] + [#unique-page-id] + = Page title + :info_titleabbrev: Short title + + [abstract] + -- + Brief summary of the page's content. + -- + + Introduction paragraph goes here. +---- + + +== Add page to navigation + +. Navigate to the parent page. The parent page of top level sections is `book.adoc`. +. At the bottom of the parent page, include the child page. ++ +`include::${filename}[leveloffset=+1]` ++ +_For example:_ ++ +`include::create-page.adoc[leveloffset=+1]` \ No newline at end of file diff --git a/latest/ug/contribute/edit-single-web.adoc b/latest/ug/contribute/edit-single-web.adoc new file mode 100644 index 000000000..ff450f82e --- /dev/null +++ b/latest/ug/contribute/edit-single-web.adoc @@ -0,0 +1,61 @@ +include::../attributes.txt[] + +[.topic] +[#edit-single-web] += Edit a single page from a web browser +:info_titleabbrev: Edit single page + + +You can easily edit a single page in the EKS User Guide directly through your web browser. + +image::images/contribute-web-edit.png["View of GitHub web edit interface"] + +If you want to edit multiple pages from your web browser, see <>. + +== Prerequisites + +* Docs page to change opened in web browser. +* Signed into GitHub. + +== Procedure + +. Navigate to the page you want to edit in the Amazon EKS User Guide. + +. In the right pane, choose the *Edit this page on GitHub* link. + +. Once on GitHub, open the editor by either: +** Pressing the `e` key on your keyboard. +** Clicking the pencil icon and selecting *Edit in Place* from the dropdown menu. +** If you don't have the option to edit, you need to login to GitHub. Your GitHub account does not need any special permissions to suggest changes. However, internal Amazon contributors should link their GitHub profile. + +. Make your required changes to the content in the GitHub editor. +** The editor provides syntax highlighting and preview capabilities. +** You can use AsciiDoc markup to format your changes. +** You can use `ctrl-f` to open a find/replace interface. + +. (Optional) Preview your changes. +** Use the `preview` tab to preview your changes with rich formatting. +** Use the `show diff` option to highlight changed sections. Removed sections have a red indicator in the left margin. New sections have a green indicator in the left margin. + +. When finished editing, click the *Commit changes...* button at the top of the editor + +. In the commit dialog: +** Verify your email address is correct. +** Add a brief but descriptive commit message explaining your changes. +** Optionally add a longer description if needed. +** Select to create a new branch and pull request. + +You have created a pull request including the proposed changes. + +== Pull request overview + +When you create a PR: + +* Your changes are submitted for review by repository maintainers. +* Reviewers can comment on your changes and request modifications. +* Automated tests may run to validate your changes. +* Once approved, your changes can be merged into the main repository. + +Pull requests help ensure quality and provide a way to discuss changes before they are integrated. + +https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/reviewing-changes-in-pull-requests/about-pull-request-reviews[Learn how pull requests are reviewed and approved in the GitHub Docs.] \ No newline at end of file diff --git a/latest/ug/contribute/edit-web.adoc b/latest/ug/contribute/edit-web.adoc new file mode 100644 index 000000000..e5dd4e4a1 --- /dev/null +++ b/latest/ug/contribute/edit-web.adoc @@ -0,0 +1,34 @@ +include::../attributes.txt[] + +[.topic] +[#edit-web] += Edit multiple files from a web browser with the GitHub Web Editor +:info_titleabbrev: Edit files with GitHub + + +If you want to propose change to multiple pages, or create a new docs page, use the GitHub.dev web editor. This web editor is based on the popular Visual Studio Code text editor. + +image::images/contribute-web-dev.png["GitHub.dev web editor user interface] + +== Prerequisites + +* Logged in to GitHub +* Familiarity with Visual Studio Code editor +* Familiarity with Git + +== Procedure + +NOTE: The EKS Docs team has created a workspace file that includes suggested configurations for the editor, such as text wrapping and AsciiDoc syntax highlighting. We suggest you load this workspace file. + +. Open the https://github.dev/awsdocs/amazon-eks-user-guide/blob/mainline/eks-docs.code-workspace?workspace=true[workspace] on GitHub.dev. +** You can bookmark the URL `https://github.dev/awsdocs/amazon-eks-user-guide/blob/mainline/eks-docs.code-workspace?workspace=true` +. (First time setup only) You may be prompted to create a fork of the repo in your own GitHub account. Accept this prompt. For more information, see https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/about-forks[About forks] in the GitHub docs. +. (First time setup only) Accept the prompt in the bottom right to install the AsciiDoc extension. +. Navigate to the docs content at `latest/ug`. +** Docs files are organized by their top level section. For example, pages in the "Security" chapter have source files under the "security/" directory. +. To view a preview of a docs page, use the *Open preview to the Side* button in the top right. The icon includes a small magnifying glass. +. Use the *Source Control* tab in the left to commit your changes. For more information, see the Visual Studio Code docs: +** https://code.visualstudio.com/docs/sourcecontrol/overview#_commit[Commit changes] +** https://code.visualstudio.com/docs/sourcecontrol/github#_creating-pull-requests[Create a pull request] + +After you create a pull request, it will be reviewed by the docs team. \ No newline at end of file diff --git a/latest/ug/contribute/images b/latest/ug/contribute/images new file mode 120000 index 000000000..5e6757319 --- /dev/null +++ b/latest/ug/contribute/images @@ -0,0 +1 @@ +../images \ No newline at end of file diff --git a/latest/ug/contribute/insert-link.adoc b/latest/ug/contribute/insert-link.adoc new file mode 100644 index 000000000..dcdfde101 --- /dev/null +++ b/latest/ug/contribute/insert-link.adoc @@ -0,0 +1,55 @@ +include::../attributes.txt[] + +[.topic] +[#insert-link] += Insert a link +:info_titleabbrev: Insert link + + +AsciiDoc supports multiple types of links. Using the right link type is important so the link works properly in different environments. + +== Link to a page or section in the EKS User Guide + +Use cross references (xref) to link between pages or sections within the same documentation site, such as the EKS User Guide. They automatically update if the target section moves or is renamed. + +=== Use page title as link text + +For most cases when linking to another ID in this user guide, use the following approach to have the link text automatically update to the latest title as needed. + +[source,asciidoc] +---- +For more information, see <>. +---- + + +=== Define custom link text + +For cases where you must have custom link text, use the following format. + +[source,asciidoc] +---- +Here's an example of a <>. +---- + + +== Link to another guide in the {aws} Docs + +. Find the link to the {aws} documentation page. +. Remove the `https://docs.aws.amazon.com/` prefix, keeping only the path. The path should start with a letter. +. Create a link as shown below: + +[source,asciidoc] +---- +link:AmazonS3/latest/userguide/create-bucket-overview.html[Create a bucket, type="documentation"] +---- + +== Link to an external webpage + +This format creates a standard link out to a page not hosted by Amazon. For example, use this for GitHub links. + +[source,asciidoc] +---- +https://example.com[Link text] +---- + +NOTE: We have an allowlist for external domains. The allowlist is at `vale/styles/EksDocs/ExternalDomains.yml` \ No newline at end of file diff --git a/latest/ug/contribute/pr-preview.adoc b/latest/ug/contribute/pr-preview.adoc new file mode 100644 index 000000000..c8cfa2ac5 --- /dev/null +++ b/latest/ug/contribute/pr-preview.adoc @@ -0,0 +1,38 @@ +include::../attributes.txt[] + +[.topic] +[#pr-preview] += View a preview of pull request content +:info_titleabbrev: View PR preview + +The Amazon EKS User Guide GitHub is configured to build and generate a preview of the docs site. This preview doesn't have the full {aws} theme, but it does check the content builds properly and links work. + +image::images/contribute-preview.png["GitHub comment with preview URL"] + +This preview is hosted at a temporary URL by {aws} Amplify. + +== View preview + +When you submit a pull request, {aws} Amplify attempts to build and deploy a preview of the content. + +If the build succeeds, *aws-amplify-us-east-1* adds a comment to the pull request that has a link to the preview. Choose the link to the right of "Access this pull request here" (as called out in the screenshot with a red outline). + +If the build fails, the repo admins can see the logs and provide feedback. + +NOTE: If you haven't contributed before, a project maintainer may need to approve running the build. + +== Preview limitations + +The preview is built as a single large HTML file. It will be displayed as multiple pages when published. + +*What works:* + +* Cross references (`xref`) +* Links to the internet +* Images +* Content hosted from `samples/` + +*What doesn't work:* + +* Links to other {aws} content, using `type="documentation"`. This is because this content doesn't exist in the preview environment. +* The attribute `\{aws}` will not display properly. The value of this changes based on the environment. \ No newline at end of file diff --git a/latest/ug/contribute/pr-status.adoc b/latest/ug/contribute/pr-status.adoc new file mode 100644 index 000000000..e69de29bb diff --git a/latest/ug/contribute/vale-github.adoc b/latest/ug/contribute/vale-github.adoc new file mode 100644 index 000000000..b4059ad95 --- /dev/null +++ b/latest/ug/contribute/vale-github.adoc @@ -0,0 +1,29 @@ +include::../attributes.txt[] + +[.topic] +[#vale-github] += View style feedback online for a pull request +:info_titleabbrev: View PR feedback + + +When you create a pull request to propose docs changes, multiple GitHub actions run. This includes a style check using Vale. + +image::images/contribute-style-web.png["View style feedback on GitHub"] + +The style check: + +* Returns an error if the string "AWS" is used instead of the variable `{aws}` +** Pull requests cannot be merged until this is resolved. +* Adds style comments to the pull request. + +== View style feedback + +. Open your Pull Request +** https://github.com/awsdocs/amazon-eks-user-guide/pulls[View a list of open pull requests] +. Choose the *Files changed* tab. +. Feedback from Vale is visible as line comments, that start with `[vale]`. +** Use the style feedback to identify typos, spelling errors, and awkward phrasing. + +When you update a pull request, the Vale check runs again. + +Learn how to https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request#making-changes-to-files-in-your-pull-request[Make changes to files in your pull request] in the GitHub docs. \ No newline at end of file diff --git a/latest/ug/contribute/vale-local.adoc b/latest/ug/contribute/vale-local.adoc new file mode 100644 index 000000000..c22387bea --- /dev/null +++ b/latest/ug/contribute/vale-local.adoc @@ -0,0 +1,52 @@ +include::../attributes.txt[] + +[.topic] +[#vale-local] += View style feedback as you type by installing Vale locally +:info_titleabbrev: View style feedback + + +You can see style feedback as you type. This helps identify awkward writing and typos. + +image::images/contribute-style-local.png["View style feedback in VS Code] + +*Overview:* + +* The Vale CLI loads style guides and runs them against source files. +* The EKS Docs repo includes a vale configuration file that loads style guides and local rules. +* The Vale extension for Visual Studio (VS) Code displays vale feedback inside the editor. + +== Install Vale + +Follow the instructions in the Vale CLI docs to https://vale.sh/docs/install#package-managers[Install Vale with a Package Manager]. + +== Install VS Code Vale extension + +. Open VS Code. +. Click the Extensions icon in the Activity Bar (or press Ctrl+Shift+X). +. Search for "Vale". +. Click Install on the "Vale VSCode" extension by Chris Chinchilla. +. Reload VS Code when prompted. + +== Sync Vale + +Vale uses the `.vale.ini` configuration file in your project root to determine which style rules to apply. + +. Open VS Code. +. Click *View* > *Terminal* (or press Ctrl+`). +. Navigate to your project root directory if needed. +. Run the command: ++ +[source,bash] +---- +vale sync +---- +. Wait for Vale to finish downloading and syncing style rules + +== View style feedback in VS Code + +. Open a Markdown or AsciiDoc file in VS Code. +. The Vale extension will automatically check your text against the style rules. +. Style issues will be underlined in the editor. +. Hover over underlined text to see the specific style suggestion. +. Fix issues by following the suggestions or consulting the style guide. \ No newline at end of file diff --git a/latest/ug/doc-history.adoc b/latest/ug/doc-history.adoc index cdacfe8e2..19b6a04b2 100644 --- a/latest/ug/doc-history.adoc +++ b/latest/ug/doc-history.adoc @@ -1,47 +1,378 @@ -//!!NODE_ROOT +include::attributes.txt[] [.topic] -[[doc-history,doc-history.title]] -// H1 title is necessary, and must occur before the [abstract], but is unused in the web page (:info_title: is used instead, and :info_titleabbrev: is used in the ToC) +[#doc-history] = Document history -:doctype: book -:sectnums: -:toc: left -:icons: font -:experimental: -:idprefix: -:idseparator: - -:sourcedir: . -:info_doctype: chapter -:info_title: Document history -:keywords: document, publish, release, history, log -:info_abstract: Important updates to the Amazon EKS documentation, sorted by date, with brief \ - descriptions of each update and when they occurred. - - -include::attributes.txt[] [abstract] -- Important updates to the Amazon EKS documentation, sorted by date, with brief descriptions of each update and when they occurred. -- -The following table describes the major updates and new features for the Amazon EKS User Guide. We also update the documentation frequently to address the feedback that you send us. +The following table describes some of the major updates and new features for the Amazon EKS User Guide. To receive notifications when this table gets a new entry, you can subscribe to the following URL with an RSS reader: + +[source,none] +---- +https://docs.aws.amazon.com/eks/latest/userguide/doc-history.rss +---- [.updates] == Updates + +[.update,date="2025-10-22"] +=== {aws} managed policy updates +[.update-ulink] +link:eks/latest/userguide/security-iam-awsmanpol.html[type="documentation"] + +Added `ec2:DescribeRouteTables` and `ec2:DescribeNetworkAcls` permissions to `AmazonEKSServiceRolePolicy`. This allows Amazon EKS to detect configuration issues with VPC route tables and network ACLs for hybrid nodes as part of cluster insights. + + +[.update,date="2025-10-15"] +=== Service linked role update +[.update-ulink] +link:eks/latest/userguide/using-service-linked-roles-eks-connector.html[type="documentation"] + +Added `ssmmessages:OpenDataChannel` permission to `AmazonEKSConnectorServiceRolePolicy` + + +[.update,date="2025-10-02"] +=== Kubernetes version `1.34` +[.update-ulink] +link:eks/latest/userguide/kubernetes-versions.html#kubernetes-1-34[type="documentation"] + +Added Kubernetes version `1.34` support for new clusters and version upgrades. + + +[.update,date="2025-09-22"] +=== Configurable node auto repair +[.update-ulink] +link:eks/latest/userguide/node-health.html[type="documentation"] + +Amazon EKS now provides more granular control over the node auto repair behavior. + + +[.update,date="2025-08-27"] +=== Refresh cluster insights +[.update-ulink] +link:eks/latest/userguide/view-cluster-insights.html[type="documentation"] + +You can now manually refresh cluster insights. + + +[.update,date="2025-08-26"] +=== {aws} managed policy updates +[.update-ulink] +link:eks/latest/userguide/security-iam-awsmanpol.html[type="documentation"] + +Added permission to `AmazonEKSServiceRolePolicy`. This role can attach new access policy `AmazonEKSEventPolicy`. Restricted permissions for `ec2:DeleteLaunchTemplate` and `ec2:TerminateInstances`. + + +[.update,date="2025-08-19"] +=== Cross-service confused deputy prevention +[.update-ulink] +link:eks/latest/userguide/cross-service-confused-deputy-prevention.html[type="documentation"] + +Added a topic with an example trust policy that you can apply for Cross-service confused deputy prevention. +Amazon EKS accepts the `aws:SourceArn` and `aws:SourceAccount` conditions in the trust policy of an EKS cluster role. + + +[.update,date="2025-07-30"] +=== Amazon EKS platform version update +[.update-ulink] +link:eks/latest/userguide/platform-versions.html[type="documentation"] + +This is a new platform version with security fixes and enhancements. This includes new patch versions of Kubernetes `1.33.2`, `1.32.6`, `1.31.10`, and `1.30.14`. + + +[.update,date="2025-07-15"] +=== VPC CNI Multi-NIC feature for multi-homed pods +[.update-ulink] +link:eks/latest/userguide/pod-multiple-network-interfaces.html[type="documentation"] + +Amazon EKS adds multi-homed pods to the VPC CNI. Now you can configure a workload and the VPC CNI assigned IP addresses from every NIC on the EC2 instance to each pod. The application can make concurrent connections to use the bandwidth from each NIC. Every network interface is configured in the same subnet and security groups as the node. + +Previously, you needed to use Multus CNI to run multiple other CNIs to create multi-homed pods. Documentation and steps to do this have been moved to link:eks/latest/userguide/pod-multus.html[type="documentation"]. + + +[.update,date="2025-06-30"] +=== VPC CNI troubleshooting content update +[.update-ulink] +link:eks/latest/userguide/network-policies-troubleshooting.html[type="documentation"] + +Expanded the troubleshooting page for Kubernetes _network policy_ in the VPC CNI. Added the CRDs and RBAC permissions, and 12 known issues. + + +[.update,date="2025-06-26"] +=== Amazon EKS platform version update +[.update-ulink] +link:eks/latest/userguide/platform-versions.html[type="documentation"] + +This is a new platform version with security fixes and enhancements. There aren't any new patch versions of Kubernetes in these platform version. + +[.update,date="2025-06-26"] +=== {aws} managed policy updates +[.update-ulink] +link:eks/latest/userguide/security-iam-awsmanpol.html[type="documentation"] + +Added `ssmmessages:OpenDataChannel` permission to `AmazonEKSLocalOutpostServiceRolePolicy`. + + +[.update,date="2025-06-20"] +=== {aws} managed policy updates +[.update-ulink] +link:eks/latest/userguide/security-iam-awsmanpol.html[type="documentation"] + +Added permissions to `AmazonEKSServiceRolePolicy` and `AmazonEKSComputePolicy` to allow Amazon EKS Auto Mode to launch instances by using the EC2 On-Demand Capacity Reservations in your account. Also, added the permissions to `AmazonEKSComputePolicy` for Amazon EKS to create the EC2 Spot service-linked role on your behalf. + + +[.update,date="2025-06-19"] +=== Managed policy updates +[.update-ulink] +link:eks/latest/userguide/security-iam-awsmanpol.html[type="documentation"] + +Added `AmazonEKSDashboardConsoleReadOnly` policy. + + +[.update,date="2025-06-13"] +=== Amazon EKS Auto Mode update to `NodeClass` +[.update-ulink] +link:eks/latest/userguide/create-node-class.html#auto-node-class-spec[type="documentation"] + +The `NodeClass` template for Auto Mode nodes added configuration for separate pod subnets. This adds the optional keys `podSubnetSelectorTerms` and `podSecurityGroupSelectorTerms` to set the subnets and security groups for the pods. + + +[.update,date="2025-06-11"] +=== Amazon EKS platform version update +[.update-ulink] +link:eks/latest/userguide/platform-versions.html[type="documentation"] + +This is a new platform version with security fixes and enhancements. There aren't any new patch versions of Kubernetes in these platform version. + +[.update,date="2025-06-11"] +=== Target secondary and cross-account roles with EKS Pod Identities +[.update-ulink] +link:eks/latest/userguide/pod-id-assign-target-role.html[type="documentation"] + +Amazon EKS adds __target IAM roles__ to EKS Pod Identities for automated role chaining. You can use this to automatically assume a role in another account and EKS Pod Identity rotates the temporary credentials. Each Pod Identity association must have an IAM role in the same account to assume first, then it uses that role to assume the target role. + + +[.update,date="2025-06-06"] +=== Amazon EKS {aws} Region expansion +Amazon EKS is now available in the Asia Pacific (Taipei) (`ap-east-2`) {aws} Region. + +[.update,date="2025-06-05"] +=== `IPv6` access control for dual-stack public endpoints for new `IPv6` clusters +[.update-ulink] +link:eks/latest/userguide/cluster-endpoint.html[type="documentation"] + +Amazon EKS adds `IPv6` CIDR blocks to control access to the public cluster endpoint for new `IPv6` clusters. Previously, you could only add `IPv4` CIDR blocks to allow traffic to the public cluster endpoint, even for dual-stack cluster endpoints. + + +[.update,date="2025-05-30"] +=== Amazon EKS platform version update +[.update-ulink] +link:eks/latest/userguide/platform-versions.html[type="documentation"] + +This is a new platform version with security fixes and enhancements. This includes new patch versions of Kubernetes `1.32.5`, `1.31.9`, and `1.30.13`. + + +[.update,date="2025-05-29"] +=== Kubernetes version `1.33` +[.update-ulink] +link:eks/latest/userguide/kubernetes-versions-standard.html#kubernetes-1-33[type="documentation"] + +Added Kubernetes version `1.33` support for new clusters and version upgrades. + + +[.update,date="2025-05-29"] +=== New cluster insights for EKS Hybrid Nodes +[.update-ulink] +link:eks/latest/userguide/cluster-insights.html[type="documentation"] + +Amazon EKS adds new cluster insights that check the configuration of your hybrid nodes. These insight checks will warn you about issues with on-premises nodes and pods and the remote network configuration of the cluster. + + +[.update,date="2025-05-23"] +=== Add-on support for Amazon FSx CSI driver +[.update-ulink] +link:eks/latest/userguide/fsx-csi.html[type="documentation"] + +You can now use the {aws} Management Console, {aws} CLI, and API to manage the Amazon FSx CSI driver. + + +[.update,date="2025-05-22"] +=== Edit Prometheus scrapers in the console +[.update-ulink] +link:eks/latest/userguide/prometheus.html#viewing-prometheus-scraper-details[type="documentation"] + +You can now edit Amazon Managed Service for Prometheus scrapers in the Amazon EKS console. + + +[.update,date="2025-05-21"] +=== Managed policy updates +[.update-ulink] +link:eks/latest/userguide/security-iam-awsmanpol.html[type="documentation"] + +Added `AmazonEKSDashboardServiceRolePolicy` policy. + + +[.update,date="2025-05-16"] +=== Amazon EKS platform version update +[.update-ulink] +link:eks/latest/userguide/platform-versions.html[type="documentation"] + +This is a new platform version with security fixes and enhancements. There aren't any new patch versions of Kubernetes in these platform versions. + + + + +[.update,date="2025-05-02"] +=== New pages for Amazon FSx for Lustre performance +[.update-ulink] +link:eks/latest/userguide/fsx-csi.html[type="documentation"] + +Added new topics with details on optimizing Amazon FSx for Lustre performance. + + +[.update,date="2025-04-30"] +=== Amazon EKS Auto Mode update to `NodeClass` +[.update-ulink] +link:eks/latest/userguide/create-node-class.html#auto-node-class-spec[type="documentation"] + +The `NodeClass` template for Auto Mode nodes added configuration for forward network proxies. This adds the optional key `advancedNetworking` to set your HTTPS proxy. + + +[.update,date="2025-04-29"] +=== Amazon EKS platform version update +[.update-ulink] +link:eks/latest/userguide/platform-versions.html[type="documentation"] + +This is a new platform version with security fixes and enhancements. There aren't any new patch versions of Kubernetes in these platform versions. + + +[.update,date="2025-04-29"] +=== Bottlerocket for hybrid nodes +[.update-ulink] +link:eks/latest/userguide/hybrid-nodes-bottlerocket.html[type="documentation"] + +Bottlerocket is now available for EKS Hybrid Nodes. + + +[.update,date="2025-04-18"] +=== New concepts pages for hybrid networking +[.update-ulink] +link:eks/latest/userguide/hybrid-nodes-concepts.html[type="documentation"] + +Added pages for concepts of EKS Hybrid Nodes. These cover the on-premises and cloud networking in detail with diagrams. + + +[.update,date="2025-04-16"] +=== {aws} managed policy updates +[.update-ulink] +link:eks/latest/userguide/security-iam-awsmanpol.html[type="documentation"] + +Added permissions to `AmazonEKSClusterPolicy` to allow Amazon EKS to elastic network interfaces created by the VPC CNI. This is required so that EKS can clean up elastic network interfaces that are left behind if the VPC CNI quits unexpectedly. + + +[.update,date="2025-04-14"] +=== {aws} managed policy updates +[.update-ulink] +link:eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-updates[type="documentation"] + +Added permissions to `AmazonEKSServiceRolePolicy` to allow EKS AI/ML customers to add Egress rules to the default EKS Cluster security group. + + +=== Kubernetes version 1.31 is now available for local clusters on {aws} Outposts +[.update-ulink] +link:eks/latest/userguide/eks-outposts-platform-versions.html[type="documentation"] + +You can now create an Amazon EKS local cluster on an {aws} Outposts using Kubernetes version 1.31. + + +[.update,date="2025-03-31"] +=== Node health for EKS Hybrid Nodes +[.update-ulink] +link:eks/latest/userguide/node-health.html[type="documentation"] + +You can use `eks-node-monitoring-agent` on hybrid nodes, starting from version `1.2.0-eksbuild.1`. Run `eks-node-monitoring-agent` as an Amazon EKS add-on to detect and show health issues. + + +[.update,date="2025-03-31"] +=== EKS Hybrid Nodes for existing clusters +[.update-ulink] +link:eks/latest/userguide/hybrid-nodes-cluster-update.html[type="documentation"] + +You can now add, change, or remove the hybrid nodes configuration of existing clusters. Previously, you could only add the hybrid nodes configuration to new clusters when you created them. +With Amazon EKS Hybrid Nodes, you can use your on-premises and edge infrastructure as nodes in Amazon EKS clusters. {aws} manages the {aws}-hosted Kubernetes control plane of the Amazon EKS cluster, and you manage the hybrid nodes that run in your on-premises or edge environments. + + +[.update,date="2025-03-28"] +=== Rollback: Prevent accidental upgrades with cluster insights +[.update-ulink] +link:eks/latest/userguide/update-cluster.html#update-cluster-control-plane[type="documentation"] + +Amazon EKS has temporarily rolled back a feature that would +require you to use a `--force` flag to upgrade your cluster when there were certain cluster insight issues. For more information, see link:https://github.com/aws/containers-roadmap/issues/2570[Temporary rollback of enforcing upgrade insights on update cluster version] on GitHub. + + +[.update,date="2025-03-27"] +=== Bottlerocket FIPS AMIs +[.update-ulink] +link:eks/latest/userguide/bottlerocket-fips-amis.html[type="documentation"] + +Bottlerocket FIPS AMIs are now available in standard managed node groups. + + +//[.update,date="2025-03-26"] +//=== Prevent accidental upgrades with cluster insights +//[.update-ulink] +//link:eks/latest/userguide/update-cluster.html#update-cluster-control-plane[type="documentation"] + +//Cluster insights will proactively prevent you from upgrading your cluster for certain issues. You can override upgrade-blocking readiness checks if needed. + + +[.update,date="2025-02-28"] +=== {aws} managed policy updates +[.update-ulink] +link:eks/latest/userguide/security-iam-awsmanpol.html[type="documentation"] + +Added permissions to `AmazonEKSServiceRolePolicy` to allow Amazon EKS to terminate EC2 instances created by Auto Mode. +Added permissions to `AmazonEKSServiceRolePolicy` to allow Amazon EKS to terminate EC2 instances created by Auto Mode. + + +[.update,date="2025-01-27"] +=== Update strategies for managed node groups +[.update-ulink] +link:eks/latest/userguide/managed-node-update-behavior.html#managed-node-update-upgrade[type="documentation"] + +You can now use update strategies to configure the version update process for managed node groups. This introduces the _minimal_ update strategy to terminate nodes before making new ones, which is useful in capacity constrained environments. The _default_ update strategy continues the existing behavior. + + +[.update,date="2025-01-23"] +=== Kubernetes version `1.32` +[.update-ulink] +link:eks/latest/userguide/kubernetes-versions.html#kubernetes-1-32[type="documentation"] + +Added Kubernetes version `1.32` support for new clusters and version upgrades. + +[.update,date="2025-01-14"] +=== Amazon EKS {aws} Region expansion +Amazon EKS is now available in the Asia Pacific (Thailand) Region (`ap-southeast-7`) and Mexico (Central) (`mx-central-1`) {aws} Regions. EKS Auto Mode and VPC Endpoints for the EKS API aren't available in either Region. + + [.update,date="2025-01-13"] === {aws} managed policy updates [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html +link:eks/latest/userguide/security-iam-awsmanpol.html[type="documentation"] Added multiple permissions to `AmazonEBSCSIDriverPolicy` to allow the Amazon EBS CSI Driver restore all snapshots, enable Fast Snapshot Restore (FSR) on EBS volumes, and modify tags on volumes. + [.update,date="2024-12-26"] === {aws} managed policy updates [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html +link:eks/latest/userguide/security-iam-awsmanpol.html[type="documentation"] Added permissions to `AmazonEKSLoadBalancingPolicy`. @@ -49,15 +380,15 @@ Added permissions to `AmazonEKSLoadBalancingPolicy`. [.update,date="2024-12-20"] === Updated cluster insights [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/cluster-insights.html +link:eks/latest/userguide/cluster-insights.html[type="documentation"] -Amazon EKS upgrade insights will now warn about more cluster health and version compatibility issues. It can detect issues between different [.noloc]`Kubernetes` and Amazon EKS components such as `kubelet`, `kube-proxy`, and Amazon EKS add-ons. +Amazon EKS upgrade insights will now warn about more cluster health and version compatibility issues. It can detect issues between different Kubernetes and Amazon EKS components such as `kubelet`, `kube-proxy`, and Amazon EKS add-ons. [.update,date="2024-12-16"] === Node monitoring agent and auto repair [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/node-health.html +link:eks/latest/userguide/node-health.html[type="documentation"] You can use the new `eks-node-monitoring-agent` as an Amazon EKS add-on to detect and show health issues. You can also enable node auto repair to automatically replace nodes when issues are detected. @@ -65,7 +396,7 @@ You can use the new `eks-node-monitoring-agent` as an Amazon EKS add-on to detec [.update,date="2024-12-01"] === Amazon EKS Hybrid Nodes [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/hybrid-nodes-overview.html +link:eks/latest/userguide/hybrid-nodes-overview.html[type="documentation"] You can now run node on-premises connected to Amazon EKS clusters. With Amazon EKS Hybrid Nodes, you can use your on-premises and edge infrastructure as nodes in Amazon EKS clusters. {aws} manages the {aws}-hosted Kubernetes control plane of the Amazon EKS cluster, and you manage the hybrid nodes that run in your on-premises or edge environments. @@ -73,7 +404,7 @@ You can now run node on-premises connected to Amazon EKS clusters. With Amazon E [.update,date="2024-12-01"] === Amazon EKS Auto Mode [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/automode.html +link:eks/latest/userguide/automode.html[type="documentation"] Amazon EKS Auto Mode fully automates Kubernetes cluster infrastructure management for compute, storage, and networking on {aws}. It simplifies Kubernetes management by automatically provisioning infrastructure, selecting optimal compute instances, dynamically scaling resources, continuously optimizing costs, patching operating systems, and integrating with {aws} security services. @@ -81,7 +412,7 @@ Amazon EKS Auto Mode fully automates Kubernetes cluster infrastructure managemen [.update,date="2024-11-22"] === {aws} managed policy updates [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html +link:eks/latest/userguide/security-iam-awsmanpol.html[type="documentation"] Updated `AWSServiceRoleForAmazonEKSNodegroup` for compatibility with China regions. @@ -89,7 +420,7 @@ Updated `AWSServiceRoleForAmazonEKSNodegroup` for compatibility with China regio [.update,date="2024-11-22"] === Amazon EKS platform version update [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/platform-versions.html +link:eks/latest/userguide/platform-versions.html[type="documentation"] This is a new platform version with security fixes and enhancements. This includes new patch versions of Kubernetes `1.31.2`, `1.30.6`, `1.29.10`, and `1.28.15`. @@ -97,31 +428,31 @@ This is a new platform version with security fixes and enhancements. This includ [.update,date="2024-11-21"] === {aws} managed policy updates [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html +link:eks/latest/userguide/security-iam-awsmanpol.html[type="documentation"] EKS updated {aws} managed policy `AmazonEKSLocalOutpostClusterPolicy`. Added `ec2:DescribeAvailabilityZones` permission so the {aws} Cloud Controller Manager on the cluster control plane can identify the Availability Zone that each node is in. [.update,date="2024-11-21"] -=== [.noloc]`Kubernetes` version 1.30 is now available for local clusters on {aws} Outposts +=== Kubernetes version 1.30 is now available for local clusters on {aws} Outposts [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/eks-outposts-platform-versions.html +link:eks/latest/userguide/eks-outposts-platform-versions.html[type="documentation"] -You can now create an Amazon EKS local cluster on an {aws} Outposts using [.noloc]`Kubernetes` version 1.30. +You can now create an Amazon EKS local cluster on an {aws} Outposts using Kubernetes version 1.30. [.update,date="2024-11-20"] -=== [.noloc]`Bottlerocket` AMIs that use FIPS 140-3 +=== Bottlerocket AMIs that use FIPS 140-3 [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/retrieve-ami-id-bottlerocket.html +link:eks/latest/userguide/retrieve-ami-id-bottlerocket.html[type="documentation"] -[.noloc]`Bottlerocket` AMIs are available that are preconfigured to use FIPS 140-3 validated cryptographic modules. This includes the Amazon Linux 2023 Kernel Crypto API Cryptographic Module and the {aws}-LC Cryptographic Module. +Bottlerocket AMIs are available that are preconfigured to use FIPS 140-3 validated cryptographic modules. This includes the Amazon Linux 2023 Kernel Crypto API Cryptographic Module and the {aws}-LC Cryptographic Module. [.update,date="2024-11-20"] === {aws} managed policy updates [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html +link:eks/latest/userguide/security-iam-awsmanpol.html[type="documentation"] Updated `AWSServiceRoleForAmazonEKSNodegroup` policy to allow `ec2:RebootInstances` for instances created by Amazon EKS managed node groups. Restricted the `ec2:CreateTags` permissions for Amazon EC2 resources. @@ -129,7 +460,7 @@ Updated `AWSServiceRoleForAmazonEKSNodegroup` policy to allow `ec2:RebootInstanc [.update,date="2024-11-18"] === Observability dashboard [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/observability-dashboard.html +link:eks/latest/userguide/observability-dashboard.html[type="documentation"] The observability dashboard helps you to quickly detect, troubleshoot, and remediate issues. There are also new link:eks/latest/userguide/cloudwatch.html[CloudWatch vended metrics,type="documentation"] available in the `AWS/EKS` namespace. @@ -137,7 +468,7 @@ The observability dashboard helps you to quickly detect, troubleshoot, and remed [.update,date="2024-11-16"] === {aws} managed policy updates [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html +link:eks/latest/userguide/security-iam-awsmanpol.html[type="documentation"] EKS updated {aws} managed policy `AmazonEKSServiceRolePolicy`. Added permissions for EKS access policies, load balancer management, and automated cluster resource cleanup. @@ -145,7 +476,7 @@ EKS updated {aws} managed policy `AmazonEKSServiceRolePolicy`. Added permissions [.update,date="2024-11-15"] === New role creation in console for add-ons that support EKS Pod Identities [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/creating-an-add-on.html#_create_add_on_console +link:eks/latest/userguide/creating-an-add-on.html#_create_add_on_console[type="documentation"] There are new steps when using the console to create or update add-ons that support EKS Pod Identities where you can automatically generate IAM roles with the appropriate name, role policy, and trust policy for the add-on. @@ -153,7 +484,7 @@ There are new steps when using the console to create or update add-ons that supp [.update,date="2024-11-15"] === Managed node groups in {aws} Local Zones [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/local-zones.html +link:eks/latest/userguide/local-zones.html[type="documentation"] Managed node groups can now be created in {aws} Local Zones. @@ -161,7 +492,7 @@ Managed node groups can now be created in {aws} Local Zones. [.update,date="2024-11-11"] === New metrics are available [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/view-raw-metrics.html +link:eks/latest/userguide/view-raw-metrics.html[type="documentation"] There are new metrics available under the API group `metrics.eks.amazonaws.com`. @@ -169,7 +500,7 @@ There are new metrics available under the API group `metrics.eks.amazonaws.com`. [.update,date="2024-11-07"] === {aws} managed policy updates [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html +link:eks/latest/userguide/security-iam-awsmanpol.html[type="documentation"] EKS updated {aws} managed policy `AmazonEKSComputePolicy`. Updated resource permissions for the `iam:AddRoleToInstanceProfile` action. @@ -177,7 +508,7 @@ EKS updated {aws} managed policy `AmazonEKSComputePolicy`. Updated resource perm [.update,date="2024-11-01"] === {aws} managed policy updates [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html +link:eks/latest/userguide/security-iam-awsmanpol.html[type="documentation"] EKS added a new {aws} managed policy: `AmazonEKSComputePolicy` @@ -185,7 +516,7 @@ EKS added a new {aws} managed policy: `AmazonEKSComputePolicy` [.update,date="2024-11-01"] === {aws} managed policy updates [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html +link:eks/latest/userguide/security-iam-awsmanpol.html[type="documentation"] Added permissions to `AmazonEKSClusterPolicy`. Added `ec2:DescribeInstanceTopology` permission to allow Amazon EKS to attach topology information to the node as labels. @@ -193,46 +524,43 @@ Added permissions to `AmazonEKSClusterPolicy`. Added `ec2:DescribeInstanceTopolo [.update,date="2024-10-30"] === {aws} managed policy updates [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html +link:eks/latest/userguide/security-iam-awsmanpol.html[type="documentation"] EKS added a new {aws} managed policy: `AmazonEKSBlockStoragePolicy` - [.update,date="2024-10-30"] === {aws} managed policy updates [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html +link:eks/latest/userguide/security-iam-awsmanpol.html[type="documentation"] EKS added a new {aws} managed policy: `AmazonEKSLoadBalancingPolicy` - [.update,date="2024-10-29"] === {aws} managed policy updates [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html +link:eks/latest/userguide/security-iam-awsmanpol.html[type="documentation"] Added `cloudwatch:PutMetricData` permissions to `AmazonEKSServiceRolePolicy` to allow Amazon EKS to publish metrics to Amazon CloudWatch. - [.update,date="2024-10-28"] === {aws} managed policy updates [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html +link:eks/latest/userguide/security-iam-awsmanpol.html[type="documentation"] EKS added a new {aws} managed policy: `AmazonEKSNetworkingPolicy` - [.update,date="2024-10-21"] === {aws} managed policy updates [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html +link:eks/latest/userguide/security-iam-awsmanpol.html[type="documentation"] Added `autoscaling:ResumeProcesses`, `autoscaling:SuspendProcesses`, and associated permissions to `AWSServiceRoleForAmazonEKSNodegroup` in China regions to integrate with Amazon Application Recovery Controller for EKS. No changes to other regions. + [.update,date="2024-10-21"] === Dual-stack endpoints for new `IPv6` clusters [.update-ulink] @@ -244,7 +572,7 @@ Connect to new `IPv6` clusters with a `eks-cluster.[.replaceable]``region``.api. [.update,date="2024-10-10"] === {aws} managed policy updates [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html +link:eks/latest/userguide/security-iam-awsmanpol.html[type="documentation"] Added permissions to `AmazonEKSServicePolicy` and `AmazonEKSServiceRolePolicy`. Added `ec2:GetSecurityGroupsForVpc` and associated tag permissions to allow EKS to read security group information and update related tags. @@ -252,7 +580,7 @@ Added permissions to `AmazonEKSServicePolicy` and `AmazonEKSServiceRolePolicy`. [.update,date="2024-10-11"] === AL2023 accelerated AMIs [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/retrieve-ami-id.html +link:eks/latest/userguide/retrieve-ami-id.html[type="documentation"] You can now use accelerated `NVIDIA` and {aws} Neuron instances for AMIs based on AL2023. @@ -266,40 +594,40 @@ We have switched over to a new source format with some layout changes. There are [.update,date="2024-10-03"] === {aws} managed policy updates - New policy [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-updates +link:eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-updates[type="documentation"] EKS added a new {aws} managed policy. [.update,date="2024-09-24"] -=== [.noloc]`Kubernetes` version `1.31` +=== Kubernetes version `1.31` [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html#kubernetes-1.31 +link:eks/latest/userguide/kubernetes-versions.html#kubernetes-1-31[type="documentation"] -Added [.noloc]`Kubernetes` version `1.31` support for new clusters and version upgrades. +Added Kubernetes version `1.31` support for new clusters and version upgrades. [.update,date="2024-08-21"] === {aws} managed policy updates - Update to an existing policy [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-updates +link:eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-updates[type="documentation"] Amazon EKS updated an existing {aws} managed policy. [.update,date="2024-08-20"] -=== [.noloc]`Kubernetes` version 1.29 is now available for local clusters on {aws} Outposts +=== Kubernetes version 1.29 is now available for local clusters on {aws} Outposts [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/eks-outposts-platform-versions.html +link:eks/latest/userguide/eks-outposts-platform-versions.html[type="documentation"] -You can now create an Amazon EKS local cluster on an {aws} Outposts using [.noloc]`Kubernetes` version 1.29. +You can now create an Amazon EKS local cluster on an {aws} Outposts using Kubernetes version 1.29. [.update,date="2024-08-14"] === EKS Pod Identity in {aws} GovCloud (US) [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/pod-identites.html +link:eks/latest/userguide/pod-identites.html[type="documentation"] -Amazon EKS Pod Identities associate an IAM role with a [.noloc]`Kubernetes` service account. With this feature, you no longer need to provide extended permissions to the node IAM role. This way, [.noloc]`Pods` on that node can call {aws} APIs. Unlike IAM roles for service accounts, EKS Pod Identities are completely inside EKS; you don't need an [.noloc]`OIDC` identity provider. +Amazon EKS Pod Identities associate an IAM role with a Kubernetes service account. With this feature, you no longer need to provide extended permissions to the node IAM role. This way, Pods on that node can call {aws} APIs. Unlike IAM roles for service accounts, EKS Pod Identities are completely inside EKS; you don't need an OIDC identity provider. [.update,date="2024-08-09"] @@ -310,7 +638,7 @@ We renamed and updated topics to be more scenario-driven throughout the entire g [.update,date="2024-08-07"] === Dual-stack VPC interface endpoints for Amazon EKS [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/vpc-interface-endpoints.html +link:eks/latest/userguide/vpc-interface-endpoints.html[type="documentation"] You can now create dual-stack VPC interface endpoints for Amazon EKS with both `IPv4` and `IPv6` IP addresses and DNS names. @@ -318,7 +646,7 @@ You can now create dual-stack VPC interface endpoints for Amazon EKS with both ` [.update,date="2024-08-01"] === New dual-stack endpoints for the Amazon EKS APIs with `IPv6` addresses [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/network-reqs.html +link:eks/latest/userguide/network-reqs.html[type="documentation"] The EKS API for creating and managing clusters, and the OIDC issuer URLs for clusters have new dual-stack endpoints. The new DNS name for the Amazon EKS API is `eks.[.replaceable]``region``.api.aws` which resolves to `IPv4` addresses and `IPv6` addresses. New clusters have a new dual-stack OIDC issuer URL (`oidc-eks.[.replaceable]``region``.api.aws`). @@ -326,7 +654,7 @@ The EKS API for creating and managing clusters, and the OIDC issuer URLs for clu [.update,date="2024-07-01"] === Capacity Blocks for managed node groups [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/capacity-blocks-mng.html +link:eks/latest/userguide/capacity-blocks-mng.html[type="documentation"] You can now use Capacity Blocks for managed node groups. @@ -334,7 +662,7 @@ You can now use Capacity Blocks for managed node groups. [.update,date="2024-06-28"] === Auto Scaling Group metrics collection enabled by default [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/enable-asg-metrics.html +link:eks/latest/userguide/enable-asg-metrics.html[type="documentation"] Amazon EKS managed node groups now have Amazon EC2 Auto Scaling group metrics enabled by default with no additional charge. Previously, you had to do several steps to enable this feature. @@ -342,7 +670,7 @@ Amazon EKS managed node groups now have Amazon EC2 Auto Scaling group metrics en [.update,date="2024-06-27"] === {aws} managed policy updates - Update to an existing policy [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-updates +link:eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-updates[type="documentation"] Amazon EKS updated an existing {aws} managed policy. @@ -350,55 +678,55 @@ Amazon EKS updated an existing {aws} managed policy. [.update,date="2024-06-12"] === Improvements to AMI information references [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-amis.html +link:eks/latest/userguide/eks-optimized-amis.html[type="documentation"] -We made improvements to the AMI information references, in particular for [.noloc]`Bottlerocket`. +We made improvements to the AMI information references, in particular for Bottlerocket. [.update,date="2024-06-12"] -=== [.noloc]`Kubernetes` version `1.26` +=== Kubernetes version `1.26` [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html#kubernetes-1.26 +link:eks/latest/userguide/kubernetes-versions.html[type="documentation"] -[.noloc]`Kubernetes` version `1.26` is now in extended support. +Kubernetes version `1.26` is now in extended support. [.update,date="2024-05-23"] -=== [.noloc]`Kubernetes` version `1.30` +=== Kubernetes version `1.30` [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html#kubernetes-1.30 +link:eks/latest/userguide/kubernetes-versions.html#kubernetes-1-30[type="documentation"] -Added [.noloc]`Kubernetes` version `1.30` support for new clusters and version upgrades. +Added Kubernetes version `1.30` support for new clusters and version upgrades. [.update,date="2024-05-14"] -=== [.noloc]`CoreDNS` Autoscaling +=== CoreDNS Autoscaling [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/coredns-autoscaling.html +link:eks/latest/userguide/coredns-autoscaling.html[type="documentation"] -[.noloc]`CoreDNS` autoscaler will dynamically adapt the number of replicas of the [.noloc]`CoreDNS` deployment in an EKS cluster based on the number of nodes and CPU cores. This feature works for [.noloc]`CoreDNS` `v1.9` and the latest platform version of EKS release version `1.25` and later. +CoreDNS autoscaler will dynamically adapt the number of replicas of the CoreDNS deployment in an EKS cluster based on the number of nodes and CPU cores. This feature works for CoreDNS `v1.9` and the latest platform version of EKS release version `1.25` and later. [.update,date="2024-05-14"] === Amazon EKS platform version update [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/platform-versions.html +link:eks/latest/userguide/platform-versions.html[type="documentation"] This is a new platform version with security fixes and enhancements. This includes new patch versions of Kubernetes `1.29.4`, `1.28.9`, and `1.27.13`. [.update,date="2024-04-10"] -=== CloudWatch [.noloc]`Container Insights` support for [.noloc]`Windows` +=== CloudWatch Container Insights support for Windows [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/cloudwatch.html +link:eks/latest/userguide/cloudwatch.html[type="documentation"] -The Amazon CloudWatch Observability Operator add-on now also allows [.noloc]`Container Insights` on [.noloc]`Windows` worker nodes in the cluster. +The Amazon CloudWatch Observability Operator add-on now also allows Container Insights on Windows worker nodes in the cluster. [.update,date="2024-04-05"] -=== [.noloc]`Kubernetes` concepts +=== Kubernetes concepts [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-concepts.html +link:eks/latest/userguide/kubernetes-concepts.html[type="documentation"] Added new Kubernetes concepts topic. @@ -406,23 +734,23 @@ Added new Kubernetes concepts topic. [.update,date="2024-04-02"] === Restructure Access and IAM Content [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/cluster-auth.html +link:eks/latest/userguide/cluster-auth.html[type="documentation"] Move existing pages related to access and IAM topics, such as auth config map, access entries, Pod ID, and IRSA into new section. Revise overview content. [.update,date="2024-03-13"] -=== [.noloc]`Bottlerocket` OS support for Amazon S3 CSI driver +=== Bottlerocket OS support for Amazon S3 CSI driver [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/s3-csi.html +link:eks/latest/userguide/s3-csi.html[type="documentation"] -The Mountpoint for Amazon S3 CSI driver is now compatible with [.noloc]`Bottlerocket`. +The Mountpoint for Amazon S3 CSI driver is now compatible with Bottlerocket. [.update,date="2024-03-04"] === {aws} managed policy updates - Update to an existing policy [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-updates +link:eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-updates[type="documentation"] Amazon EKS updated an existing {aws} managed policy. @@ -430,39 +758,39 @@ Amazon EKS updated an existing {aws} managed policy. [.update,date="2024-02-29"] === Amazon Linux 2023 [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/al2023.html +link:eks/latest/userguide/al2023.html[type="documentation"] Amazon Linux 2023 (AL2023) is a new Linux-based operating system designed to provide a secure, stable, and high-performance environment for your cloud applications. [.update,date="2024-02-26"] -=== EKS Pod Identity and IRSA support sidecars in [.noloc]`Kubernetes` `1.29` +=== EKS Pod Identity and IRSA support sidecars in Kubernetes `1.29` [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html#kubernetes-1.29 +link:eks/latest/userguide/kubernetes-versions.html#kubernetes-1-29[type="documentation"] -In [.noloc]`Kubernetes` `1.29`, sidecar containers are available in Amazon EKS clusters. Sidecar containers are supported with IAM roles for service accounts or EKS Pod Identity. For more information about sidecars, see https://kubernetes.io/docs/concepts/workloads/pods/sidecar-containers/[Sidecar Containers] in the [.noloc]`Kubernetes` documentation. +In Kubernetes `1.29`, sidecar containers are available in Amazon EKS clusters. Sidecar containers are supported with IAM roles for service accounts or EKS Pod Identity. For more information about sidecars, see https://kubernetes.io/docs/concepts/workloads/pods/sidecar-containers/[Sidecar Containers] in the Kubernetes documentation. [.update,date="2024-01-23"] -=== [.noloc]`Kubernetes` version `1.29` +=== Kubernetes version `1.29` [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html#kubernetes-1.29 +link:eks/latest/userguide/kubernetes-versions.html#kubernetes-1-29[type="documentation"] -Added [.noloc]`Kubernetes` version `1.29` support for new clusters and version upgrades. +Added Kubernetes version `1.29` support for new clusters and version upgrades. [.update,date="2024-01-16"] -=== Full release: Amazon EKS Extended Support for [.noloc]`Kubernetes` versions +=== Full release: Amazon EKS Extended Support for Kubernetes versions [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html +link:eks/latest/userguide/kubernetes-versions.html[type="documentation"] -Extended [.noloc]`Kubernetes` version support allows you to stay at a specific [.noloc]`Kubernetes` version for longer than 14 months. +Extended Kubernetes version support allows you to stay at a specific Kubernetes version for longer than 14 months. [.update,date="2023-12-28"] === Amazon EKS cluster health detection in the {aws} Cloud [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/troubleshooting.html#cluster-health-status +link:eks/latest/userguide/troubleshooting.html#cluster-health-status[type="documentation"] Amazon EKS detects issues with your Amazon EKS clusters and the infrastructure of the cluster prerequisites in _cluster health_. You can view the issues with your EKS clusters in the {aws-management-console} and in the `health` of the cluster in the EKS API. These issues are in addition to the issues that are detected by and displayed by the console. Previously, cluster health was only available for local clusters on {aws} Outposts. @@ -475,7 +803,7 @@ Amazon EKS is now available in the Canada West (Calgary) (`ca-west-1`) {aws} Reg [.update,date="2023-12-20"] === Cluster insights [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/cluster-insights.html +link:eks/latest/userguide/cluster-insights.html[type="documentation"] You can now get recommendations on your cluster based on recurring checks. @@ -483,7 +811,7 @@ You can now get recommendations on your cluster based on recurring checks. [.update,date="2023-12-18"] === You can now grant IAM roles and users access to your cluster using access entries [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/access-entries.html +link:eks/latest/userguide/access-entries.html[type="documentation"] Before the introduction of access entries, you granted IAM roles and users access to your cluster by adding entries to the `aws-auth` `ConfigMap`. Now each cluster has an access mode, and you can switch to using access entries on your schedule. After you switch modes, you can add users by adding access entries in the {aws} CLI, {aws} CloudFormation, and the {aws} SDKs. @@ -491,47 +819,47 @@ Before the introduction of access entries, you granted IAM roles and users acces [.update,date="2023-12-12"] === Amazon EKS platform version update [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/platform-versions.html +link:eks/latest/userguide/platform-versions.html[type="documentation"] This is a new platform version with security fixes and enhancements. This includes new patch versions of Kubernetes `1.28.4`, `1.27.8`, `1.26.11`, and `1.25.16`. [.update,date="2023-11-27"] -=== [.noloc]`Mountpoint` for Amazon S3 CSI driver +=== Mountpoint for Amazon S3 CSI driver [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/s3-csi.html +link:eks/latest/userguide/s3-csi.html[type="documentation"] -You can now install the [.noloc]`Mountpoint` for Amazon S3 CSI driver on Amazon EKS clusters. +You can now install the Mountpoint for Amazon S3 CSI driver on Amazon EKS clusters. [.update,date="2023-11-26"] === Amazon EKS Pod Identities [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/pod-identites.html +link:eks/latest/userguide/pod-identites.html[type="documentation"] -Amazon EKS Pod Identities associate an IAM role with a [.noloc]`Kubernetes` service account. With this feature, you no longer need to provide extended permissions to the node IAM role. This way, [.noloc]`Pods` on that node can call {aws} APIs. Unlike IAM roles for service accounts, EKS Pod Identities are completely inside EKS; you don't need an [.noloc]`OIDC` identity provider. +Amazon EKS Pod Identities associate an IAM role with a Kubernetes service account. With this feature, you no longer need to provide extended permissions to the node IAM role. This way, Pods on that node can call {aws} APIs. Unlike IAM roles for service accounts, EKS Pod Identities are completely inside EKS; you don't need an OIDC identity provider. [.update,date="2023-11-26"] === {aws} managed policy updates - Update to an existing policy [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-updates +link:eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-updates[type="documentation"] Amazon EKS updated an existing {aws} managed policy. [.update,date="2023-11-26"] -=== Turn on [.noloc]`Prometheus` metrics when creating a cluster +=== Turn on Prometheus metrics when creating a cluster [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/prometheus.html +link:eks/latest/userguide/prometheus.html[type="documentation"] -In the {aws-management-console}, you can now turn on [.noloc]`Prometheus` metrics when creating a cluster. You can also view [.noloc]`Prometheus` scraper details in the *Observability* tab. +In the {aws-management-console}, you can now turn on Prometheus metrics when creating a cluster. You can also view Prometheus scraper details in the *Observability* tab. [.update,date="2023-11-17"] === CSI snapshot controller [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/csi-snapshot-controller.html +link:eks/latest/userguide/csi-snapshot-controller.html[type="documentation"] You can now install the CSI snapshot controller for use with compatible CSI drivers, such as the Amazon EBS CSI driver. @@ -539,23 +867,23 @@ You can now install the CSI snapshot controller for use with compatible CSI driv [.update,date="2023-11-14"] === ADOT Operator topic rewrite [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/opentelemetry.html +link:eks/latest/userguide/opentelemetry.html[type="documentation"] -The Amazon EKS add-on support for ADOT Operator section was redundant with the {aws} Distro for [.noloc]`OpenTelemetry` documentation. We migrated remaining essential information to that resource to reduce outdated and inconsistent information. +The Amazon EKS add-on support for ADOT Operator section was redundant with the {aws} Distro for OpenTelemetry documentation. We migrated remaining essential information to that resource to reduce outdated and inconsistent information. [.update,date="2023-11-10"] -=== [.noloc]`CoreDNS` EKS add-on support for Prometheus metrics +=== CoreDNS EKS add-on support for Prometheus metrics [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/managing-coredns.html +link:eks/latest/userguide/managing-coredns.html[type="documentation"] -The `v1.10.1-eksbuild.5`, `v1.9.3-eksbuild.9`, and `v1.8.7-eksbuild.8` versions of the EKS add-on for [.noloc]`CoreDNS` expose the port that [.noloc]`CoreDNS` published metrics to, in the `kube-dns` service. This makes it easier to include the [.noloc]`CoreDNS` metrics in your monitoring systems. +The `v1.10.1-eksbuild.5`, `v1.9.3-eksbuild.9`, and `v1.8.7-eksbuild.8` versions of the EKS add-on for CoreDNS expose the port that CoreDNS published metrics to, in the `kube-dns` service. This makes it easier to include the CoreDNS metrics in your monitoring systems. [.update,date="2023-11-06"] === Amazon EKS CloudWatch Observability Operator add-on [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/cloudwatch.html +link:eks/latest/userguide/cloudwatch.html[type="documentation"] Added Amazon EKS CloudWatch Observability Operator page. @@ -563,7 +891,7 @@ Added Amazon EKS CloudWatch Observability Operator page. [.update,date="2023-10-31"] === Capacity Blocks for self-managed P5 instances in US East (Ohio) [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/capacity-blocks.html +link:eks/latest/userguide/capacity-blocks.html[type="documentation"] In US East (Ohio), you can now use Capacity Blocks for self-managed P5 instances. @@ -571,7 +899,7 @@ In US East (Ohio), you can now use Capacity Blocks for self-managed P5 instances [.update,date="2023-10-24"] === Clusters support modifying subnets and security groups [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/network-reqs.html +link:eks/latest/userguide/network-reqs.html[type="documentation"] You can update the cluster to change which subnets and security groups the cluster uses. You can update from the {aws-management-console}, the latest version of the {aws} CLI, {aws} CloudFormation, and `eksctl` version `v0.164.0-rc.0` or later. You might need to do this to provide subnets with more available IP addresses to successfully upgrade a cluster version. @@ -579,7 +907,7 @@ You can update the cluster to change which subnets and security groups the clust [.update,date="2023-10-23"] === Cluster role and managed node group role supports customer managed {aws} Identity and Access Management policies [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/cluster-iam-role.html +link:eks/latest/userguide/cluster-iam-role.html[type="documentation"] You can use a custom IAM policy on the cluster role, instead of the link:aws-managed-policy/latest/reference/AmazonEKSClusterPolicy.html[AmazonEKSClusterPolicy,type="documentation"] {aws} managed policy. Also, you can use a custom IAM policy on the node role in a managed node group, instead of the link:aws-managed-policy/latest/reference/AmazonEKSWorkerNodePolicy.html[AmazonEKSWorkerNodePolicy,type="documentation"] {aws} managed policy. Do this to create a policy with the least privilege to meet strict compliance requirements. @@ -590,11 +918,11 @@ Fix install link for eksctl after the page was moved. [.update,date="2023-10-04"] -=== Preview release: Amazon EKS Extended Support for [.noloc]`Kubernetes` versions +=== Preview release: Amazon EKS Extended Support for Kubernetes versions [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html +link:eks/latest/userguide/kubernetes-versions.html[type="documentation"] -Extended [.noloc]`Kubernetes` version support allows you to stay at a specific [.noloc]`Kubernetes` version for longer than 14 months. +Extended Kubernetes version support allows you to stay at a specific Kubernetes version for longer than 14 months. [.update,date="2023-09-29"] @@ -603,53 +931,53 @@ Amazon EKS integrations with {aws} App Mesh remain for existing customers of App [.update,date="2023-09-26"] -=== [.noloc]`Kubernetes` version `1.28` +=== Kubernetes version `1.28` [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html#kubernetes-1.28 +link:eks/latest/userguide/kubernetes-versions.html#kubernetes-1-28[type="documentation"] -Added [.noloc]`Kubernetes` version `1.28` support for new clusters and version upgrades. +Added Kubernetes version `1.28` support for new clusters and version upgrades. [.update,date="2023-09-15"] -=== [.noloc]`CoreDNS` Amazon EKS add-on supports modifying PDB +=== CoreDNS Amazon EKS add-on supports modifying PDB [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/managing-coredns.html +link:eks/latest/userguide/managing-coredns.html[type="documentation"] -You can modify the `PodDisruptionBudget` of the EKS add-on for [.noloc]`CoreDNS` in versions `v1.9.3-eksbuild.7` and later and `v1.10.1-eksbuild.4` and later. +You can modify the `PodDisruptionBudget` of the EKS add-on for CoreDNS in versions `v1.9.3-eksbuild.7` and later and `v1.10.1-eksbuild.4` and later. [.update,date="2023-09-15"] -=== Existing clusters support [.noloc]`Kubernetes` network policy enforcement in the [.noloc]`Amazon VPC CNI plugin for Kubernetes` +=== Existing clusters support Kubernetes network policy enforcement in the Amazon VPC CNI plugin for Kubernetes [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/cni-network-policy.html +link:eks/latest/userguide/cni-network-policy.html[type="documentation"] -You can use [.noloc]`Kubernetes` _network policy_ in existing clusters with the [.noloc]`Amazon VPC CNI plugin for Kubernetes`, instead of requiring a third party solution. -You can use [.noloc]`Kubernetes` _network policy_ in existing clusters with the [.noloc]`Amazon VPC CNI plugin for Kubernetes`, instead of requiring a third party solution. +You can use Kubernetes _network policy_ in existing clusters with the Amazon VPC CNI plugin for Kubernetes, instead of requiring a third party solution. +You can use Kubernetes _network policy_ in existing clusters with the Amazon VPC CNI plugin for Kubernetes, instead of requiring a third party solution. [.update,date="2023-09-07"] === Amazon EKS support for shared subnets [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/network-reqs.html#network-requirements-shared +link:eks/latest/userguide/network-reqs.html#network-requirements-shared[type="documentation"] -New link:eks/latest/userguide/network-reqs.html#network-requirements-shared[Shared subnet requirements and considerations,type="documentation"] for making Amazon EKS clusters in shared subnets. +New link:eks/latest/userguide/network-reqs.html#network-requirements-shared[Shared subnet requirements and considerations,type="documentation"] for making Amazon EKS clusters in shared subnets. [.update,date="2023-09-06"] === Updates to What is Amazon EKS? [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/what-is-eks.html +link:eks/latest/userguide/what-is-eks.html[type="documentation"] Added new link:eks/latest/userguide/common-use-cases.html[Common use cases,type="documentation"] and link:eks/latest/userguide/eks-architecture.html[Architecture,type="documentation"] topics. Refreshed other topics. [.update,date="2023-08-29"] -=== [.noloc]`Kubernetes` network policy enforcement in the [.noloc]`Amazon VPC CNI plugin for Kubernetes` +=== Kubernetes network policy enforcement in the Amazon VPC CNI plugin for Kubernetes [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/cni-network-policy.html +link:eks/latest/userguide/cni-network-policy.html[type="documentation"] -You can use [.noloc]`Kubernetes` _network policy_ with the [.noloc]`Amazon VPC CNI plugin for Kubernetes`, instead of requiring a third party solution. -You can use [.noloc]`Kubernetes` _network policy_ with the [.noloc]`Amazon VPC CNI plugin for Kubernetes`, instead of requiring a third party solution. +You can use Kubernetes _network policy_ with the Amazon VPC CNI plugin for Kubernetes, instead of requiring a third party solution. +You can use Kubernetes _network policy_ with the Amazon VPC CNI plugin for Kubernetes, instead of requiring a third party solution. [.update,date="2023-08-01"] @@ -660,15 +988,15 @@ Amazon EKS is now available in the Israel (Tel Aviv) (`il-central-1`) {aws} Regi [.update,date="2023-07-31"] === Configurable ephemeral storage for Fargate [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/fargate-pod-configuration.html#fargate-storage +link:eks/latest/userguide/fargate-pod-configuration.html#fargate-storage[type="documentation"] -You can increase the total amount of ephemeral storage for each [.noloc]`Pod` running on Amazon EKS Fargate. +You can increase the total amount of ephemeral storage for each Pod running on Amazon EKS Fargate. [.update,date="2023-07-26"] === Add-on support for Amazon EFS CSI driver [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/eks-add-ons.html#add-ons-aws-efs-csi-driver +link:eks/latest/userguide/eks-add-ons.html#add-ons-aws-efs-csi-driver[type="documentation"] You can now use the {aws-management-console}, {aws} CLI, and API to manage the Amazon EFS CSI driver. @@ -676,73 +1004,73 @@ You can now use the {aws-management-console}, {aws} CLI, and API to manage the A [.update,date="2023-07-26"] === {aws} managed policy updates - New policy [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-updates +link:eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-updates[type="documentation"] Amazon EKS added a new {aws} managed policy. [.update,date="2023-07-20"] -=== [.noloc]`Kubernetes` version updates for 1.27, 1.26, 1.25, and 1.24 are now available for local clusters on {aws} Outposts +=== Kubernetes version updates for 1.27, 1.26, 1.25, and 1.24 are now available for local clusters on {aws} Outposts [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/eks-outposts-platform-versions.html +link:eks/latest/userguide/eks-outposts-platform-versions.html[type="documentation"] -[.noloc]`Kubernetes` version updates to 1.27.3, 1.26.6, 1.25.11, and 1.24.15 are now available for local clusters on {aws} Outposts +Kubernetes version updates to 1.27.3, 1.26.6, 1.25.11, and 1.24.15 are now available for local clusters on {aws} Outposts [.update,date="2023-07-06"] -=== IP prefixes support for [.noloc]`Windows` nodes +=== IP prefixes support for Windows nodes [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/cni-increase-ip-addresses.html +link:eks/latest/userguide/cni-increase-ip-addresses.html[type="documentation"] -Assigning IP prefixes to your nodes can enable you to host a significantly higher number of [.noloc]`Pods` on your nodes than you can when assigning individual secondary IP addresses to your nodes. +Assigning IP prefixes to your nodes can enable you to host a significantly higher number of Pods on your nodes than you can when assigning individual secondary IP addresses to your nodes. [.update,date="2023-06-30"] === Amazon FSx for OpenZFS CSI driver [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/fsx-openzfs-csi.html +link:eks/latest/userguide/fsx-openzfs-csi.html[type="documentation"] You can now install the Amazon FSx for OpenZFS CSI driver on Amazon EKS clusters. [.update,date="2023-06-19"] -=== [.noloc]`Pods` on Linux nodes in `IPv4` clusters can now communicate with `IPv6` endpoints. +=== Pods on Linux nodes in `IPv4` clusters can now communicate with `IPv6` endpoints. [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/cni-ipv6-egress.html +link:eks/latest/userguide/cni-ipv6-egress.html[type="documentation"] -After assigning an IPv6 address to your node, your [.noloc]`Pods`' `IPv4` address is network address translated to the `IPv6` address of the node that it's running on. +After assigning an IPv6 address to your node, your Pods' `IPv4` address is network address translated to the `IPv6` address of the node that it's running on. [.update,date="2023-05-30"] -=== [.noloc]`Windows` managed node groups in {aws} GovCloud (US) Regions +=== Windows managed node groups in {aws} GovCloud (US) Regions [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/create-managed-node-group.html +link:eks/latest/userguide/create-managed-node-group.html[type="documentation"] -In the {aws} GovCloud (US) Regions, Amazon EKS managed node groups can now run [.noloc]`Windows` containers. +In the {aws} GovCloud (US) Regions, Amazon EKS managed node groups can now run Windows containers. [.update,date="2023-05-24"] -=== [.noloc]`Kubernetes` version `1.27` +=== Kubernetes version `1.27` [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html#kubernetes-1.27 +link:eks/latest/userguide/kubernetes-versions.html#kubernetes-1-27[type="documentation"] -Added [.noloc]`Kubernetes` version `1.27` support for new clusters and version upgrades. +Added Kubernetes version `1.27` support for new clusters and version upgrades. [.update,date="2023-04-11"] -=== [.noloc]`Kubernetes` version `1.26` +=== Kubernetes version `1.26` [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html#kubernetes-1.26 +link:eks/latest/userguide/kubernetes-versions.html[type="documentation"] -Added [.noloc]`Kubernetes` version `1.26` support for new clusters and version upgrades. +Added Kubernetes version `1.26` support for new clusters and version upgrades. [.update,date="2023-03-27"] -=== Domainless [.noloc]`gMSA` +=== Domainless gMSA [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-windows-ami.html#ad-and-gmsa-support +link:eks/latest/userguide/eks-optimized-windows-ami.html#ad-and-gmsa-support[type="documentation"] -You can now use domainless [.noloc]`gMSA` with [.noloc]`Windows` [.noloc]`Pods`. +You can now use domainless gMSA with Windows Pods. [.update,date="2023-03-10"] @@ -753,31 +1081,28 @@ Amazon EKS is now available in the Asia Pacific (Melbourne) (`ap-southeast-4`) { [.update,date="2023-03-03"] === Amazon File Cache CSI driver [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/file-cache-csi.html +link:eks/latest/userguide/file-cache-csi.html[type="documentation"] You can now install the Amazon File Cache CSI driver on Amazon EKS clusters. [.update,date="2023-03-01"] -=== [.noloc]`Kubernetes` version 1.25 is now available for local clusters on {aws} Outposts +=== Kubernetes version 1.25 is now available for local clusters on {aws} Outposts [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/eks-outposts-local-cluster-create.html +link:eks/latest/userguide/eks-outposts-local-cluster-create.html[type="documentation"] -You can now create an Amazon EKS local cluster on an Outpost using [.noloc]`Kubernetes` versions `1.22` – `1.25`. +You can now create an Amazon EKS local cluster on an Outpost using Kubernetes versions `1.22` – `1.25`. [.update,date="2023-02-22"] -=== [.noloc]`Kubernetes` version `1.25` -[.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html#kubernetes-1.25 - -Added [.noloc]`Kubernetes` version `1.25` support for new clusters and version upgrades. +=== Kubernetes version `1.25` +Added Kubernetes version `1.25` support for new clusters and version upgrades. [.update,date="2023-02-07"] === {aws} managed policy updates - Update to an existing policy [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-updates +link:eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-updates[type="documentation"] Amazon EKS updated an existing {aws} managed policy. @@ -788,17 +1113,17 @@ Amazon EKS is now available in the Asia Pacific (Hyderabad) (`ap-south-2`), Euro [.update,date="2023-01-17"] -=== [.noloc]`Kubernetes` versions `1.21` – `1.24` are now available for local clusters on {aws} Outposts. +=== Kubernetes versions `1.21` – `1.24` are now available for local clusters on {aws} Outposts. [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/eks-outposts-local-cluster-create.html +link:eks/latest/userguide/eks-outposts-local-cluster-create.html[type="documentation"] -You can now create an Amazon EKS local cluster on an Outpost using [.noloc]`Kubernetes` versions `1.21` – `1.24`. Previously, only version `1.21` was available. +You can now create an Amazon EKS local cluster on an Outpost using Kubernetes versions `1.21` – `1.24`. Previously, only version `1.21` was available. [.update,date="2022-12-16"] === Amazon EKS now supports {aws} PrivateLink [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/vpc-interface-endpoints.html +link:eks/latest/userguide/vpc-interface-endpoints.html[type="documentation"] You can use an {aws} PrivateLink to create a private connection between your VPC and Amazon EKS. @@ -806,7 +1131,7 @@ You can use an {aws} PrivateLink to create a private connection between your VPC [.update,date="2022-12-15"] === Managed node group Windows support [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html +link:eks/latest/userguide/managed-node-groups.html[type="documentation"] You can now use Windows for Amazon EKS managed node groups. @@ -814,7 +1139,7 @@ You can now use Windows for Amazon EKS managed node groups. [.update,date="2022-11-28"] === Amazon EKS add-ons from independent software vendors are now available in the {aws} Marketplace [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/eks-add-ons.html +link:eks/latest/userguide/eks-add-ons.html[type="documentation"] You can now browse and subscribe to Amazon EKS add-ons from independent software vendors through the {aws} Marketplace. @@ -822,17 +1147,14 @@ You can now browse and subscribe to Amazon EKS add-ons from independent software [.update,date="2022-11-17"] === {aws} managed policy updates - Update to an existing policy [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-updates +link:eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-updates[type="documentation"] Amazon EKS updated an existing {aws} managed policy. [.update,date="2022-11-15"] -=== [.noloc]`Kubernetes` version `1.24` -[.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html#kubernetes-1.24 - -Added [.noloc]`Kubernetes` version `1.24` support for new clusters and version upgrades. +=== Kubernetes version `1.24` +Added Kubernetes version `1.24` support for new clusters and version upgrades. [.update,date="2022-11-03"] @@ -843,7 +1165,7 @@ Amazon EKS is now available in the Middle East (UAE) (`me-central-1`) {aws} Regi [.update,date="2022-10-24"] === {aws} managed policy updates - Update to an existing policy [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-updates +link:eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-updates[type="documentation"] Amazon EKS updated an existing {aws} managed policy. @@ -851,7 +1173,7 @@ Amazon EKS updated an existing {aws} managed policy. [.update,date="2022-10-20"] === {aws} managed policy updates - Update to an existing policy [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-updates +link:eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-updates[type="documentation"] Amazon EKS updated an existing {aws} managed policy. @@ -859,7 +1181,7 @@ Amazon EKS updated an existing {aws} managed policy. [.update,date="2022-09-19"] === Local clusters on {aws} Outposts are now available [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/eks-outposts-local-cluster-create.html +link:eks/latest/userguide/eks-outposts-local-cluster-create.html[type="documentation"] You can now create an Amazon EKS local cluster on an Outpost. @@ -867,15 +1189,15 @@ You can now create an Amazon EKS local cluster on an Outpost. [.update,date="2022-09-08"] === Fargate vCPU based quotas [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/service-quotas.html#service-quotas-eks-fargate +link:eks/latest/userguide/service-quotas.html#service-quotas-eks-fargate[type="documentation"] -Fargate is transitioning from [.noloc]`Pod` based quotas to vCPU based quotas. +Fargate is transitioning from Pod based quotas to vCPU based quotas. [.update,date="2022-08-31"] === {aws} managed policy updates - Update to an existing policy [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-updates +link:eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-updates[type="documentation"] Amazon EKS updated an existing {aws} managed policy. @@ -883,7 +1205,7 @@ Amazon EKS updated an existing {aws} managed policy. [.update,date="2022-08-24"] === {aws} managed policy updates - New policy [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-updates +link:eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-updates[type="documentation"] Amazon EKS added a new {aws} managed policy. @@ -891,15 +1213,15 @@ Amazon EKS added a new {aws} managed policy. [.update,date="2022-08-24"] === Cost monitoring [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/cost-monitoring +link:eks/latest/userguide/cost-monitoring[type="documentation"] -Amazon EKS now supports [.noloc]`Kubecost`, which enables you to monitor costs broken down by [.noloc]`Kubernetes` resources including [.noloc]`Pods`, nodes, namespaces, and labels. +Amazon EKS now supports Kubecost, which enables you to monitor costs broken down by Kubernetes resources including Pods, nodes, namespaces, and labels. [.update,date="2022-08-23"] === {aws} managed policy updates - New policy [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-updates +link:eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-updates[type="documentation"] Amazon EKS added a new {aws} managed policy. @@ -907,7 +1229,7 @@ Amazon EKS added a new {aws} managed policy. [.update,date="2022-08-16"] === Tag resources for billing [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/eks-using-tags.html#tag-resources-for-billing +link:eks/latest/userguide/eks-using-tags.html#tag-resources-for-billing[type="documentation"] Added `aws:eks:cluster-name` generated cost allocation tag support for all clusters. @@ -915,25 +1237,22 @@ Added `aws:eks:cluster-name` generated cost allocation tag support for all clust [.update,date="2022-08-16"] === Fargate profile wildcards [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/fargate-profile.html#fargate-profile-wildcards +link:eks/latest/userguide/fargate-profile.html#fargate-profile-wildcards[type="documentation"] Added support for Fargate profile wildcards in the selector criteria for namespaces, label keys, and label values. [.update,date="2022-08-11"] -=== [.noloc]`Kubernetes` version `1.23` -[.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html#kubernetes-1.23 - -Added [.noloc]`Kubernetes` version `1.23` support for new clusters and version upgrades. +=== Kubernetes version `1.23` +Added Kubernetes version `1.23` support for new clusters and version upgrades. [.update,date="2022-05-03"] -=== View [.noloc]`Kubernetes` resources in the {aws-management-console} +=== View Kubernetes resources in the {aws-management-console} [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/view-kubernetes-resources.html +link:eks/latest/userguide/view-kubernetes-resources.html[type="documentation"] -You can now view information about the [.noloc]`Kubernetes` resources deployed to your cluster using the {aws-management-console}. +You can now view information about the Kubernetes resources deployed to your cluster using the {aws-management-console}. [.update,date="2022-05-02"] @@ -944,36 +1263,36 @@ Amazon EKS is now available in the Asia Pacific (Jakarta) (`ap-southeast-3`) {aw [.update,date="2022-04-21"] === Observability page and ADOT add-on support [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/eks-observe.html +link:eks/latest/userguide/eks-observe.html[type="documentation"] -Added Observability page and {aws} Distro for [.noloc]`OpenTelemetry` (ADOT). +Added Observability page and {aws} Distro for OpenTelemetry (ADOT). [.update,date="2022-04-04"] === {aws} managed policy updates - New policy [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-updates +link:eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-updates[type="documentation"] Amazon EKS added a new {aws} managed policy. [.update,date="2022-04-04"] -=== [.noloc]`Kubernetes` version `1.22` -Added [.noloc]`Kubernetes` version `1.22` support for new clusters and version upgrades. +=== Kubernetes version `1.22` +Added Kubernetes version `1.22` support for new clusters and version upgrades. [.update,date="2022-04-01"] -=== Added Fargate [.noloc]`Pod` patching details +=== Added Fargate Pod patching details [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/fargate-pod-patching.html +link:eks/latest/userguide/fargate-pod-patching.html[type="documentation"] -When upgrading Fargate [.noloc]`Pods`, Amazon EKS first tries to evict [.noloc]`Pods` based on your [.noloc]`Pod` disruption budgets. You can create event rules to react to failed evictions before the [.noloc]`Pods` are deleted. +When upgrading Fargate Pods, Amazon EKS first tries to evict Pods based on your Pod disruption budgets. You can create event rules to react to failed evictions before the Pods are deleted. [.update,date="2022-03-31"] === Full release: Add-on support for Amazon EBS CSI driver [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/ebs-csi.html +link:eks/latest/userguide/ebs-csi.html[type="documentation"] You can now use the {aws-management-console}, {aws} CLI, and API to manage the Amazon EBS CSI driver. @@ -981,7 +1300,7 @@ You can now use the {aws-management-console}, {aws} CLI, and API to manage the A [.update,date="2022-03-22"] === {aws} Outposts content update [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/outposts.html +link:eks/latest/userguide/outposts.html[type="documentation"] Instructions to deploy an Amazon EKS cluster on {aws} Outposts. @@ -989,7 +1308,7 @@ Instructions to deploy an Amazon EKS cluster on {aws} Outposts. [.update,date="2022-03-21"] === {aws} managed policy updates - Update to an existing policy [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-updates +link:eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-updates[type="documentation"] Amazon EKS updated an existing {aws} managed policy. @@ -997,31 +1316,31 @@ Amazon EKS updated an existing {aws} managed policy. [.update,date="2022-03-14"] === Windows `containerd` support [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-windows-ami.html.html +link:eks/latest/userguide/eks-optimized-windows-ami.html.html[type="documentation"] -You can now select the `containerd` runtime for [.noloc]`Windows` nodes. +You can now select the `containerd` runtime for Windows nodes. [.update,date="2022-02-25"] === Added Amazon EKS Connector considerations to security documentation [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/connector-considerations.html +link:eks/latest/userguide/connector-considerations.html[type="documentation"] Describes the shared responsibility model as it relates to connected clusters. [.update,date="2022-01-06"] -=== Assign `IPv6` addresses to your [.noloc]`Pods` and services +=== Assign `IPv6` addresses to your Pods and services [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/cni-ipv6.html +link:eks/latest/userguide/cni-ipv6.html[type="documentation"] -You can now create a `1.21` or later cluster that assigns `IPv6` addresses to your [.noloc]`Pods` and services. +You can now create a `1.21` or later cluster that assigns `IPv6` addresses to your Pods and services. [.update,date="2021-12-13"] === {aws} managed policy updates - Update to an existing policy [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-updates +link:eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-updates[type="documentation"] Amazon EKS updated an existing {aws} managed policy. @@ -1029,7 +1348,7 @@ Amazon EKS updated an existing {aws} managed policy. [.update,date="2021-12-09"] === Preview release: Add-on support for Amazon EBS CSI driver [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/ebs-csi.html +link:eks/latest/userguide/ebs-csi.html[type="documentation"] You can now preview using the {aws-management-console}, {aws} CLI, and API to manage the Amazon EBS CSI driver. @@ -1037,39 +1356,39 @@ You can now preview using the {aws-management-console}, {aws} CLI, and API to ma [.update,date="2021-11-29"] === Karpenter autoscaler support [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/autoscaling.html#karpenter +link:eks/latest/userguide/autoscaling.html#karpenter[type="documentation"] You can now use the Karpenter open-source project to autoscale your nodes. [.update,date="2021-11-10"] -=== Fluent Bit [.noloc]`Kubernetes` filter support in Fargate logging +=== Fluent Bit Kubernetes filter support in Fargate logging [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/fargate-logging.html#fargate-logging-kubernetes-filter +link:eks/latest/userguide/fargate-logging.html#fargate-logging-kubernetes-filter[type="documentation"] -You can now use the Fluent Bit [.noloc]`Kubernetes` filter with Fargate logging. +You can now use the Fluent Bit Kubernetes filter with Fargate logging. [.update,date="2021-11-09"] -=== [.noloc]`Windows` support available in the control plane +=== Windows support available in the control plane [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html +link:eks/latest/userguide/windows-support.html[type="documentation"] -[.noloc]`Windows` support is now available in your control plane. You no longer need to enable it in your data plane. +Windows support is now available in your control plane. You no longer need to enable it in your data plane. [.update,date="2021-10-28"] -=== [.noloc]`Bottlerocket` added as an AMI type for managed node groups +=== Bottlerocket added as an AMI type for managed node groups [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami-bottlerocket.html +link:eks/latest/userguide/eks-optimized-ami-bottlerocket.html[type="documentation"] -Previously, [.noloc]`Bottlerocket` was only available as a self-managed node option. Now it can be configured as a managed node group, reducing the effort that's required to meet node compliance requirements. +Previously, Bottlerocket was only available as a self-managed node option. Now it can be configured as a managed node group, reducing the effort that's required to meet node compliance requirements. [.update,date="2021-10-25"] === DL1 driver support [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/eks-ami-build-scripts.html +link:eks/latest/userguide/eks-ami-build-scripts.html[type="documentation"] Custom Amazon Linux AMIs now support deep learning workloads for Amazon Linux 2. This enablement allows a generic on-premises or cloud baseline configuration. @@ -1077,7 +1396,7 @@ Custom Amazon Linux AMIs now support deep learning workloads for Amazon Linux 2. [.update,date="2021-09-13"] === VT1 video support [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/eks-ami-build-scripts.html +link:eks/latest/userguide/eks-ami-build-scripts.html[type="documentation"] Custom Amazon Linux AMIs now support VT1 for some distributions. This enablement advertises Xilinx U30 devices on your Amazon EKS cluster. @@ -1085,114 +1404,114 @@ Custom Amazon Linux AMIs now support VT1 for some distributions. This enablement [.update,date="2021-09-08"] === Amazon EKS Anywhere is now available [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/eks-deployment-options.html +link:eks/latest/userguide/eks-deployment-options.html[type="documentation"] -Amazon EKS Anywhere is a new deployment option for Amazon EKS that you can use to create and operate [.noloc]`Kubernetes` clusters on-premises. +Amazon EKS Anywhere is a new deployment option for Amazon EKS that you can use to create and operate Kubernetes clusters on-premises. [.update,date="2021-09-08"] === Amazon EKS Connector is now available [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/eks-connector.html +link:eks/latest/userguide/eks-connector.html[type="documentation"] -You can use Amazon EKS Connector to register and connect any conformant [.noloc]`Kubernetes` cluster to {aws} and visualize it in the Amazon EKS console. +You can use Amazon EKS Connector to register and connect any conformant Kubernetes cluster to {aws} and visualize it in the Amazon EKS console. [.update,date="2021-09-02"] === Amazon FSx for NetApp ONTAP CSI driver [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/fsx-ontap.html +link:eks/latest/userguide/fsx-ontap.html[type="documentation"] Added topic that summarizes the Amazon FSx for NetApp ONTAP CSI driver and gives links to other references. [.update,date="2021-08-30"] -=== Managed node groups now auto-calculates the Amazon EKS recommended maximum [.noloc]`Pods` for nodes +=== Managed node groups now auto-calculates the Amazon EKS recommended maximum Pods for nodes [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/cni-increase-ip-addresses.html +link:eks/latest/userguide/cni-increase-ip-addresses.html[type="documentation"] -Managed node groups now auto-calculate the Amazon EKS maximum [.noloc]`Pods` for nodes that you deploy without a launch template, or with a launch template that you haven't specified an AMI ID in. +Managed node groups now auto-calculate the Amazon EKS maximum Pods for nodes that you deploy without a launch template, or with a launch template that you haven't specified an AMI ID in. [.update,date="2021-08-20"] === Remove Amazon EKS management of add-on settings without removing the Amazon EKS add-on software [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/managing-vpc-cni.html#removing-vpc-cni-eks-add-on +link:eks/latest/userguide/managing-vpc-cni.html#removing-vpc-cni-eks-add-on[type="documentation"] You can now remove an Amazon EKS add-on without removing the add-on software from your cluster. [.update,date="2021-08-02"] -=== Create multi-homed [.noloc]`Pods` using Multus +=== Create multi-homed Pods using Multus [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/pod-multiple-network-interfaces.html +link:eks/latest/userguide/pod-multiple-network-interfaces.html[type="documentation"] -You can now add multiple network interfaces to a [.noloc]`Pod` using Multus. +You can now add multiple network interfaces to a Pod using Multus. [.update,date="2021-07-27"] -=== Add more IP addresses to your [.noloc]`Linux` Amazon EC2 nodes +=== Add more IP addresses to your Linux Amazon EC2 nodes [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/cni-increase-ip-addresses.html +link:eks/latest/userguide/cni-increase-ip-addresses.html[type="documentation"] -You can now add significantly more IP addresses to your [.noloc]`Linux` Amazon EC2 nodes. This means that you can run a higher density of [.noloc]`Pods` on each node. -You can now add significantly more IP addresses to your [.noloc]`Linux` Amazon EC2 nodes. This means that you can run a higher density of [.noloc]`Pods` on each node. +You can now add significantly more IP addresses to your Linux Amazon EC2 nodes. This means that you can run a higher density of Pods on each node. +You can now add significantly more IP addresses to your Linux Amazon EC2 nodes. This means that you can run a higher density of Pods on each node. [.update,date="2021-07-19"] -=== [.noloc]`Kubernetes` version `1.21` -Added [.noloc]`Kubernetes` version `1.21` support. +=== Kubernetes version `1.21` +Added Kubernetes version `1.21` support. [.update,date="2021-07-19"] === `containerd` runtime bootstrap [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html#containerd-bootstrap +link:eks/latest/userguide/eks-optimized-ami.html[type="documentation"] -The Amazon EKS optimized accelerated Amazon Linux Amazon Machine Image (AMI) now contains a bootstrap flag that you can use to enable the `containerd` runtime in Amazon EKS optimized and [.noloc]`Bottlerocket` AMIs. This flag is available in all supported [.noloc]`Kubernetes` versions of the AMI. +The Amazon EKS optimized accelerated Amazon Linux Amazon Machine Image (AMI) now contains a bootstrap flag that you can use to enable the `containerd` runtime in Amazon EKS optimized and Bottlerocket AMIs. This flag is available in all supported Kubernetes versions of the AMI. [.update,date="2021-06-17"] === Added managed policies topic [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html +link:eks/latest/userguide/security-iam-awsmanpol.html[type="documentation"] A list of all Amazon EKS IAM managed policies and changes that were made to them since June 17, 2021. [.update,date="2021-06-01"] -=== Use security groups for [.noloc]`Pods` with Fargate +=== Use security groups for Pods with Fargate [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html +link:eks/latest/userguide/security-groups-for-pods.html[type="documentation"] -You can now use security groups for [.noloc]`Pods` with Fargate, in addition to using them with Amazon EC2 nodes. +You can now use security groups for Pods with Fargate, in addition to using them with Amazon EC2 nodes. [.update,date="2021-05-19"] -=== Added [.noloc]`CoreDNS` and `kube-proxy` Amazon EKS add-ons +=== Added CoreDNS and `kube-proxy` Amazon EKS add-ons [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/eks-add-ons.html +link:eks/latest/userguide/eks-add-ons.html[type="documentation"] -Amazon EKS can now help you manage the [.noloc]`CoreDNS` and `kube-proxy` Amazon EKS add-ons for your cluster. +Amazon EKS can now help you manage the CoreDNS and `kube-proxy` Amazon EKS add-ons for your cluster. [.update,date="2021-05-18"] -=== [.noloc]`Kubernetes` version `1.20` -Added [.noloc]`Kubernetes` version `1.20` support for new clusters and version upgrades. +=== Kubernetes version `1.20` +Added Kubernetes version `1.20` support for new clusters and version upgrades. [.update,date="2021-05-14"] -=== [.noloc]`{aws} Load Balancer Controller` `2.2.0` released +=== {aws} Load Balancer Controller `2.2.0` released [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/aws-load-balancer-controller.html +link:eks/latest/userguide/aws-load-balancer-controller.html[type="documentation"] -You can now use the [.noloc]`{aws} Load Balancer Controller` to create Elastic Load Balancers using instance or IP targets. +You can now use the {aws} Load Balancer Controller to create Elastic Load Balancers using instance or IP targets. [.update,date="2021-05-11"] === Node taints for managed node groups [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/node-taints-managed-node-groups.html +link:eks/latest/userguide/node-taints-managed-node-groups.html[type="documentation"] Amazon EKS now supports adding note taints to managed node groups. @@ -1200,20 +1519,20 @@ Amazon EKS now supports adding note taints to managed node groups. [.update,date="2021-02-26"] === Secrets encryption for existing clusters [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html#enable-kms +link:eks/latest/userguide/update-cluster.html#enable-kms[type="documentation"] Amazon EKS now supports adding https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/[secrets encryption] to existing clusters. [.update,date="2021-02-16"] -=== [.noloc]`Kubernetes` version `1.19` -Added [.noloc]`Kubernetes` version `1.19` support for new clusters and version upgrades. +=== Kubernetes version `1.19` +Added Kubernetes version `1.19` support for new clusters and version upgrades. [.update,date="2021-02-12"] -=== Amazon EKS now supports [.noloc]`OpenID Connect` (OIDC) identity providers as a method to authenticate users to a version `1.16` or later cluster. +=== Amazon EKS now supports OpenID Connect (OIDC) identity providers as a method to authenticate users to a version `1.16` or later cluster. [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/authenticate-oidc-identity-provider.html +link:eks/latest/userguide/authenticate-oidc-identity-provider.html[type="documentation"] OIDC identity providers can be used with, or as an alternative to {aws} Identity and Access Management (IAM). @@ -1221,7 +1540,7 @@ OIDC identity providers can be used with, or as an alternative to {aws} Identity [.update,date="2020-12-01"] === Amazon EKS can now manage specific add-ons for your cluster [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/eks-add-ons.html +link:eks/latest/userguide/eks-add-ons.html[type="documentation"] You can manage add-ons yourself, or allow Amazon EKS to control the launch and version of an add-on through the Amazon EKS API. @@ -1229,7 +1548,7 @@ You can manage add-ons yourself, or allow Amazon EKS to control the launch and v [.update,date="2020-12-01"] === Deploy Spot Instance types in a managed node group [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html#managed-node-group-capacity-types +link:eks/latest/userguide/managed-node-groups.html#managed-node-group-capacity-types[type="documentation"] You can now deploy multiple Spot or On-Demand Instance types to a managed node group. @@ -1237,52 +1556,52 @@ You can now deploy multiple Spot or On-Demand Instance types to a managed node g [.update,date="2020-12-01"] === View node and workload resources in the {aws-management-console} [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/view-kubernetes-resources.html +link:eks/latest/userguide/view-kubernetes-resources.html[type="documentation"] -You can now view details about your managed, self-managed, and Fargate nodes and your deployed [.noloc]`Kubernetes` workloads in the {aws-management-console}. +You can now view details about your managed, self-managed, and Fargate nodes and your deployed Kubernetes workloads in the {aws-management-console}. [.update,date="2020-10-23"] === NLB IP target support [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/networkg-load-balancing.html#network-load-balancer +link:eks/latest/userguide/networkg-load-balancing.html#network-load-balancer[type="documentation"] -You can now deploy a Network Load Balancer with IP targets. This means that you can use an NLB to load balance network traffic to Fargate [.noloc]`Pods` and directly to [.noloc]`Pods` that are running on Amazon EC2 nodes. +You can now deploy a Network Load Balancer with IP targets. This means that you can use an NLB to load balance network traffic to Fargate Pods and directly to Pods that are running on Amazon EC2 nodes. [.update,date="2020-10-23"] === Share an ALB across multiple Ingresses [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/alb-ingress.html +link:eks/latest/userguide/alb-ingress.html[type="documentation"] -You can now share an {aws} Application Load Balancer (ALB) across multiple [.noloc]`Kubernetes` Ingresses. In the past, you had to deploy a separate ALB for each Ingress. +You can now share an {aws} Application Load Balancer (ALB) across multiple Kubernetes Ingresses. In the past, you had to deploy a separate ALB for each Ingress. [.update,date="2020-10-13"] -=== [.noloc]`Kubernetes` version `1.18` -Added [.noloc]`Kubernetes` version `1.18` support for new clusters and version upgrades. +=== Kubernetes version `1.18` +Added Kubernetes version `1.18` support for new clusters and version upgrades. [.update,date="2020-09-29"] -=== Specify a custom CIDR block for [.noloc]`Kubernetes` service IP address assignment. +=== Specify a custom CIDR block for Kubernetes service IP address assignment. [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/create-cluster.html +link:eks/latest/userguide/create-cluster.html[type="documentation"] -You can now specify a custom CIDR block that [.noloc]`Kubernetes` assigns service IP addresses from. +You can now specify a custom CIDR block that Kubernetes assigns service IP addresses from. [.update,date="2020-09-09"] -=== Assign security groups to individual [.noloc]`Pods` +=== Assign security groups to individual Pods [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html +link:eks/latest/userguide/security-groups-for-pods.html[type="documentation"] -You can now associate different security groups to some of the individual [.noloc]`Pods` that are running on many Amazon EC2 instance types. +You can now associate different security groups to some of the individual Pods that are running on many Amazon EC2 instance types. [.update,date="2020-08-31"] -=== Deploy [.noloc]`Bottlerocket` on your nodes +=== Deploy Bottlerocket on your nodes [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/launch-node-bottlerocket.html +link:eks/latest/userguide/launch-node-bottlerocket.html[type="documentation"] You can now deploy nodes that are running link:bottlerocket/[Bottlerocket,type="marketing"]. @@ -1290,7 +1609,7 @@ You can now deploy nodes that are running link:bottlerocket/[Bottlerocket,type=" [.update,date="2020-08-17"] === Managed node group launch templates and custom AMI [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html +link:eks/latest/userguide/launch-templates.html[type="documentation"] You can now deploy a managed node group that uses an Amazon EC2 launch template. The launch template can specify a custom AMI, if you choose. @@ -1298,7 +1617,7 @@ You can now deploy a managed node group that uses an Amazon EC2 launch template. [.update,date="2020-08-17"] === The ability to launch Arm nodes is generally available [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html#arm-ami +link:eks/latest/userguide/eks-optimized-ami.html#arm-ami[type="documentation"] You can now launch Arm nodes in managed and self-managed node groups. @@ -1306,7 +1625,7 @@ You can now launch Arm nodes in managed and self-managed node groups. [.update,date="2020-08-17"] === EFS support for {aws} Fargate [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/efs-csi.html +link:eks/latest/userguide/efs-csi.html[type="documentation"] You can now use Amazon EFS with {aws} Fargate. @@ -1314,9 +1633,9 @@ You can now use Amazon EFS with {aws} Fargate. [.update,date="2020-08-12"] === Amazon EKS platform version update [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/platform-versions.html +link:eks/latest/userguide/platform-versions.html[type="documentation"] -This is a new platform version with security fixes and enhancements. This includes UDP support for services of type `LoadBalancer` when using Network Load Balancers with [.noloc]`Kubernetes` version `1.15` or later. For more information, see the https://github.com/kubernetes/kubernetes/pull/92109[Allow UDP for {aws} Network Load Balancer] issue on [.noloc]`GitHub`. +This is a new platform version with security fixes and enhancements. This includes UDP support for services of type `LoadBalancer` when using Network Load Balancers with Kubernetes version `1.15` or later. For more information, see the https://github.com/kubernetes/kubernetes/pull/92109[Allow UDP for {aws} Network Load Balancer] issue on GitHub. [.update,date="2020-08-06"] @@ -1327,28 +1646,28 @@ Amazon EKS is now available in the Africa (Cape Town) (`af-south-1`) and Europe [.update,date="2020-08-03"] === Fargate usage metrics [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/monitoring-fargate-usage.html +link:eks/latest/userguide/monitoring-fargate-usage.html[type="documentation"] {aws} Fargate provides CloudWatch usage metrics that provide visibility into your account's usage of Fargate On-Demand resources. [.update,date="2020-07-10"] -=== [.noloc]`Kubernetes` version `1.17` -Added [.noloc]`Kubernetes` version `1.17` support for new clusters and version upgrades. +=== Kubernetes version `1.17` +Added Kubernetes version `1.17` support for new clusters and version upgrades. [.update,date="2020-06-18"] -=== Create and manage App Mesh resources from within [.noloc]`Kubernetes` with the App Mesh controller for [.noloc]`Kubernetes` +=== Create and manage App Mesh resources from within Kubernetes with the App Mesh controller for Kubernetes [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/mesh-k8s-integration.html +link:eks/latest/userguide/mesh-k8s-integration.html[type="documentation"] -You can create and manage App Mesh resources from within [.noloc]`Kubernetes`. The controller also automatically injects the Envoy proxy and init containers into [.noloc]`Pods` that you deploy. +You can create and manage App Mesh resources from within Kubernetes. The controller also automatically injects the Envoy proxy and init containers into Pods that you deploy. [.update,date="2020-06-04"] === Amazon EKS now supports Amazon EC2 Inf1 nodes [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/inferentia-support.html +link:eks/latest/userguide/inferentia-support.html[type="documentation"] You can add Amazon EC2 Inf1 nodes to your cluster. @@ -1359,29 +1678,29 @@ Amazon EKS is now available in the {aws} GovCloud (US-East) (`us-gov-east-1`) an [.update,date="2020-05-12"] -=== [.noloc]`Kubernetes` `1.12` is no longer supported on Amazon EKS +=== Kubernetes `1.12` is no longer supported on Amazon EKS [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html +link:eks/latest/userguide/update-cluster.html[type="documentation"] -[.noloc]`Kubernetes` version `1.12` is no longer supported on Amazon EKS. Update any `1.12` clusters to version `1.13` or later to avoid service interruption. +Kubernetes version `1.12` is no longer supported on Amazon EKS. Update any `1.12` clusters to version `1.13` or later to avoid service interruption. [.update,date="2020-04-30"] -=== [.noloc]`Kubernetes` version `1.16` -Added [.noloc]`Kubernetes` version `1.16` support for new clusters and version upgrades. +=== Kubernetes version `1.16` +Added Kubernetes version `1.16` support for new clusters and version upgrades. [.update,date="2020-04-16"] -=== Added the *AWSServiceRoleForAmazonEKS* service-linked role +=== Added the *AWSServiceRoleForAmazonEKS* service-linked role [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/using-service-linked-roles-eks.html +link:eks/latest/userguide/using-service-linked-roles-eks.html[type="documentation"] Added the *AWSServiceRoleForAmazonEKS* service-linked role. [.update,date="2020-03-10"] -=== [.noloc]`Kubernetes` version `1.15` -Added [.noloc]`Kubernetes` version `1.15` support for new clusters and version upgrades. +=== Kubernetes version `1.15` +Added Kubernetes version `1.15` support for new clusters and version upgrades. [.update,date="2020-02-26"] @@ -1392,31 +1711,31 @@ Amazon EKS is now available in the Beijing (`cn-north-1`) and Ningxia (`cn-north [.update,date="2019-12-23"] === FSx for Lustre CSI driver [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/fsx-csi.html +link:eks/latest/userguide/fsx-csi.html[type="documentation"] -Added topic for installing the FSx for Lustre CSI driver on [.noloc]`Kubernetes` `1.14` Amazon EKS clusters. +Added topic for installing the FSx for Lustre CSI driver on Kubernetes `1.14` Amazon EKS clusters. [.update,date="2019-12-20"] === Restrict network access to the public access endpoint of a cluster [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/cluster-endpoint.html +link:eks/latest/userguide/cluster-endpoint.html[type="documentation"] -With this update, you can use Amazon EKS to restrict the CIDR ranges that can communicate to the public access endpoint of the [.noloc]`Kubernetes` API server. +With this update, you can use Amazon EKS to restrict the CIDR ranges that can communicate to the public access endpoint of the Kubernetes API server. [.update,date="2019-12-13"] === Resolve the private access endpoint address for a cluster from outside of a VPC [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/cluster-endpoint.html +link:eks/latest/userguide/cluster-endpoint.html[type="documentation"] -With this update, you can use Amazon EKS to resolve the private access endpoint of the [.noloc]`Kubernetes` API server from outside of a VPC. +With this update, you can use Amazon EKS to resolve the private access endpoint of the Kubernetes API server from outside of a VPC. [.update,date="2019-12-04"] === (Beta) Amazon EC2 A1 Amazon EC2 instance nodes [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/arm-support.html +link:eks/latest/userguide/arm-support.html[type="documentation"] Launch link:ec2/instance-types/a1/[Amazon EC2 A1,type="marketing"] Amazon EC2 instance nodes that register with your Amazon EKS cluster. @@ -1424,7 +1743,7 @@ Launch link:ec2/instance-types/a1/[Amazon EC2 A1,type="marketing"] Amazon EC2 in [.update,date="2019-12-03"] === Creating a cluster on {aws} Outposts [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/eks-on-outposts.html +link:eks/latest/userguide/eks-on-outposts.html[type="documentation"] Amazon EKS now supports creating clusters on {aws} Outposts. @@ -1432,9 +1751,9 @@ Amazon EKS now supports creating clusters on {aws} Outposts. [.update,date="2019-12-03"] === {aws} Fargate on Amazon EKS [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/fargate.html +link:eks/latest/userguide/fargate.html[type="documentation"] -Amazon EKS [.noloc]`Kubernetes` clusters now support running [.noloc]`Pods` on Fargate. +Amazon EKS Kubernetes clusters now support running Pods on Fargate. [.update,date="2019-11-21"] @@ -1445,25 +1764,25 @@ Amazon EKS is now available in the Canada (Central) (`ca-central-1`) {aws} Regio [.update,date="2019-11-18"] === Managed node groups [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html +link:eks/latest/userguide/managed-node-groups.html[type="documentation"] -Amazon EKS managed node groups automate the provisioning and lifecycle management of nodes (Amazon EC2 instances) for Amazon EKS [.noloc]`Kubernetes` clusters. +Amazon EKS managed node groups automate the provisioning and lifecycle management of nodes (Amazon EC2 instances) for Amazon EKS Kubernetes clusters. [.update,date="2019-11-06"] === Amazon EKS platform version update [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/platform-versions.html +link:eks/latest/userguide/platform-versions.html[type="documentation"] New platform versions to address https://groups.google.com/forum/#!msg/kubernetes-security-announce/jk8polzSUxs/dfq6a-MnCQAJ[CVE-2019-11253]. [.update,date="2019-11-04"] -=== [.noloc]`Kubernetes` `1.11` is no longer supported on Amazon EKS +=== Kubernetes `1.11` is no longer supported on Amazon EKS [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html +link:eks/latest/userguide/update-cluster.html[type="documentation"] -[.noloc]`Kubernetes` version `1.11` is no longer supported on Amazon EKS. Please update any `1.11` clusters to version `1.12` or higher to avoid service interruption. +Kubernetes version `1.11` is no longer supported on Amazon EKS. Please update any `1.11` clusters to version `1.12` or higher to avoid service interruption. [.update,date="2019-10-16"] @@ -1472,41 +1791,41 @@ Amazon EKS is now available in the South America (São Paulo) (`sa-east-1`) {aws [.update,date="2019-10-07"] -=== [.noloc]`Windows` support +=== Windows support [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html +link:eks/latest/userguide/windows-support.html[type="documentation"] -Amazon EKS clusters running [.noloc]`Kubernetes` version `1.14` now support [.noloc]`Windows` workloads. +Amazon EKS clusters running Kubernetes version `1.14` now support Windows workloads. [.update,date="2019-09-30"] === Autoscaling [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/autoscaling.html +link:eks/latest/userguide/autoscaling.html[type="documentation"] -Added a chapter to cover some of the different types of [.noloc]`Kubernetes` autoscaling that are supported on Amazon EKS clusters. +Added a chapter to cover some of the different types of Kubernetes autoscaling that are supported on Amazon EKS clusters. [.update,date="2019-09-28"] -=== [.noloc]`Kubernetes` Dashboard update +=== Kubernetes Dashboard update [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/dashboard-tutorial.html +link:eks/latest/userguide/dashboard-tutorial.html[type="documentation"] -Updated topic for installing the [.noloc]`Kubernetes` Dashboard on Amazon EKS clusters to use the beta `2.0` version. +Updated topic for installing the Kubernetes Dashboard on Amazon EKS clusters to use the beta `2.0` version. [.update,date="2019-09-19"] === Amazon EFS CSI driver [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/efs-csi.html +link:eks/latest/userguide/efs-csi.html[type="documentation"] -Added topic for installing the Amazon EFS CSI driver on [.noloc]`Kubernetes` `1.14` Amazon EKS clusters. +Added topic for installing the Amazon EFS CSI driver on Kubernetes `1.14` Amazon EKS clusters. [.update,date="2019-09-18"] === Amazon EC2 Systems Manager parameter for Amazon EKS optimized AMI ID [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/retrieve-ami-id.html +link:eks/latest/userguide/retrieve-ami-id.html[type="documentation"] Added topic for retrieving the Amazon EKS optimized AMI ID using an Amazon EC2 Systems Manager parameter. The parameter eliminates the need for you to look up AMI IDs. @@ -1514,7 +1833,7 @@ Added topic for retrieving the Amazon EKS optimized AMI ID using an Amazon EC2 S [.update,date="2019-09-16"] === Amazon EKS resource tagging [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/eks-using-tags.html +link:eks/latest/userguide/eks-using-tags.html[type="documentation"] You can manage the tagging of your Amazon EKS clusters. @@ -1522,38 +1841,38 @@ You can manage the tagging of your Amazon EKS clusters. [.update,date="2019-09-09"] === Amazon EBS CSI driver [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/ebs-csi.html +link:eks/latest/userguide/ebs-csi.html[type="documentation"] -Added topic for installing the Amazon EBS CSI driver on [.noloc]`Kubernetes` `1.14` Amazon EKS clusters. +Added topic for installing the Amazon EBS CSI driver on Kubernetes `1.14` Amazon EKS clusters. [.update,date="2019-09-06"] === New Amazon EKS optimized AMI patched for `CVE-2019-9512` and `CVE-2019-9514` [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html +link:eks/latest/userguide/eks-optimized-ami.html[type="documentation"] Amazon EKS has updated the Amazon EKS optimized AMI to address https://groups.google.com/forum/#!topic/kubernetes-security-announce/wlHLHit1BqA[CVE-2019-9512 and CVE-2019-9514]. [.update,date="2019-09-04"] -=== Announcing deprecation of [.noloc]`Kubernetes` `1.11` in Amazon EKS +=== Announcing deprecation of Kubernetes `1.11` in Amazon EKS [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html +link:eks/latest/userguide/kubernetes-versions.html[type="documentation"] -Amazon EKS discontinued support for [.noloc]`Kubernetes` version `1.11` on November 4, 2019. +Amazon EKS discontinued support for Kubernetes version `1.11` on November 4, 2019. [.update,date="2019-09-03"] === IAM roles for service accounts [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html +link:eks/latest/userguide/iam-roles-for-service-accounts.html[type="documentation"] -With IAM roles for service accounts on Amazon EKS clusters, you can associate an IAM role with a [.noloc]`Kubernetes` service account. With this feature, you no longer need to provide extended permissions to the node IAM role. This way, [.noloc]`Pods` on that node can call {aws} APIs. +With IAM roles for service accounts on Amazon EKS clusters, you can associate an IAM role with a Kubernetes service account. With this feature, you no longer need to provide extended permissions to the node IAM role. This way, Pods on that node can call {aws} APIs. [.update,date="2019-09-03"] -=== [.noloc]`Kubernetes` version `1.14` -Added [.noloc]`Kubernetes` version `1.14` support for new clusters and version upgrades. +=== Kubernetes version `1.14` +Added Kubernetes version `1.14` support for new clusters and version upgrades. [.update,date="2019-08-29"] @@ -1564,7 +1883,7 @@ Amazon EKS is now available in the Middle East (Bahrain) (`me-south-1`) {aws} Re [.update,date="2019-08-28"] === Amazon EKS platform version update [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/platform-versions.html +link:eks/latest/userguide/platform-versions.html[type="documentation"] New platform versions to address https://groups.google.com/forum/#!topic/kubernetes-security-announce/wlHLHit1BqA[CVE-2019-9512 and CVE-2019-9514]. @@ -1572,7 +1891,7 @@ New platform versions to address https://groups.google.com/forum/#!topic/kuberne [.update,date="2019-08-05"] === Amazon EKS platform version update [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/platform-versions.html +link:eks/latest/userguide/platform-versions.html[type="documentation"] New platform versions to address https://groups.google.com/forum/#!topic/kubernetes-security-announce/vUtEcSEY6SM[CVE-2019-11247 and CVE-2019-11249]. @@ -1583,38 +1902,38 @@ Amazon EKS is now available in the Asia Pacific (Hong Kong) (`ap-east-1`) {aws} [.update,date="2019-07-30"] -=== [.noloc]`Kubernetes` `1.10` no longer supported on Amazon EKS +=== Kubernetes `1.10` no longer supported on Amazon EKS [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html +link:eks/latest/userguide/update-cluster.html[type="documentation"] -[.noloc]`Kubernetes` version `1.10` is no longer supported on Amazon EKS. Update any `1.10` clusters to version `1.11` or higher to avoid service interruption. +Kubernetes version `1.10` is no longer supported on Amazon EKS. Update any `1.10` clusters to version `1.11` or higher to avoid service interruption. [.update,date="2019-07-11"] === Added topic on ALB ingress controller [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/alb-ingress.html +link:eks/latest/userguide/alb-ingress.html[type="documentation"] -The {aws} ALB Ingress Controller for [.noloc]`Kubernetes` is a controller that causes an ALB to be created when ingress resources are created. +The {aws} ALB Ingress Controller for Kubernetes is a controller that causes an ALB to be created when ingress resources are created. [.update,date="2019-07-03"] === New Amazon EKS optimized AMI [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html +link:eks/latest/userguide/eks-optimized-ami.html[type="documentation"] Removing unnecessary `kubectl` binary from AMIs. [.update,date="2019-06-18"] -=== [.noloc]`Kubernetes` version `1.13` -Added [.noloc]`Kubernetes` version `1.13` support for new clusters and version upgrades. +=== Kubernetes version `1.13` +Added Kubernetes version `1.13` support for new clusters and version upgrades. [.update,date="2019-06-17"] === New Amazon EKS optimized AMI patched for `{aws}-2019-005` [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html +link:eks/latest/userguide/eks-optimized-ami.html[type="documentation"] Amazon EKS has updated the Amazon EKS optimized AMI to address the vulnerabilities that are described in link:security/security-bulletins/{aws}-2019-005/[{aws}-2019-005,type="marketing"]. @@ -1622,118 +1941,118 @@ Amazon EKS has updated the Amazon EKS optimized AMI to address the vulnerabiliti [.update,date="2019-05-21"] === Amazon EKS platform version update [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/platform-versions.html +link:eks/latest/userguide/platform-versions.html[type="documentation"] -New platform version for [.noloc]`Kubernetes` `1.11` and `1.10` clusters to support custom DNS names in the `kubelet` certificate and improve `etcd` performance. +New platform version for Kubernetes `1.11` and `1.10` clusters to support custom DNS names in the `kubelet` certificate and improve `etcd` performance. [.update,date="2019-05-21"] -=== Announcing discontinuation of support of [.noloc]`Kubernetes` `1.10` in Amazon EKS +=== Announcing discontinuation of support of Kubernetes `1.10` in Amazon EKS [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html +link:eks/latest/userguide/kubernetes-versions.html[type="documentation"] -Amazon EKS stopped supporting [.noloc]`Kubernetes` version `1.10` on July 22, 2019. +Amazon EKS stopped supporting Kubernetes version `1.10` on July 22, 2019. [.update,date="2019-05-10"] === Getting started with `eksctl` [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/getting-started-eksctl.html +link:eks/latest/userguide/getting-started-eksctl.html[type="documentation"] -This getting started guide describes how you can install all of the required resources to get started with Amazon EKS using `eksctl`. This is a simple command line utility for creating and managing [.noloc]`Kubernetes` clusters on Amazon EKS. +This getting started guide describes how you can install all of the required resources to get started with Amazon EKS using `eksctl`. This is a simple command line utility for creating and managing Kubernetes clusters on Amazon EKS. [.update,date="2019-05-10"] === {aws} CLI `get-token` command -The `aws eks get-token` command was added to the {aws} CLI. You no longer need to install the {aws} IAM Authenticator for [.noloc]`Kubernetes` to create client security tokens for cluster API server communication. Upgrade your {aws} CLI installation to the latest version to use this new functionality. For more information, see link:cli/latest/userguide/installing.html[Installing the {aws} Command Line Interface,type="documentation"] in the _{aws} Command Line Interface User Guide_. +The `aws eks get-token` command was added to the {aws} CLI. You no longer need to install the {aws} IAM Authenticator for Kubernetes to create client security tokens for cluster API server communication. Upgrade your {aws} CLI installation to the latest version to use this new functionality. For more information, see link:cli/latest/userguide/installing.html[Installing the {aws} Command Line Interface,type="documentation"] in the _{aws} Command Line Interface User Guide_. [.update,date="2019-05-08"] === Amazon EKS platform version update [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/platform-versions.html +link:eks/latest/userguide/platform-versions.html[type="documentation"] -New platform version for [.noloc]`Kubernetes` `1.12` clusters to support custom DNS names in the `kubelet` certificate and improve `etcd` performance. This fixes a bug that caused node `kubelet` daemons to request a new certificate every few seconds. +New platform version for Kubernetes `1.12` clusters to support custom DNS names in the `kubelet` certificate and improve `etcd` performance. This fixes a bug that caused node `kubelet` daemons to request a new certificate every few seconds. [.update,date="2019-04-05"] -=== [.noloc]`Prometheus` tutorial +=== Prometheus tutorial [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/prometheus.html +link:eks/latest/userguide/prometheus.html[type="documentation"] -Added topic for deploying [.noloc]`Prometheus` to your Amazon EKS cluster. +Added topic for deploying Prometheus to your Amazon EKS cluster. [.update,date="2019-04-04"] === Amazon EKS control plane logging [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html +link:eks/latest/userguide/control-plane-logs.html[type="documentation"] With this update, you can get audit and diagnostic logs directly from the Amazon EKS control pane. You can use these CloudWatch logs in your account as reference for securing and running clusters. [.update,date="2019-03-28"] -=== [.noloc]`Kubernetes` version `1.12` -Added [.noloc]`Kubernetes` version `1.12` support for new clusters and version upgrades. +=== Kubernetes version `1.12` +Added Kubernetes version `1.12` support for new clusters and version upgrades. [.update,date="2019-03-27"] === Added App Mesh getting started guide [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/mesh-gs-k8s.html +link:eks/latest/userguide/mesh-gs-k8s.html[type="documentation"] -Added documentation for getting started with App Mesh and [.noloc]`Kubernetes`. +Added documentation for getting started with App Mesh and Kubernetes. [.update,date="2019-03-19"] === Amazon EKS API server endpoint private access [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/cluster-endpoint.html +link:eks/latest/userguide/cluster-endpoint.html[type="documentation"] -Added documentation for disabling public access for your Amazon EKS cluster's [.noloc]`Kubernetes` API server endpoint. +Added documentation for disabling public access for your Amazon EKS cluster's Kubernetes API server endpoint. [.update,date="2019-03-18"] -=== Added topic for installing the [.noloc]`Kubernetes` Metrics Server +=== Added topic for installing the Kubernetes Metrics Server [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/metrics-server.html +link:eks/latest/userguide/metrics-server.html[type="documentation"] -The [.noloc]`Kubernetes` Metrics Server is an aggregator of resource usage data in your cluster. +The Kubernetes Metrics Server is an aggregator of resource usage data in your cluster. [.update,date="2019-03-15"] === Added list of related open source projects [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/metrics-server.html +link:eks/latest/userguide/metrics-server.html[type="documentation"] -These open source projects extend the functionality of [.noloc]`Kubernetes` clusters running on {aws}, including clusters that are managed by Amazon EKS. +These open source projects extend the functionality of Kubernetes clusters running on {aws}, including clusters that are managed by Amazon EKS. [.update,date="2019-03-11"] === Added topic for installing Helm locally [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/helm.html +link:eks/latest/userguide/helm.html[type="documentation"] -The `helm` package manager for [.noloc]`Kubernetes` helps you install and manage applications on your [.noloc]`Kubernetes` cluster. This topic shows how to install and run the `helm` and `tiller` binaries locally. That way, you can install and manage charts using the Helm CLI on your local system. +The `helm` package manager for Kubernetes helps you install and manage applications on your Kubernetes cluster. This topic shows how to install and run the `helm` and `tiller` binaries locally. That way, you can install and manage charts using the Helm CLI on your local system. [.update,date="2019-03-08"] === Amazon EKS platform version update [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/platform-versions.html +link:eks/latest/userguide/platform-versions.html[type="documentation"] -New platform version that updates Amazon EKS [.noloc]`Kubernetes` `1.11` clusters to patch level `1.11.8` to address https://discuss.kubernetes.io/t/kubernetes-security-announcement-v1-11-8-1-12-6-1-13-4-released-to-address-medium-severity-cve-2019-1002100/5147[CVE-2019-1002100]. +New platform version that updates Amazon EKS Kubernetes `1.11` clusters to patch level `1.11.8` to address https://discuss.kubernetes.io/t/kubernetes-security-announcement-v1-11-8-1-12-6-1-13-4-released-to-address-medium-severity-cve-2019-1002100/5147[CVE-2019-1002100]. [.update,date="2019-02-13"] === Amazon EKS {aws} Region expansion -Amazon EKS is now available in the Europe (London) (`eu-west-2`), Europe (Paris) (`eu-west-3`), and Asia Pacific (Mumbai) (``ap-south-1`) {aws} Regions. +Amazon EKS is now available in the Europe (London) (`eu-west-2`), Europe (Paris) (`eu-west-3`), and Asia Pacific (Mumbai) (`ap-south-1`) {aws} Regions. [.update,date="2019-02-13"] === Increased cluster limit [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/service_limits.html +link:eks/latest/userguide/service_limits.html[type="documentation"] Amazon EKS has increased the number of clusters that you can create in an {aws} Region from 3 to 50. @@ -1741,7 +2060,7 @@ Amazon EKS has increased the number of clusters that you can create in an {aws} [.update,date="2019-02-11"] === New Amazon EKS optimized AMI patched for `ALAS-2019-1156` [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html +link:eks/latest/userguide/eks-optimized-ami.html[type="documentation"] Amazon EKS has updated the Amazon EKS optimized AMI to address the vulnerability that's described in https://alas.aws.amazon.com/ALAS-2019-1156.html[ALAS-2019-1156]. @@ -1749,7 +2068,7 @@ Amazon EKS has updated the Amazon EKS optimized AMI to address the vulnerability [.update,date="2019-01-09"] === New Amazon EKS optimized AMI patched for `ALAS2-2019-1141` [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html +link:eks/latest/userguide/eks-optimized-ami.html[type="documentation"] Amazon EKS has updated the Amazon EKS optimized AMI to address the CVEs that are referenced in https://alas.aws.amazon.com/AL2/ALAS-2019-1141.html[ALAS2-2019-1141]. @@ -1767,7 +2086,7 @@ Amazon EKS is now available in the following additional {aws} Regions: Europe (F [.update,date="2018-12-12"] === Amazon EKS cluster updates [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html +link:eks/latest/userguide/update-cluster.html[type="documentation"] Added documentation for Amazon EKS link:eks/latest/userguide/update-cluster.html[cluster Kubernetes version updates,type="documentation"] and link:eks/latest/userguide/update-workers.html[node replacement,type="documentation"]. @@ -1780,9 +2099,9 @@ Amazon EKS is now available in the Europe (Stockholm) (`eu-north-1`) {aws} Regio [.update,date="2018-12-04"] === Amazon EKS platform version update [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/platform-versions.html +link:eks/latest/userguide/platform-versions.html[type="documentation"] -New platform version updating [.noloc]`Kubernetes` to patch level `1.10.11` to address link:security/security-bulletins/{aws}-2018-020/[CVE-2018-1002105,type="marketing"]. +New platform version updating Kubernetes to patch level `1.10.11` to address link:security/security-bulletins/{aws}-2018-020/[CVE-2018-1002105,type="marketing"]. [.update,date="2018-11-20"] @@ -1796,15 +2115,15 @@ The ALB ingress controller releases version `1.0.0` with formal support from {aw [.update,date="2018-10-16"] === Added support for CNI network configuration [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html +link:eks/latest/userguide/cni-custom-network.html[type="documentation"] -The [.noloc]`Amazon VPC CNI plugin for Kubernetes` version `1.2.1` now supports custom network configuration for secondary [.noloc]`Pod` network interfaces. +The Amazon VPC CNI plugin for Kubernetes version `1.2.1` now supports custom network configuration for secondary Pod network interfaces. [.update,date="2018-10-10"] === Added support for `MutatingAdmissionWebhook` and `ValidatingAdmissionWebhook` [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/platform-versions.html +link:eks/latest/userguide/platform-versions.html[type="documentation"] Amazon EKS platform version `1.10-eks.2` now supports `MutatingAdmissionWebhook` and `ValidatingAdmissionWebhook` admission controllers. @@ -1812,7 +2131,7 @@ Amazon EKS platform version `1.10-eks.2` now supports `MutatingAdmissionWebhook` [.update,date="2018-10-03"] === Added partner AMI information [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/eks-partner-amis.html +link:eks/latest/userguide/eks-partner-amis.html[type="documentation"] Canonical has partnered with Amazon EKS to create node AMIs that you can use in your clusters. @@ -1820,7 +2139,7 @@ Canonical has partnered with Amazon EKS to create node AMIs that you can use in [.update,date="2018-09-21"] === Added instructions for {aws} CLI `update-kubeconfig` command [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/create-kubeconfig.html +link:eks/latest/userguide/create-kubeconfig.html[type="documentation"] Amazon EKS has added the `update-kubeconfig` to the {aws} CLI to simplify the process of creating a `kubeconfig` file for accessing your cluster. @@ -1828,7 +2147,7 @@ Amazon EKS has added the `update-kubeconfig` to the {aws} CLI to simplify the pr [.update,date="2018-09-13"] === New Amazon EKS optimized AMIs [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html +link:eks/latest/userguide/eks-optimized-ami.html[type="documentation"] Amazon EKS has updated the Amazon EKS optimized AMIs (with and without GPU support) to provide various security fixes and AMI optimizations. @@ -1841,15 +2160,15 @@ Amazon EKS is now available in the Europe (Ireland) (`eu-west-1`) Region. [.update,date="2018-08-31"] === Amazon EKS platform version update [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/platform-versions.html +link:eks/latest/userguide/platform-versions.html[type="documentation"] -New platform version with support for [.noloc]`Kubernetes` https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/apiserver-aggregation/[aggregation layer] and the https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/[Horizontal Pod Autoscaler](HPA). +New platform version with support for Kubernetes https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/apiserver-aggregation/[aggregation layer] and the https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/[Horizontal Pod Autoscaler](HPA). [.update,date="2018-08-22"] === New Amazon EKS optimized AMIs and GPU support [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html +link:eks/latest/userguide/eks-optimized-ami.html[type="documentation"] Amazon EKS has updated the Amazon EKS optimized AMI to use a new {aws} CloudFormation node template and https://github.com/awslabs/amazon-eks-ami/blob/main/templates/al2/runtime/bootstrap.sh[bootstrap script]. In addition, a new link:eks/latest/userguide/eks-optimized-ami.html#gpu-ami[Amazon EKS optimized AMI with GPU support,type="documentation"] is available. @@ -1857,7 +2176,7 @@ Amazon EKS has updated the Amazon EKS optimized AMI to use a new {aws} CloudForm [.update,date="2018-08-14"] === New Amazon EKS optimized AMI patched for `ALAS2-2018-1058` [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html +link:eks/latest/userguide/eks-optimized-ami.html[type="documentation"] Amazon EKS has updated the Amazon EKS optimized AMI to address the CVEs that are referenced in https://alas.aws.amazon.com/AL2/ALAS-2018-1058.html[ALAS2-2018-1058]. @@ -1865,9 +2184,9 @@ Amazon EKS has updated the Amazon EKS optimized AMI to address the CVEs that are [.update,date="2018-07-10"] === Amazon EKS optimized AMI build scripts [.update-ulink] -https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html +link:eks/latest/userguide/eks-optimized-ami.html[type="documentation"] -Amazon EKS has open-sourced the build scripts that are used to build the Amazon EKS optimized AMI. These build scripts are now available on [.noloc]`GitHub`. +Amazon EKS has open-sourced the build scripts that are used to build the Amazon EKS optimized AMI. These build scripts are now available on GitHub. [.update,date="2018-06-05"] diff --git a/latest/ug/getting-started/getting-started-automode.adoc b/latest/ug/getting-started/getting-started-automode.adoc index 734e148be..711b94dec 100644 --- a/latest/ug/getting-started/getting-started-automode.adoc +++ b/latest/ug/getting-started/getting-started-automode.adoc @@ -1,15 +1,9 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[getting-started-automode,getting-started-automode.title]] +[#getting-started-automode] = Get started with Amazon EKS – EKS Auto Mode -:info_doctype: section -:info_title: Get started with Amazon EKS – \ - EKS Auto Mode -:info_titleabbrev: Create your first cluster – EKS Auto Mode -:keywords: using, Auto, getting, started, tutorial -:info_abstract: Learn how to create your first Amazon EKS cluster with nodes using EKS Auto Mode +:info_titleabbrev: Create cluster (EKS Auto Mode) Like other EKS getting started experiences, creating your first cluster with EKS Auto Mode delegates the management of the cluster itself to {aws}. However, EKS Auto Mode extends EKS automation by handing responsibility of many essential services needed to set up workload infrastructure (nodes, networks, and various services), making it easier to manage nodes and scale up to meet workload demands. @@ -17,7 +11,7 @@ However, EKS Auto Mode extends EKS automation by handing responsibility of many Choose from one of the following ways to create a cluster with EKS Auto Mode: * <>: Use the `aws` command line interface to create a cluster. -* <>: Use the {aws}} Management Console to create a cluster. +* <>: Use the {aws-management-console} to create a cluster. * <>: Use the `eksctl` command line interface to create a cluster. If you are comparing different approaches to creating your first EKS cluster, @@ -28,4 +22,4 @@ that include setting up components to: * Regularly upgrade the cluster itself (control plane), node operating systems, and services running on nodes. * Choose default settings that determine things like the size and speed of node storage and Pod network configuration. -For details on what you get with EKS Auto Mode clusters, see <>. +For details on what you get with EKS Auto Mode clusters, see <>. \ No newline at end of file diff --git a/latest/ug/getting-started/getting-started-console.adoc b/latest/ug/getting-started/getting-started-console.adoc index f593f89c3..79af586bf 100644 --- a/latest/ug/getting-started/getting-started-console.adoc +++ b/latest/ug/getting-started/getting-started-console.adoc @@ -1,16 +1,9 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[getting-started-console,getting-started-console.title]] +[#getting-started-console] = Get started with Amazon EKS – {aws-management-console} and {aws} CLI -:info_doctype: section -:info_title: Get started with Amazon EKS – {aws-management-console} and \ - {aws} CLI -:info_titleabbrev: Create your first cluster – {aws-management-console} -:keywords: using, {aws-management-console}, {aws} CLI, getting, started, tutorial -:info_abstract: Learn how to create your first Amazon EKS cluster with nodes using the {aws-management-console} and \ - {aws} CLI. +:info_titleabbrev: Create cluster (Console and CLI) [abstract] -- @@ -20,28 +13,27 @@ Learn how to create your first Amazon EKS cluster with nodes using the {aws-mana [NOTE] ==== -This topic covers getting started *without* EKS Auto Mode. +This topic covers getting started *without* EKS Auto Mode. It uses Managed Node Groups to deploy nodes. -EKS Auto Mode automates routine tasks for cluster compute, storage, and networking. xref:getting-started-automode[Learn how to get started with Amazon EKS Auto Mode. ] +EKS Auto Mode automates routine tasks for cluster compute, storage, and networking. <> EKS Auto Mode is the preferred method of deploying nodes. ==== This guide helps you to create all of the required resources to get started with Amazon Elastic Kubernetes Service (Amazon EKS) using the {aws-management-console} and the {aws} CLI. In this guide, you manually create each resource. At the end of this tutorial, you will have a running Amazon EKS cluster that you can deploy applications to. The procedures in this guide give you complete visibility into how each resource is created and how the resources interact with each other. If you'd rather have most of the resources created for you automatically, use the `eksctl` CLI to create your cluster and nodes. For more information, see <>. -[[eks-prereqs,eks-prereqs.title]] +[#eks-prereqs] == Prerequisites Before starting this tutorial, you must install and configure the following tools and resources that you need to create and manage an Amazon EKS cluster. - * *{aws} CLI* - – A command line tool for working with {aws} services, including Amazon EKS. For more information, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] in the {aws} Command Line Interface User Guide. After installing the {aws} CLI, we recommend that you also configure it. For more information, see link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the {aws} Command Line Interface User Guide. Note that {aws} CLI v2 is required to use the *update-kubeconfig* option shown in this page. + – A command line tool for working with {aws} services, including Amazon EKS. For more information, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] in the {aws} Command Line Interface User Guide. After installing the {aws} CLI, we recommend that you also configure it. For more information, see link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the {aws} Command Line Interface User Guide. Note that {aws} CLI v2 is required to use the *update-kubeconfig* option shown in this page. * *`kubectl`* - – A command line tool for working with [.noloc]`Kubernetes` clusters. For more information, see <>. + – A command line tool for working with Kubernetes clusters. For more information, see <>. * *Required IAM permissions* - – The IAM security principal that you're using must have permissions to work with Amazon EKS IAM roles, service linked roles, {aws} CloudFormation, a VPC, and related resources. For more information, see link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html[Actions, resources, and condition keys for Amazon Elastic Kubernetes Service,type="documentation"] and link:IAM/latest/UserGuide/using-service-linked-roles.html[Using service-linked roles,type="documentation"] in the IAM User Guide. You must complete all steps in this guide as the same user. To check the current user, run the following command: + – The IAM security principal that you're using must have permissions to work with Amazon EKS IAM roles, service linked roles, {aws} CloudFormation, a VPC, and related resources. For more information, see link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html[Actions, resources, and condition keys for Amazon Elastic Kubernetes Service,type="documentation"] and link:IAM/latest/UserGuide/using-service-linked-roles.html[Using service-linked roles,type="documentation"] in the IAM User Guide. You must complete all steps in this guide as the same user. To check the current user, run the following command: + [source,bash,subs="verbatim,attributes"] ---- @@ -51,7 +43,7 @@ aws sts get-caller-identity We recommend that you complete the steps in this topic in a Bash shell. If you aren't using a Bash shell, some script commands such as line continuation characters and the way variables are set and used require adjustment for your shell. Additionally, the quoting and escaping rules for your shell might be different. For more information, see link:cli/latest/userguide/cli-usage-parameters-quoting-strings.html[Using quotation marks with strings in the {aws} CLI,type="documentation"] in the {aws} Command Line Interface User Guide. -[[eks-create-cluster,eks-create-cluster.title]] +[#eks-create-cluster] == Step 1: Create your Amazon EKS cluster [IMPORTANT] @@ -62,33 +54,22 @@ To get started as simply and quickly as possible, this topic includes steps to c ==== . Create an Amazon VPC with public and private subnets that meets Amazon EKS requirements. Replace [.replaceable]`region-code` with any {aws} Region that is supported by Amazon EKS. For a list of {aws} Regions, see link:general/latest/gr/eks.html[Amazon EKS endpoints and quotas,type="documentation"] in the {aws} General Reference guide. You can replace [.replaceable]`my-eks-vpc-stack` with any name you choose. + -[source,bash,subs="verbatim,attributes"] +[source,bash,subs="verbatim,attributes,quotes"] ---- aws cloudformation create-stack \ - --region region-code \ + --region [.replaceable]`region-code` \ --stack-name my-eks-vpc-stack \ --template-url https://s3.us-west-2.amazonaws.com/amazon-eks/cloudformation/2020-10-29/amazon-eks-vpc-private-subnets.yaml ---- + -TIP: For a list of all the resources the previous command creates, open the {aws} CloudFormation console at link:cloudformation/[cloudformation,type="console"]. Choose the [.replaceable]`my-eks-vpc-stack` stack and then choose the *Resources* tab. -. Create a cluster IAM role and attach the required Amazon EKS IAM managed policy to it. [.noloc]`Kubernetes` clusters managed by Amazon EKS make calls to other {aws} services on your behalf to manage the resources that you use with the service. +TIP: For a list of all the resources the previous command creates, open the {aws} CloudFormation console at https://console.aws.amazon.com/cloudformation/. Choose the [.replaceable]`my-eks-vpc-stack` stack and then choose the *Resources* tab. +. Create a cluster IAM role and attach the required Amazon EKS IAM managed policy to it. Kubernetes clusters managed by Amazon EKS make calls to other {aws} services on your behalf to manage the resources that you use with the service. + .. Copy the following contents to a file named [.replaceable]`eks-cluster-role-trust-policy.json`. + [source,json,subs="verbatim,attributes"] ---- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Service": "eks.amazonaws.com" - }, - "Action": "sts:AssumeRole" - } - ] -} +include::iam_snippet:a8ac9ca8-ab01-49a8-9de7-ab3a2a9195b5[] ---- .. Create the role. + @@ -106,32 +87,35 @@ aws iam attach-role-policy \ --policy-arn {arn-aws}iam::aws:policy/AmazonEKSClusterPolicy \ --role-name myAmazonEKSClusterRole ---- -. Open the Amazon EKS console at https://console.aws.amazon.com/eks/home#/clusters[https://console.aws.amazon.com/eks/home#/clusters]. +. Open the Amazon EKS console at link:eks/home#/clusters[https://console.aws.amazon.com/eks/home#/clusters,type="console"]. + Make sure that the {aws} Region shown in the upper right of your console is the {aws} Region that you want to create your cluster in. If it's not, choose the dropdown next to the {aws} Region name and choose the {aws} Region that you want to use. -. Choose *Add cluster*, and then choose *Create*. If you don't see this option, then choose *Clusters* in the left navigation pane first. +. Choose *Create cluster*. If you don't see this option, then choose *Clusters* in the left navigation pane first. . On the *Configure cluster* page, do the following: + -.. Enter a *Name* for your cluster, such as `my-cluster`. The name can contain only alphanumeric characters (case-sensitive) and hyphens. It must start with an alphanumeric character and can't be longer than 100 characters. The name must be unique within the {aws} Region and {aws} account that you're creating the cluster in. +.. Select *Custom configuration* and disable *Use EKS Auto Mode*. (If you prefer an EKS Auto Mode cluster, refer instead to <>.) +.. Enter a *Name* for your cluster, such as [.replaceable]`my-cluster`. The name can contain only alphanumeric characters (case-sensitive) and hyphens. It must start with an alphanumeric character and can't be longer than 100 characters. The name must be unique within the {aws} Region and {aws} account that you're creating the cluster in. .. For *Cluster Service Role*, choose [.replaceable]`myAmazonEKSClusterRole`. .. Leave the remaining settings at their default values and choose *Next*. . On the *Specify networking* page, do the following: + -.. Choose the ID of the VPC that you created in a previous step from the *VPC* dropdown list. It is something like [.replaceable]`vpc-00x0000x000x0x000` | [.replaceable]`my-eks-vpc-stack-VPC`. +.. Choose the ID of the VPC that you created in a previous step from the *VPC* dropdown list. It is something like [.replaceable]`* | my-eks-vpc-stack-VPC`. +.. Choose the subnets created in a previous step from the *Subnets* dropdown list. The subnets will be something like [.replaceable]`* | my-eks-vpc-stack-*`. +.. Choose the security group created in a previous step from the *Additional security groups* dropdown list. It is something like [.replaceable]`* | my-eks-vpc-stack-ControlPlaneSecurityGroup-*`. .. Leave the remaining settings at their default values and choose *Next*. -. On the *Configure observability* page, choose *Next*. -. On the *Select add-ons* page, choose *Next*. +. On the *Configure observability* page, choose *Next*. +. On the *Select add-ons* page, choose *Next*. + For more information on add-ons, see <>. -. On the *Configure selected add-ons settings* page, choose *Next*. -. On the *Review and create* page, choose *Create*. +. On the *Configure selected add-ons settings* page, choose *Next*. +. On the *Review and create* page, choose *Create*. + -To the right of the cluster's name, the cluster status is *Creating* for several minutes until the cluster provisioning process completes. Don't continue to the next step until the status is *Active*. +To the right of the cluster's name, the cluster status is *Creating* for several minutes until the cluster provisioning process completes. Don't continue to the next step until the status is *Active*. + NOTE: You might receive an error that one of the Availability Zones in your request doesn't have sufficient capacity to create an Amazon EKS cluster. If this happens, the error output contains the Availability Zones that can support a new cluster. Retry creating your cluster with at least two subnets that are located in the supported Availability Zones for your account. For more information, see <>. -[[eks-configure-kubectl,eks-configure-kubectl.title]] +[#eks-configure-kubectl] == Step 2: Configure your computer to communicate with your cluster In this section, you create a `kubeconfig` file for your cluster. The settings in this file enable the `kubectl` CLI to communicate with your cluster. @@ -140,9 +124,9 @@ Before proceeding, be sure that your cluster creation completed successfully in . Create or update a `kubeconfig` file for your cluster. Replace [.replaceable]`region-code` with the {aws} Region that you created your cluster in. Replace [.replaceable]`my-cluster` with the name of your cluster. + -[source,bash,subs="verbatim,attributes"] +[source,bash,subs="verbatim,attributes,quotes"] ---- -aws eks update-kubeconfig --region region-code --name my-cluster +aws eks update-kubeconfig --region [.replaceable]`region-code` --name [.replaceable]`my-cluster` ---- + By default, the `config` file is created in `~/.kube` or the new cluster's configuration is added to an existing `config` file in `~/.kube`. @@ -164,141 +148,31 @@ svc/kubernetes ClusterIP 10.100.0.1 443/TCP 1m ---- -[[eks-launch-workers,eks-launch-workers.title]] +[#eks-launch-workers] == Step 3: Create nodes [IMPORTANT] ==== -To get started as simply and quickly as possible, this topic includes steps to create nodes with default settings. Before creating nodes for production use, we recommend that you familiarize yourself with all settings and deploy nodes with the settings that meet your requirements. For more information, see <>. Some settings can only be enabled when creating your nodes. +To get started as simply and quickly as possible, this topic includes steps to create nodes with mostly default settings. Before creating nodes for production use, we recommend that you familiarize yourself with all settings and deploy nodes with the settings that meet your requirements. For more information, see <>. Some settings can only be enabled when creating your nodes. ==== -You can create a cluster with one of the following node types. To learn more about each type, see <>. After your cluster is deployed, you can add other node types. - -* *Fargate – [.noloc]``Linux``* – Choose this type of node if you want to run [.noloc]``Linux`` applications on <>. Fargate is a serverless compute engine that lets you deploy [.noloc]``Kubernetes``[.noloc]``Pods`` without managing Amazon EC2 instances. -* *Managed nodes – [.noloc]``Linux``* – Choose this type of node if you want to run Amazon Linux applications on Amazon EC2 instances. Though not covered in this guide, you can also add <> and <> nodes to your cluster. +This procedure configures your cluster to use Managed node groups to create nodes, specifying the subnets and node IAM role that you created in previous steps. +It lets you run Amazon Linux applications on Amazon EC2 instances. -==== -[role="tablist"] -Fargate - [.noloc]`Linux`:: - -Create a Fargate profile. When [.noloc]``Kubernetes``[.noloc]``Pods`` are deployed with criteria that matches the criteria defined in the profile, the [.noloc]``Pods`` are deployed to Fargate. -+ -*To create a Fargate profile* -+ -. Create an IAM role and attach the required Amazon EKS IAM managed policy to it. When your cluster creates [.noloc]``Pods`` on Fargate infrastructure, the components running on the Fargate infrastructure must make calls to {aws} APIs on your behalf. This is so that they can do actions such as pull container images from Amazon ECR or route logs to other {aws} services. The Amazon EKS [.noloc]``Pod`` execution role provides the IAM permissions to do this. - -.. Copy the following contents to a file named `pod-execution-role-trust-policy.json`. Replace [.replaceable]`region-code` with the {aws} Region that your cluster is in. If you want to use the same role in all {aws} Regions in your account, replace [.replaceable]`region-code` with `{asterisk}`. Replace [.replaceable]`111122223333` with your account ID and [.replaceable]`my-cluster` with the name of your cluster. If you want to use the same role for all clusters in your account, replace [.replaceable]`my-cluster` with `{asterisk}`. -+ -[source,json,subs="verbatim,attributes"] ----- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Condition": { - "ArnLike": { - "aws:SourceArn": "{arn-aws}eks:region-code:111122223333:fargateprofile/my-cluster/*" - } - }, - "Principal": { - "Service": "eks-fargate-pods.amazonaws.com" - }, - "Action": "sts:AssumeRole" - } - ] -} ----- - -.. Create a [.noloc]``Pod`` execution IAM role. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws iam create-role \ - --role-name AmazonEKSFargatePodExecutionRole \ - --assume-role-policy-document file://"pod-execution-role-trust-policy.json" ----- - -.. Attach the required Amazon EKS managed IAM policy to the role. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws iam attach-role-policy \ - --policy-arn {arn-aws}iam::aws:policy/AmazonEKSFargatePodExecutionRolePolicy \ - --role-name AmazonEKSFargatePodExecutionRole ----- -.. Open the Amazon EKS console at https://console.aws.amazon.com/eks/home#/clusters[https://console.aws.amazon.com/eks/home#/clusters]. -.. On the *Clusters* page, choose the [.replaceable]`my-cluster` cluster. -.. On the *[.replaceable]`my-cluster`* page, do the following: -+ -.. Choose the *Compute* tab. -.. Under *Fargate Profiles*, choose *Add Fargate Profile*. -. On the *Configure Fargate Profile* page, do the following: -+ -.. For *Name*, enter a unique name for your Fargate profile, such as [.replaceable]`my-profile`. -.. For *Pod execution role*, choose the *AmazonEKSFargatePodExecutionRole* that you created in a previous step. -.. Choose the *Subnets* dropdown and deselect any subnet with `Public` in its name. Only private subnets are supported for [.noloc]``Pods`` that are running on Fargate. -.. Choose *Next*. -. On the *Configure [.noloc]``Pod`` selection* page, do the following: -+ -.. For *Namespace*, enter `default`. -.. Choose *Next*. -. On the *Review and create* page, review the information for your Fargate profile and choose *Create*. -. After a few minutes, the *Status* in the *Fargate Profile configuration* section will change from *Creating* to *Active*. Don't continue to the next step until the status is *Active*. -. If you plan to deploy all [.noloc]`Pods` to Fargate (none to Amazon EC2 nodes), do the following to create another Fargate profile and run the default name resolver ([.noloc]`CoreDNS`) on Fargate. -+ -NOTE: If you don't do this, you won't have any nodes at this time. -+ -.. On the *Fargate Profile* page, choose [.replaceable]`my-profile`. -.. Under *Fargate profiles*, choose *Add Fargate Profile*. -.. For *Name*, enter [.noloc]`CoreDNS`. -.. For *Pod execution role*, choose the *AmazonEKSFargatePodExecutionRole* that you created in a previous step. -.. Choose the *Subnets* dropdown and deselect any subnet with `Public` in its name. Only private subnets are supported for [.noloc]`Pods` running on Fargate. -.. Choose *Next*. -.. For *Namespace*, enter `kube-system`. -.. Choose *Match labels*, and then choose *Add label*. -.. Enter `k8s-app` for *Key* and `kube-dns` for value. This is necessary for the default name resolver ([.noloc]`CoreDNS`) to deploy to Fargate. -.. Choose *Next*. -.. On the *Review and create* page, review the information for your Fargate profile and choose *Create*. -.. Run the following command to remove the default `eks.amazonaws.com/compute-type : ec2` annotation from the [.noloc]`CoreDNS` [.noloc]`Pods`. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl patch deployment coredns \ - -n kube-system \ - --type json \ - -p='[{"op": "remove", "path": "/spec/template/metadata/annotations/eks.amazonaws.com~1compute-type"}]' ----- -+ -NOTE: The system creates and deploys two nodes based on the Fargate profile label you added. You won't see anything listed in *Node groups* because they aren't applicable for Fargate nodes, but you will see the new nodes listed in the *Overview* tab. +To learn more about different ways to configure nodes in EKS, see <>. After your cluster is deployed, you can add other node types. Though not covered in this guide, you can also add <> and <> nodes to your cluster. -Managed nodes - [.noloc]`Linux`:: +*To create your EC2 Linux managed node group* -Create a managed node group, specifying the subnets and node IAM role that you created in previous steps. -+ -*To create your {ec2} [.noloc]`Linux` managed node group* -+ . Create a node IAM role and attach the required Amazon EKS IAM managed policy to it. The Amazon EKS node `kubelet` daemon makes calls to {aws} APIs on your behalf. Nodes receive permissions for these API calls through an IAM instance profile and associated policies. -+ + .. Copy the following contents to a file named `node-role-trust-policy.json`. + [source,json,subs="verbatim,attributes"] ---- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Service": "ec2.amazonaws.com" - }, - "Action": "sts:AssumeRole" - } - ] -} +include::iam_snippet:563b8603-744b-4278-b4ec-a1f971ffd1f0[] ---- .. Create the node IAM role. + @@ -322,7 +196,7 @@ aws iam attach-role-policy \ --policy-arn {arn-aws}iam::aws:policy/AmazonEKS_CNI_Policy \ --role-name myAmazonEKSNodeRole ---- -.. Open the Amazon EKS console at https://console.aws.amazon.com/eks/home#/clusters[https://console.aws.amazon.com/eks/home#/clusters]. +.. Open the Amazon EKS console at link:eks/home#/clusters[https://console.aws.amazon.com/eks/home#/clusters,type="console"]. .. Choose the name of the cluster that you created in <>, such as [.replaceable]`my-cluster`. .. On the *[.replaceable]`my-cluster`* page, do the following: + @@ -333,42 +207,38 @@ aws iam attach-role-policy \ .. For *Name*, enter a unique name for your managed node group, such as [.replaceable]`my-nodegroup`. The node group name can't be longer than 63 characters. It must start with letter or digit, but can also include hyphens and underscores for the remaining characters. .. For *Node IAM role name*, choose [.replaceable]`myAmazonEKSNodeRole` role that you created in a previous step. We recommend that each node group use its own unique IAM role. .. Choose *Next*. -. On the *Set compute and scaling configuration* page, accept the default values and choose *Next*. -. On the *Specify networking* page, accept the default values and choose *Next*. -. On the *Review and create* page, review your managed node group configuration and choose *Create*. -. After several minutes, the *Status* in the *Node Group configuration* section will change from *Creating* to *Active*. Don't continue to the next step until the status is *Active*. -==== +. On the *Set compute and scaling configuration* page, accept the default values and choose *Next*. +. On the *Specify networking* page, accept the default values and choose *Next*. +. On the *Review and create* page, review your managed node group configuration and choose *Create*. +. After several minutes, the *Status* in the *Node Group configuration* section will change from *Creating* to *Active*. Don't continue to the next step until the status is *Active*. -[[gs-view-resources,gs-view-resources.title]] +[#gs-view-resources] == Step 4: View resources -You can view your nodes and [.noloc]`Kubernetes` workloads. +You can view your nodes and Kubernetes workloads. -. In the left navigation pane, choose *Clusters*. In the list of *Clusters*, choose the name of the cluster that you created, such as [.replaceable]`my-cluster`. +. In the left navigation pane, choose *Clusters*. In the list of *Clusters*, choose the name of the cluster that you created, such as [.replaceable]`my-cluster`. . On the *[.replaceable]`my-cluster`* page, choose the following: + .. *Compute* - tab – You see the list of *Nodes* that were deployed for the cluster. You can choose the name of a node to see more information about it. + tab – You see the list of *Nodes* that were deployed for the cluster. You can choose the name of a node to see more information about it. .. *Resources* tab - – You see all of the [.noloc]`Kubernetes` resources that are deployed by default to an Amazon EKS cluster. Select any resource type in the console to learn more about it. + – You see all of the Kubernetes resources that are deployed by default to an Amazon EKS cluster. Select any resource type in the console to learn more about it. -[[gs-console-clean-up,gs-console-clean-up.title]] +[#gs-console-clean-up] == Step 5: Delete resources After you've finished with the cluster and nodes that you created for this tutorial, you should delete the resources that you created. If you want to do more with this cluster before you delete the resources, see <>. -. Delete any node groups or Fargate profiles that you created. +. Delete any node groups profiles that you created. + -.. Open the Amazon EKS console at https://console.aws.amazon.com/eks/home#/clusters[https://console.aws.amazon.com/eks/home#/clusters]. +.. Open the Amazon EKS console at link:eks/home#/clusters[https://console.aws.amazon.com/eks/home#/clusters,type="console"]. .. In the left navigation pane, choose *Clusters*. In the list of clusters, choose [.replaceable]`my-cluster`. .. Choose the *Compute* tab. .. If you created a node group, choose the [.replaceable]`my-nodegroup` node group and then choose *Delete*. Enter [.replaceable]`my-nodegroup`, and then choose *Delete*. -.. For each Fargate profile that you created, choose it and then choose *Delete*. Enter the name of the profile, and then choose *Delete*. -+ -NOTE: When deleting a second Fargate profile, you may need to wait for the first one to finish deleting. -.. Don't continue until the node group or Fargate profiles are deleted. +.. Don't continue until the node group profiles are deleted. . Delete the cluster. + .. In the left navigation pane, choose *Clusters*. In the list of clusters, choose [.replaceable]`my-cluster`. @@ -376,24 +246,22 @@ NOTE: When deleting a second Fargate profile, you may need to wait for the first .. Enter [.replaceable]`my-cluster` and then choose *Delete*. Don't continue until the cluster is deleted. . Delete the VPC {aws} CloudFormation stack that you created. + -.. Open the link:cloudformation/[{aws} CloudFormation console,type="console"]. +.. Open the CloudFormation console at https://console.aws.amazon.com/cloudformation/. .. Choose the [.replaceable]`my-eks-vpc-stack` stack, and then choose *Delete*. -.. In the *Delete [.replaceable]`my-eks-vpc-stack`* confirmation dialog box, choose *Delete stack*. +.. In the *Delete [.replaceable]`my-eks-vpc-stack`* confirmation dialog box, choose *Delete stack*. . Delete the IAM roles that you created. + .. Open the IAM console at https://console.aws.amazon.com/iam/. .. In the left navigation pane, choose *Roles*. -.. Select each role you created from the list (*[.replaceable]`myAmazonEKSClusterRole`*, as well as *AmazonEKSFargatePodExecutionRole* or [.replaceable]`myAmazonEKSNodeRole`). Choose *Delete*, enter the requested confirmation text, then choose *Delete*. +.. Select each role you created from the list (*[.replaceable]`myAmazonEKSClusterRole`*, as well as [.replaceable]`myAmazonEKSNodeRole`). Choose *Delete*, enter the requested confirmation text, then choose *Delete*. -[[gs-console-next-steps,gs-console-next-steps.title]] +[#gs-console-next-steps] == Next steps The following documentation topics help you to extend the functionality of your cluster. - - -* The link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] that created the cluster is the only principal that can make calls to the [.noloc]`Kubernetes` API server with `kubectl` or the {aws-management-console}. If you want other IAM principals to have access to your cluster, then you need to add them. For more information, see <> and <>. +* The link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] that created the cluster is the only principal that can make calls to the Kubernetes API server with `kubectl` or the {aws-management-console}. If you want other IAM principals to have access to your cluster, then you need to add them. For more information, see <> and <>. * Deploy a <> to your cluster. * Before deploying a cluster for production use, we recommend familiarizing yourself with all of the settings for <> and <>. Some settings (such as enabling SSH access to Amazon EC2 nodes) must be made when the cluster is created. * To increase security for your cluster, <>. diff --git a/latest/ug/getting-started/getting-started-eksctl.adoc b/latest/ug/getting-started/getting-started-eksctl.adoc index 4b05e9744..3402a4e5d 100644 --- a/latest/ug/getting-started/getting-started-eksctl.adoc +++ b/latest/ug/getting-started/getting-started-eksctl.adoc @@ -1,16 +1,9 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[getting-started-eksctl,getting-started-eksctl.title]] +[#getting-started-eksctl] = Get started with Amazon EKS – `eksctl` -:info_doctype: section -:info_title: Get started with Amazon EKS – \ - eksctl -:info_titleabbrev: Create your first cluster – eksctl -:keywords: using, eksctl, getting, started, tutorial -:info_abstract: Learn how to create your first Amazon EKS cluster with nodes using the eksctl command \ - line tool. +:info_titleabbrev: Create cluster (eksctl) [abstract] -- @@ -21,20 +14,20 @@ Learn how to create your first Amazon EKS cluster with nodes using the `eksctl` ==== This topic covers getting started *without* EKS Auto Mode. -EKS Auto Mode automates routine tasks for cluster compute, storage, and networking. xref:getting-started-automode[Learn how to get started with Amazon EKS Auto Mode. ] +EKS Auto Mode automates routine tasks for cluster compute, storage, and networking. <> ==== -This guide helps you to create all of the required resources to get started with Amazon Elastic Kubernetes Service (Amazon EKS) using `eksctl`, a simple command line utility for creating and managing [.noloc]`Kubernetes` clusters on Amazon EKS. At the end of this tutorial, you will have a running Amazon EKS cluster that you can deploy applications to. +This guide helps you to create all of the required resources to get started with Amazon Elastic Kubernetes Service (Amazon EKS) using `eksctl`, a simple command line utility for creating and managing Kubernetes clusters on Amazon EKS. At the end of this tutorial, you will have a running Amazon EKS cluster that you can deploy applications to. The procedures in this guide create several resources for you automatically that you have to create manually when you create your cluster using the {aws-management-console}. If you'd rather manually create most of the resources to better understand how they interact with each other, then use the {aws-management-console} to create your cluster and compute. For more information, see <>. -[[eksctl-prereqs,eksctl-prereqs.title]] +[#eksctl-prereqs] == Prerequisites -Before starting this tutorial, you must install and configure the {aws} CLI, kubectl, and eksctl tools as described in <>. +Before starting this tutorial, you must install and configure the {aws} CLI, kubectl, and eksctl tools as described in <>. -[[create-cluster-gs-eksctl,create-cluster-gs-eksctl.title]] +[#create-cluster-gs-eksctl] == Step 1: Create your Amazon EKS cluster and nodes [IMPORTANT] @@ -47,21 +40,21 @@ To get started as simply and quickly as possible, this topic includes steps to c You can create a cluster with one of the following node types. To learn more about each type, see <>. After your cluster is deployed, you can add other node types. -* *Fargate – [.noloc]``Linux``* – Select this type of node if you want to run [.noloc]``Linux`` applications on <>. Fargate is a serverless compute engine that lets you deploy [.noloc]``Kubernetes`` [.noloc]``Pods`` without managing Amazon EC2 instances. -* *Managed nodes – [.noloc]``Linux``* – Select this type of node if you want to run Amazon Linux applications on Amazon EC2 instances. Though not covered in this guide, you can also add <> and <> nodes to your cluster. +* *Fargate – Linux* – Select this type of node if you want to run Linux applications on <>. Fargate is a serverless compute engine that lets you deploy Kubernetes Pods without managing Amazon EC2 instances. +* *Managed nodes – Linux* – Select this type of node if you want to run Amazon Linux applications on Amazon EC2 instances. Though not covered in this guide, you can also add <> and <> nodes to your cluster. Create your Amazon EKS cluster with the following command. You can replace [.replaceable]`my-cluster` with your own value. The name can contain only alphanumeric characters (case-sensitive) and hyphens. It must start with an alphanumeric character and can't be longer than 100 characters. The name must be unique within the {aws} Region and {aws} account that you're creating the cluster in. Replace [.replaceable]`region-code` with any {aws} Region that is supported by Amazon EKS. For a list of {aws} Regions, see link:general/latest/gr/eks.html[Amazon EKS endpoints and quotas,type="documentation"] in the {aws} General Reference guide. ==== [role="tablist"] -Fargate - [.noloc]`Linux`:: +Fargate - Linux:: + [source,bash,subs="verbatim,attributes"] ---- eksctl create cluster --name my-cluster --region region-code --fargate ---- -Managed nodes - [.noloc]`Linux`:: +Managed nodes - Linux:: + [source,bash,subs="verbatim,attributes"] ---- @@ -74,17 +67,17 @@ Cluster creation takes several minutes. During creation you'll see several lines ---- [...] -[✓] EKS cluster "my-cluster" in "`region-code`" region is ready +[✓] EKS cluster "my-cluster" in "region-code" region is ready ---- `eksctl` created a `kubectl` config file in `~/.kube/config` or added the new cluster's configuration within an existing config file in `~/.kube/config` on your computer. After cluster creation is complete, view the {aws} CloudFormation stack named `eksctl-[.replaceable]``my-cluster``-cluster` in the {aws} CloudFormation link:cloudformation/[console,type="console"] to see all of the resources that were created. -[[gs-eksctl-view-resources,gs-eksctl-view-resources.title]] +[#gs-eksctl-view-resources] -== Step 2: View [.noloc]`Kubernetes` resources +== Step 2: View Kubernetes resources . View your cluster nodes. + [source,bash,subs="verbatim,attributes"] @@ -96,7 +89,7 @@ An example output is as follows. + ==== [role="tablist"] -Fargate - [.noloc]`Linux`:: +Fargate - Linux:: + [source,none,subs="verbatim,attributes"] ---- @@ -105,7 +98,7 @@ fargate-ip-192-0-2-0.region-code.compute.internal Ready 8m3s v1 fargate-ip-192-0-2-1.region-code.compute.internal Ready 7m30s v1.2.3-eks-1234567 192-0-2-1 Amazon Linux 2 1.23.456-789.012.amzn2.x86_64 containerd://1.2.3 ---- -Managed nodes - [.noloc]`Linux`:: +Managed nodes - Linux:: + [source,none,subs="verbatim,attributes"] ---- @@ -115,7 +108,7 @@ ip-192-0-2-1.region-code.compute.internal Ready 6m4s v1.2.3-eks- ---- ==== + -For more information about what you see in the output, see <>. +For more information about what you see in the output, see <>. . View the workloads running on your cluster. + @@ -128,7 +121,7 @@ An example output is as follows. + ==== [role="tablist"] -Fargate - [.noloc]`Linux`:: +Fargate - Linux:: + [source,none,subs="verbatim,attributes"] ---- @@ -137,7 +130,7 @@ kube-system coredns-1234567890-abcde 1/1 Running 0 18m 192. kube-system coredns-1234567890-12345 1/1 Running 0 18m 192.0.2.1 fargate-ip-192-0-2-1.region-code.compute.internal ---- -Managed nodes - [.noloc]`Linux`:: +Managed nodes - Linux:: + [source,none,subs="verbatim,attributes"] ---- @@ -151,10 +144,10 @@ kube-system kube-proxy-67890 1/1 Running 0 7m43s 19 ---- ==== + -For more information about what you see in the output, see <>. +For more information about what you see in the output, see <>. -[[gs-eksctl-clean-up,gs-eksctl-clean-up.title]] +[#gs-eksctl-clean-up] == Step 3: Delete your cluster and nodes @@ -166,7 +159,7 @@ eksctl delete cluster --name my-cluster --region region-code ---- -[[gs-eksctl-next-steps,gs-eksctl-next-steps.title]] +[#gs-eksctl-next-steps] == Next steps The following documentation topics help you to extend the functionality of your cluster. @@ -174,6 +167,6 @@ The following documentation topics help you to extend the functionality of your * Deploy a <> to your cluster. -* The link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] that created the cluster is the only principal that can make calls to the [.noloc]`Kubernetes` API server with `kubectl` or the {aws-management-console}. If you want other IAM principals to have access to your cluster, then you need to add them. For more information, see <> and <>. +* The link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] that created the cluster is the only principal that can make calls to the Kubernetes API server with `kubectl` or the {aws-management-console}. If you want other IAM principals to have access to your cluster, then you need to add them. For more information, see <> and <>. * Before deploying a cluster for production use, we recommend familiarizing yourself with all of the settings for <> and <>. Some settings (such as enabling SSH access to Amazon EC2 nodes) must be made when the cluster is created. -* To increase security for your cluster, <>. +* To increase security for your cluster, <>. \ No newline at end of file diff --git a/latest/ug/getting-started/getting-started.adoc b/latest/ug/getting-started/getting-started.adoc index ae1fc70f3..c9421da33 100644 --- a/latest/ug/getting-started/getting-started.adoc +++ b/latest/ug/getting-started/getting-started.adoc @@ -1,20 +1,8 @@ -//!!NODE_ROOT include::../attributes.txt[] -[[getting-started,getting-started.title]] + +[#getting-started] = Get started with Amazon EKS -:doctype: book -:sectnums: -:toc: left -:icons: font -:experimental: -:idprefix: -:idseparator: - -:sourcedir: . -:info_doctype: chapter -:info_title: Get started with Amazon EKS :info_titleabbrev: Get started -:info_abstract: Learn about the tools needed for creating and working with an Amazon EKS cluster. -:keywords: getting, started, tutorials, quick, start [abstract] -- @@ -23,11 +11,11 @@ Learn about the tools needed for creating and working with an Amazon EKS cluster Make sure that you are set up to use Amazon EKS before going through the getting started guides. For more information, see <>. -There are two getting started guides available for creating a new [.noloc]`Kubernetes` cluster with nodes in Amazon EKS: +There are two getting started guides available for creating a new Kubernetes cluster with nodes in Amazon EKS: -* <> – This getting started guide helps you to install all of the required resources to get started with Amazon EKS using `eksctl`, a simple command line utility for creating and managing [.noloc]`Kubernetes` clusters on Amazon EKS. At the end of the tutorial, you will have a running Amazon EKS cluster that you can deploy applications to. This is the fastest and simplest way to get started with Amazon EKS. +* <> – This getting started guide helps you to install all of the required resources to get started with Amazon EKS using `eksctl`, a simple command line utility for creating and managing Kubernetes clusters on Amazon EKS. At the end of the tutorial, you will have a running Amazon EKS cluster that you can deploy applications to. This is the fastest and simplest way to get started with Amazon EKS. * <> – This getting started guide helps you to create all of the required resources to get started with Amazon EKS using the {aws-management-console} and {aws} CLI. At the end of the tutorial, you will have a running Amazon EKS cluster that you can deploy applications to. In this guide, you manually create each resource required for an Amazon EKS cluster. The procedures give you visibility into how each resource is created and how they interact with each other. We also offer the following references: @@ -35,11 +23,11 @@ We also offer the following references: * For a collection of hands-on tutorials, see https://community.aws/tags/eks-cluster-setup[EKS Cluster Setup] on _{aws} Community_. -* For code examples, see link:code-library/latest/ug/eks_code_examples.html[Code examples for Amazon EKS using {aws} SDKs,type="documentation"]. +* For code examples, see link:code-library/latest/ug/eks_code_examples.html[Code examples for Amazon EKS using {aws} SDKs,type="documentation"]. include::getting-started-automode.adoc[leveloffset=+1] include::getting-started-eksctl.adoc[leveloffset=+1] -include::getting-started-console.adoc[leveloffset=+1] +include::getting-started-console.adoc[leveloffset=+1] \ No newline at end of file diff --git a/latest/ug/getting-started/images b/latest/ug/getting-started/images new file mode 120000 index 000000000..5e6757319 --- /dev/null +++ b/latest/ug/getting-started/images @@ -0,0 +1 @@ +../images \ No newline at end of file diff --git a/latest/ug/getting-started/install-awscli.adoc b/latest/ug/getting-started/install-awscli.adoc index 08e3a9cf8..f367b1d10 100644 --- a/latest/ug/getting-started/install-awscli.adoc +++ b/latest/ug/getting-started/install-awscli.adoc @@ -1,15 +1,9 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[install-awscli,install-awscli.title]] +[#install-awscli] = Set up {aws} CLI -:info_doctype: section -:info_title: Set up {aws} CLI :info_titleabbrev: Set up {aws} CLI -:keywords: setting up, setup -:info_abstract: Set up the {aws} CLI for managing {aws} resources needed to use Amazon EKS. Follow these \ - instructions to set up the credentials with {aws} CLI. [abstract] -- @@ -18,23 +12,23 @@ Set up the {aws} CLI for managing {aws} resources needed to use Amazon EKS. Foll The link:cli/[{aws} CLI,type="marketing"] is a command line tool for working with {aws} services, including Amazon EKS. It is also used to authenticate IAM users or roles for access to the Amazon EKS cluster and other {aws} resources from your local machine. To provision resources in {aws} from the command line, you need to obtain an {aws} access key ID and secret key to use in the command line. Then you need to configure these credentials in the {aws} CLI. If you haven't already installed the {aws} CLI, see link:cli/latest/userguide/cli-chap-install.html[Install or update the latest version of the {aws} CLI,type="documentation"] in the _{aws} Command Line Interface User Guide_. -[[create-access-key,create-access-key.title]] +[#create-access-key] == To create an access key . Sign into the https://console.aws.amazon.com/[{aws-management-console}]. . For single-user or multiple-user accounts: + ** *Single-user account –*:: - In the top right, choose your {aws} user name to open the navigation menu. For example, choose *`webadmin`*. + In the top right, choose your {aws} user name to open the navigation menu. For example, choose *`webadmin`*. ** *Multiple-user account –*:: - Choose IAM from the list of services. From the IAM Dashboard, select *Users*, and choose the name of the user. + Choose IAM from the list of services. From the IAM Dashboard, select *Users*, and choose the name of the user. . Choose *Security credentials*. -. Under *Access keys*, choose *Create access key*. -. Choose *Command Line Interface (CLI)*, then choose *Next*. +. Under *Access keys*, choose *Create access key*. +. Choose *Command Line Interface (CLI)*, then choose *Next*. . Choose *Create access key*. . Choose *Download .csv file*. -[[configure-cli,configure-cli.title]] +[#configure-cli] == To configure the {aws} CLI After installing the {aws} CLI, do the following steps to configure it. For more information, see link:cli/latest/userguide/cli-chap-configure.html[Configure the {aws} CLI,type="documentation"] in the _{aws} Command Line Interface User Guide_. @@ -58,7 +52,7 @@ Default output format [None]: json ---- -[[security-token,security-token.title]] +[#security-token] == To get a security token If needed, run the following command to get a new security token for the {aws} CLI. For more information, see link:cli/latest/reference/sts/get-session-token.html[get-session-token,type="documentation"] in the _{aws} CLI Command Reference_. @@ -85,7 +79,7 @@ This command returns the temporary security credentials for an {aws} CLI session ---- -[[verify-identity,verify-identity.title]] +[#verify-identity] == To verify the user identity If needed, run the following command to verify the {aws} credentials for your IAM user identity (such as [.replaceable]`ClusterAdmin`) for the terminal session. @@ -107,8 +101,8 @@ This command returns the Amazon Resource Name (ARN) of the IAM entity that's con ---- -[[install-awscli-next-steps,install-awscli-next-steps.title]] +[#install-awscli-next-steps] == Next steps * <> -* <> +* <> \ No newline at end of file diff --git a/latest/ug/getting-started/install-kubectl.adoc b/latest/ug/getting-started/install-kubectl.adoc index 384eb84c5..a35ab4983 100644 --- a/latest/ug/getting-started/install-kubectl.adoc +++ b/latest/ug/getting-started/install-kubectl.adoc @@ -1,35 +1,23 @@ -//!!NODE_ROOT
- +include::../attributes.txt[] [.topic] -[[install-kubectl,install-kubectl.title]] +[#install-kubectl] = Set up `kubectl` and `eksctl` -:info_doctype: section -:info_title: Set up kubectl and eksctl :info_titleabbrev: Set up kubectl and eksctl -:keywords: install, update, kubectl -:info_abstract: Learn how to install or update the kubectl and eksctl command line tools \ - to work with Kubernetes and Amazon EKS features. - - -include::../attributes.txt[] [abstract] -- -Learn how to install or update the `kubectl` and `eksctl` command line tools to work with [.noloc]`Kubernetes` and Amazon EKS features. +Learn how to install or update the `kubectl` and `eksctl` command line tools to work with Kubernetes and Amazon EKS features. -- -`Kubectl` is a command line tool that you use to communicate with the [.noloc]`Kubernetes` API server. The `kubectl` binary is available in many operating system package managers. Using a package manager for your installation is often easier than a manual download and install process. The `eksctl` command lets you create and modify Amazon EKS clusters. +Once the {aws} CLI is installed, there are two other tools you should install to create and manage your Kubernetes clusters: -Topics on this page help you install and set up these tools: +* `kubectl`: The `kubectl` command line tool is the main tool you will use to manage resources within your Kubernetes cluster. This page describes how to download and set up the `kubectl` binary that matches the version of your Kubernetes cluster. See <>. +* `eksctl`: The `eksctl` command line tool is made for creating EKS clusters in the {aws} cloud or on-premises (with EKS Anywhere), as well as modifying and deleting those clusters. See <>. -* <> -* <> - - -[[kubectl-install-update,kubectl-install-update.title]] +[#kubectl-install-update] == Install or update `kubectl` This topic helps you to download and install, or update, the `kubectl` binary on your device. The binary is identical to the https://kubernetes.io/docs/tasks/tools/#kubectl[upstream community versions]. The binary is not unique to Amazon EKS or {aws}. Use the steps below to get the specific version of `kubectl` that you need, although many builders simply run `brew install kubectl` to install it. @@ -37,7 +25,7 @@ This topic helps you to download and install, or update, the `kubectl` binary on [NOTE] ==== -You must use a `kubectl` version that is within one minor version difference of your Amazon EKS cluster control plane. For example, a `1.30` `kubectl` client works with [.noloc]`Kubernetes` `1.29`, `1.30`, and `1.31` clusters. +You must use a `kubectl` version that is within one minor version difference of your Amazon EKS cluster control plane. For example, a `{k8s-n-1}` `kubectl` client works with Kubernetes `{k8s-n-2}`, `{k8s-n-1}`, and `{k8s-n}` clusters. ==== @@ -68,157 +56,141 @@ Install or update `kubectl` on one of the following operating systems: * <> * <> +[NOTE] +==== + +If downloads are slow to your {aws} Region from the {aws} Regions used in this section, +consider setting up CloudFront to front the content. +For further information, see link:AmazonCloudFront/latest/DeveloperGuide/GettingStartedSimpleDistributon.html[Get started with a basic CloudFront distribution,type="documentation"]. + +==== + + === macOS [[macos_kubectl]] -. Download the binary for your cluster's [.noloc]`Kubernetes` version from Amazon S3. -+ -**** [.noloc]`Kubernetes` `1.32` -+ -[source,bash,subs="verbatim,attributes"] ----- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.32.0/2024-12-20/bin/darwin/amd64/kubectl ----- -**** [.noloc]`Kubernetes` `1.31` +Follow the steps below to install `kubectl` on macOS. The steps include: + +* Choosing and downloading the binary for the Kubernetes version you want. +* Optionally checking the binary's checksum. +* Adding execute to the binary's permissions. +* Copying the binary to a folder in your PATH. +* Optionally adding the binary's directory to your PATH. + +Procedure: + +. Download the binary for your cluster's Kubernetes version from Amazon S3. + -[source,bash,subs="verbatim,attributes"] ----- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.31.3/2024-12-12/bin/darwin/amd64/kubectl ----- -**** [.noloc]`Kubernetes` `1.30` +**** Kubernetes `1.34` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.30.7/2024-12-12/bin/darwin/amd64/kubectl +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.34.1/2025-09-19/bin/darwin/amd64/kubectl ---- -**** [.noloc]`Kubernetes` `1.29` +**** Kubernetes `1.33` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.29.10/2024-12-12/bin/darwin/amd64/kubectl +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.33.5/2025-09-19/bin/darwin/amd64/kubectl ---- -**** [.noloc]`Kubernetes` `1.28` +**** Kubernetes `1.32` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.28.15/2024-12-12/bin/darwin/amd64/kubectl +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.32.9/2025-09-19/bin/darwin/amd64/kubectl ---- -**** [.noloc]`Kubernetes` `1.27` +**** Kubernetes `1.31` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.27.16/2024-12-12/bin/darwin/amd64/kubectl +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.31.13/2025-09-19/bin/darwin/amd64/kubectl ---- -**** [.noloc]`Kubernetes` `1.26` +**** Kubernetes `1.30` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.26.15/2024-12-12/bin/darwin/amd64/kubectl +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.30.14/2025-09-19/bin/darwin/amd64/kubectl ---- -**** [.noloc]`Kubernetes` `1.25` +**** Kubernetes `1.29` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.25.16/2024-12-12/bin/darwin/amd64/kubectl +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.29.15/2025-09-19/bin/darwin/amd64/kubectl ---- -**** [.noloc]`Kubernetes` `1.24` +**** Kubernetes `1.28` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.24.17/2024-12-12/bin/darwin/amd64/kubectl +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.28.15/2025-09-19/bin/darwin/amd64/kubectl ---- -**** [.noloc]`Kubernetes` `1.23` +**** Kubernetes `1.27` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.23.17/2024-09-11/bin/darwin/amd64/kubectl +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.27.16/2025-01-10/bin/darwin/amd64/kubectl ---- -**** [.noloc]`Kubernetes` `1.22` +**** Kubernetes `1.26` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.22.17/2024-09-11/bin/darwin/amd64/kubectl ----- -**** [.noloc]`Kubernetes` `1.21` -+ -[source,bash,subs="verbatim,attributes"] ----- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.21.14/2024-09-11/bin/darwin/amd64/kubectl +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.26.15/2024-12-12/bin/darwin/amd64/kubectl ---- . (Optional) Verify the downloaded binary with the `SHA-256` checksum for your binary. + -.. Download the `SHA-256` checksum for your cluster's [.noloc]`Kubernetes` version. -+ -***** [.noloc]`Kubernetes` `1.32` -+ -[source,bash,subs="verbatim,attributes"] ----- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.32.0/2024-12-20/bin/darwin/amd64/kubectl.sha256 ----- -***** [.noloc]`Kubernetes` `1.31` -+ -[source,bash,subs="verbatim,attributes"] ----- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.31.3/2024-12-12/bin/darwin/amd64/kubectl.sha256 ----- -***** [.noloc]`Kubernetes` `1.30` +.. Download the `SHA-256` checksum for your cluster's Kubernetes version. + -[source,bash,subs="verbatim,attributes"] ----- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.30.7/2024-12-12/bin/darwin/amd64/kubectl.sha256 ----- -***** [.noloc]`Kubernetes` `1.29` +***** Kubernetes `1.34` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.29.10/2024-12-12/bin/darwin/amd64/kubectl.sha256 +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.34.1/2025-09-19/bin/darwin/amd64/kubectl.sha256 ---- -***** [.noloc]`Kubernetes` `1.28` +***** Kubernetes `1.33` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.28.15/2024-12-12/bin/darwin/amd64/kubectl.sha256 +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.33.5/2025-09-19/bin/darwin/amd64/kubectl.sha256 ---- -***** [.noloc]`Kubernetes` `1.27` +***** Kubernetes `1.32` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.27.16/2024-12-12/bin/darwin/amd64/kubectl.sha256 +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.32.9/2025-09-19/bin/darwin/amd64/kubectl.sha256 ---- -***** [.noloc]`Kubernetes` `1.26` +***** Kubernetes `1.31` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.26.15/2024-12-12/bin/darwin/amd64/kubectl.sha256 +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.31.13/2025-09-19/bin/darwin/amd64/kubectl.sha256 ---- -***** [.noloc]`Kubernetes` `1.25` +***** Kubernetes `1.30` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.25.16/2024-12-12/bin/darwin/amd64/kubectl.sha256 +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.30.14/2025-09-19/bin/darwin/amd64/kubectl.sha256 ---- -***** [.noloc]`Kubernetes` `1.24` +***** Kubernetes `1.29` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.24.17/2024-12-12/bin/darwin/amd64/kubectl.sha256 +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.29.15/2025-09-19/bin/darwin/amd64/kubectl.sha256 ---- -***** [.noloc]`Kubernetes` `1.23` +***** Kubernetes `1.28` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.23.17/2024-09-11/bin/darwin/amd64/kubectl.sha256 +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.28.15/2025-09-19/bin/darwin/amd64/kubectl.sha256 ---- -***** [.noloc]`Kubernetes` `1.22` +***** Kubernetes `1.27` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.22.17/2024-09-11/bin/darwin/amd64/kubectl.sha256 +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.27.16/2025-01-10/bin/darwin/amd64/kubectl.sha256 ---- -***** [.noloc]`Kubernetes` `1.21` +***** Kubernetes `1.26` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.21.14/2024-09-11/bin/darwin/amd64/kubectl.sha256 +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.26.15/2024-12-12/bin/darwin/amd64/kubectl.sha256 ---- .. Check the `SHA-256` checksum for your downloaded binary. + @@ -248,155 +220,129 @@ echo 'export PATH=$HOME/bin:$PATH' >> ~/.bash_profile === Linux (amd64) [[linux_amd64_kubectl]] -. Download the `kubectl` binary for your cluster's [.noloc]`Kubernetes` version from Amazon S3. -+ -**** [.noloc]`Kubernetes` `1.32` -+ -[source,bash,subs="verbatim,attributes"] ----- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.32.0/2024-12-20/bin/linux/amd64/kubectl ----- -**** [.noloc]`Kubernetes` `1.31` -+ -[source,bash,subs="verbatim,attributes"] ----- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.31.3/2024-12-12/bin/linux/amd64/kubectl ----- -**** [.noloc]`Kubernetes` `1.30` +Follow the steps below to install `kubectl` on Linux (amd64). The steps include: + +* Choosing and downloading the binary for the Kubernetes version you want. +* Optionally checking the binary's checksum. +* Adding execute to the binary's permissions. +* Copying the binary to a folder in your PATH. +* Optionally adding the binary's directory to your PATH. + +Procedure: + +. Download the `kubectl` binary for your cluster's Kubernetes version from Amazon S3. + -[source,bash,subs="verbatim,attributes"] ----- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.30.7/2024-12-12/bin/linux/amd64/kubectl ----- -**** [.noloc]`Kubernetes` `1.29` +**** Kubernetes `1.34` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.29.10/2024-12-12/bin/linux/amd64/kubectl +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.34.1/2025-09-19/bin/linux/amd64/kubectl ---- -**** [.noloc]`Kubernetes` `1.28` +**** Kubernetes `1.33` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.28.15/2024-12-12/bin/linux/amd64/kubectl +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.33.5/2025-09-19/bin/linux/amd64/kubectl ---- -**** [.noloc]`Kubernetes` `1.27` +**** Kubernetes `1.32` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.27.16/2024-12-12/bin/linux/amd64/kubectl +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.32.9/2025-09-19/bin/linux/amd64/kubectl ---- -**** [.noloc]`Kubernetes` `1.26` +**** Kubernetes `1.31` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.26.15/2024-12-12/bin/linux/amd64/kubectl +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.31.13/2025-09-19/bin/linux/amd64/kubectl ---- -**** [.noloc]`Kubernetes` `1.25` +**** Kubernetes `1.30` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.25.16/2024-12-12/bin/linux/amd64/kubectl +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.30.14/2025-09-19/bin/linux/amd64/kubectl ---- -**** [.noloc]`Kubernetes` `1.24` +**** Kubernetes `1.29` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.24.17/2024-12-12/bin/linux/amd64/kubectl +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.29.15/2025-09-19/bin/linux/amd64/kubectl ---- -**** [.noloc]`Kubernetes` `1.23` +**** Kubernetes `1.28` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.23.17/2024-09-11/bin/linux/amd64/kubectl +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.28.15/2025-09-19/bin/linux/amd64/kubectl ---- -**** [.noloc]`Kubernetes` `1.22` +**** Kubernetes `1.27` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.22.17/2024-09-11/bin/linux/amd64/kubectl +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.27.16/2024-12-12/bin/linux/amd64/kubectl ---- -**** [.noloc]`Kubernetes` `1.21` +**** Kubernetes `1.26` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.21.14/2024-09-11/bin/linux/amd64/kubectl +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.26.15/2024-12-12/bin/linux/amd64/kubectl ---- . (Optional) Verify the downloaded binary with the `SHA-256` checksum for your binary. + -.. Download the `SHA-256` checksum for your cluster's [.noloc]`Kubernetes` version from Amazon S3using the command for your device's hardware platform. -+ -***** [.noloc]`Kubernetes` `1.32` +.. Download the `SHA-256` checksum for your cluster's Kubernetes version from Amazon S3using the command for your device's hardware platform. + -[source,bash,subs="verbatim,attributes"] ----- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.32.0/2024-12-20/bin/linux/amd64/kubectl.sha256 ----- -***** [.noloc]`Kubernetes` `1.31` -+ -[source,bash,subs="verbatim,attributes"] ----- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.31.3/2024-12-12/bin/linux/amd64/kubectl.sha256 ----- -***** [.noloc]`Kubernetes` `1.30` -+ -[source,bash,subs="verbatim,attributes"] ----- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.30.7/2024-12-12/bin/linux/amd64/kubectl.sha256 ----- -***** [.noloc]`Kubernetes` `1.29` +***** Kubernetes `1.34` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.29.10/2024-12-12/bin/linux/amd64/kubectl.sha256 +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.34.1/2025-09-19/bin/linux/amd64/kubectl.sha256 ---- -***** [.noloc]`Kubernetes` `1.28` +***** Kubernetes `1.33` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.28.15/2024-12-12/bin/linux/amd64/kubectl.sha256 +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.33.5/2025-09-19/bin/linux/amd64/kubectl.sha256 ---- -***** [.noloc]`Kubernetes` `1.27` +***** Kubernetes `1.32` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.27.16/2024-12-12/bin/linux/amd64/kubectl.sha256 +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.32.9/2025-09-19/bin/linux/amd64/kubectl.sha256 ---- -***** [.noloc]`Kubernetes` `1.26` +***** Kubernetes `1.31` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.26.15/2024-12-12/bin/linux/amd64/kubectl.sha256 +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.31.13/2025-09-19/bin/linux/amd64/kubectl.sha256 ---- -***** [.noloc]`Kubernetes` `1.25` +***** Kubernetes `1.30` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.25.16/2024-12-12/bin/linux/amd64/kubectl.sha256 +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.30.14/2025-09-19/bin/linux/amd64/kubectl.sha256 ---- -***** [.noloc]`Kubernetes` `1.24` +***** Kubernetes `1.29` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.24.17/2024-12-12/bin/linux/amd64/kubectl.sha256 +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.29.15/2025-09-19/bin/linux/amd64/kubectl.sha256 ---- -***** [.noloc]`Kubernetes` `1.23` +***** Kubernetes `1.28` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.23.17/2024-09-11/bin/linux/amd64/kubectl.sha256 +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.28.15/2025-09-19/bin/linux/amd64/kubectl.sha256 ---- -***** [.noloc]`Kubernetes` `1.22` +***** Kubernetes `1.27` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.22.17/2024-09-11/bin/linux/amd64/kubectl.sha256 +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.27.16/2024-12-12/bin/linux/amd64/kubectl.sha256 ---- -***** [.noloc]`Kubernetes` `1.21` +***** Kubernetes `1.26` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.21.14/2024-09-11/bin/linux/amd64/kubectl.sha256 +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.26.15/2024-12-12/bin/linux/amd64/kubectl.sha256 ---- .. Check the `SHA-256` checksum for your downloaded binary with one of the following commands. + @@ -406,6 +352,7 @@ sha256sum -c kubectl.sha256 ---- or + +[source,bash,subs="verbatim,attributes"] ---- openssl sha1 -sha256 kubectl ---- @@ -433,155 +380,129 @@ echo 'export PATH=$HOME/bin:$PATH' >> ~/.bashrc === Linux (arm64) [[linux_arm64_kubectl]] -. Download the `kubectl` binary for your cluster's [.noloc]`Kubernetes` version from Amazon S3. -+ -**** [.noloc]`Kubernetes` `1.32` -+ -[source,bash,subs="verbatim,attributes"] ----- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.32.0/2024-12-20/bin/linux/arm64/kubectl ----- -**** [.noloc]`Kubernetes` `1.31` -+ -[source,bash,subs="verbatim,attributes"] ----- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.31.3/2024-12-12/bin/linux/arm64/kubectl ----- -**** [.noloc]`Kubernetes` `1.30` +Follow the steps below to install `kubectl` on Linux (arm64). The steps include: + +* Choosing and downloading the binary for the Kubernetes version you want. +* Optionally checking the binary's checksum. +* Adding execute to the binary's permissions. +* Copying the binary to a folder in your PATH. +* Optionally adding the binary's directory to your PATH. + +Procedure: + +. Download the `kubectl` binary for your cluster's Kubernetes version from Amazon S3. + -[source,bash,subs="verbatim,attributes"] ----- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.30.7/2024-12-12/bin/linux/arm64/kubectl ----- -**** [.noloc]`Kubernetes` `1.29` +**** Kubernetes `1.34` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.29.10/2024-12-12/bin/linux/arm64/kubectl +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.34.1/2025-09-19/bin/linux/arm64/kubectl ---- -**** [.noloc]`Kubernetes` `1.28` +**** Kubernetes `1.33` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.28.15/2024-12-12/bin/linux/arm64/kubectl +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.33.5/2025-09-19/bin/linux/arm64/kubectl ---- -**** [.noloc]`Kubernetes` `1.27` +**** Kubernetes `1.32` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.27.16/2024-12-12/bin/linux/arm64/kubectl +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.32.9/2025-09-19/bin/linux/arm64/kubectl ---- -**** [.noloc]`Kubernetes` `1.26` +**** Kubernetes `1.31` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.26.15/2024-12-12/bin/linux/arm64/kubectl +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.31.13/2025-09-19/bin/linux/arm64/kubectl ---- -**** [.noloc]`Kubernetes` `1.25` +**** Kubernetes `1.30` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.25.16/2024-12-12/bin/linux/arm64/kubectl +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.30.14/2025-09-19/bin/linux/arm64/kubectl ---- -**** [.noloc]`Kubernetes` `1.24` +**** Kubernetes `1.29` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.24.17/2024-12-12/bin/linux/arm64/kubectl +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.29.15/2025-09-19/bin/linux/arm64/kubectl ---- -**** [.noloc]`Kubernetes` `1.23` +**** Kubernetes `1.28` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.23.17/2024-09-11/bin/linux/arm64/kubectl +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.28.15/2025-09-19/bin/linux/arm64/kubectl ---- -**** [.noloc]`Kubernetes` `1.22` +**** Kubernetes `1.27` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.22.17/2024-09-11/bin/linux/arm64/kubectl +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.27.16/2024-12-12/bin/linux/arm64/kubectl ---- -**** [.noloc]`Kubernetes` `1.21` +**** Kubernetes `1.26` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.21.14/2024-09-11/bin/linux/arm64/kubectl +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.26.15/2024-12-12/bin/linux/arm64/kubectl ---- . (Optional) Verify the downloaded binary with the `SHA-256` checksum for your binary. + -.. Download the `SHA-256` checksum for your cluster's [.noloc]`Kubernetes` version from Amazon S3using the command for your device's hardware platform. +.. Download the `SHA-256` checksum for your cluster's Kubernetes version from Amazon S3using the command for your device's hardware platform. + -***** [.noloc]`Kubernetes` `1.32` +***** Kubernetes `1.34` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.32.0/2024-12-20/bin/linux/arm64/kubectl.sha256 +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.34.1/2025-09-19/bin/linux/arm64/kubectl.sha256 ---- -***** [.noloc]`Kubernetes` `1.31` +***** Kubernetes `1.33` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.31.3/2024-12-12/bin/linux/arm64/kubectl.sha256 +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.33.5/2025-09-19/bin/linux/arm64/kubectl.sha256 ---- -***** [.noloc]`Kubernetes` `1.30` +***** Kubernetes `1.32` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.30.7/2024-12-12/bin/linux/arm64/kubectl.sha256 +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.32.9/2025-09-19/bin/linux/arm64/kubectl.sha256 ---- -***** [.noloc]`Kubernetes` `1.29` +***** Kubernetes `1.31` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.29.10/2024-12-12/bin/linux/arm64/kubectl.sha256 ----- -***** [.noloc]`Kubernetes` `1.28` -+ -[source,bash,subs="verbatim,attributes"] ----- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.28.15/2024-12-12/bin/linux/arm64/kubectl.sha256 ----- -***** [.noloc]`Kubernetes` `1.27` -+ -[source,bash,subs="verbatim,attributes"] ----- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.27.16/2024-12-12/bin/linux/arm64/kubectl.sha256 ----- -***** [.noloc]`Kubernetes` `1.26` -+ -[source,bash,subs="verbatim,attributes"] ----- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.26.15/2024-12-12/bin/linux/arm64/kubectl.sha256 +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.31.13/2025-09-19/bin/linux/arm64/kubectl.sha256 ---- -***** [.noloc]`Kubernetes` `1.25` +***** Kubernetes `1.30` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.25.16/2024-12-12/bin/linux/arm64/kubectl.sha256 +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.30.14/2025-09-19/bin/linux/arm64/kubectl.sha256 ---- -***** [.noloc]`Kubernetes` `1.24` +***** Kubernetes `1.29` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.24.17/2024-12-12/bin/linux/arm64/kubectl.sha256 +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.29.15/2025-09-19/bin/linux/arm64/kubectl.sha256 ---- -***** [.noloc]`Kubernetes` `1.23` +***** Kubernetes `1.28` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.23.17/2024-09-11/bin/linux/arm64/kubectl.sha256 +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.28.15/2025-09-19/bin/linux/arm64/kubectl.sha256 ---- -***** [.noloc]`Kubernetes` `1.22` +***** Kubernetes `1.27` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.22.17/2024-09-11/bin/linux/arm64/kubectl.sha256 +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.27.16/2024-12-12/bin/linux/arm64/kubectl.sha256 ---- -***** [.noloc]`Kubernetes` `1.21` +***** Kubernetes `1.26` + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.21.14/2024-09-11/bin/linux/arm64/kubectl.sha256 +curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.26.15/2024-12-12/bin/linux/arm64/kubectl.sha256 ---- .. Check the `SHA-256` checksum for your downloaded binary with one of the following commands. + @@ -591,6 +512,7 @@ sha256sum -c kubectl.sha256 ---- or + +[source,bash,subs="verbatim,attributes"] ---- openssl sha1 -sha256 kubectl ---- @@ -618,156 +540,129 @@ echo 'export PATH=$HOME/bin:$PATH' >> ~/.bashrc === Windows [[windows_kubectl]] -. Open a [.noloc]`PowerShell` terminal. -. Download the `kubectl` binary for your cluster's [.noloc]`Kubernetes` version from Amazon S3. -+ -**** [.noloc]`Kubernetes` `1.32` -+ -[source,bash,subs="verbatim,attributes"] ----- -curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.32.0/2024-12-20/bin/windows/amd64/kubectl.exe ----- -**** [.noloc]`Kubernetes` `1.31` -+ -[source,bash,subs="verbatim,attributes"] ----- -curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.31.3/2024-12-12/bin/windows/amd64/kubectl.exe ----- -**** [.noloc]`Kubernetes` `1.30` +Follow the steps below to install `kubectl` on Windows. The steps include: + +* Choosing and downloading the binary for the Kubernetes version you want. +* Optionally checking the binary's checksum. +* Copying the binary to a folder in your PATH. +* Optionally adding the binary's directory to your PATH. + +Procedure: + +. Open a PowerShell terminal. +. Download the `kubectl` binary for your cluster's Kubernetes version from Amazon S3. + -[source,bash,subs="verbatim,attributes"] ----- -curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.30.7/2024-12-12/bin/windows/amd64/kubectl.exe ----- -**** [.noloc]`Kubernetes` `1.29` +**** Kubernetes `1.34` + [source,bash,subs="verbatim,attributes"] ---- -curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.29.10/2024-12-12/bin/windows/amd64/kubectl.exe +curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.34.1/2025-09-19/bin/windows/amd64/kubectl.exe ---- -**** [.noloc]`Kubernetes` `1.28` +**** Kubernetes `1.33` + [source,bash,subs="verbatim,attributes"] ---- -curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.28.15/2024-12-12/bin/windows/amd64/kubectl.exe +curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.33.5/2025-09-19/bin/windows/amd64/kubectl.exe ---- -**** [.noloc]`Kubernetes` `1.27` +**** Kubernetes `1.32` + [source,bash,subs="verbatim,attributes"] ---- -curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.27.16/2024-12-12/bin/windows/amd64/kubectl.exe +curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.32.9/2025-09-19/bin/windows/amd64/kubectl.exe ---- -**** [.noloc]`Kubernetes` `1.26` +**** Kubernetes `1.31` + [source,bash,subs="verbatim,attributes"] ---- -curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.26.15/2024-12-12/bin/windows/amd64/kubectl.exe +curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.31.13/2025-09-19/bin/windows/amd64/kubectl.exe ---- -**** [.noloc]`Kubernetes` `1.25` +**** Kubernetes `1.30` + [source,bash,subs="verbatim,attributes"] ---- -curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.25.16/2024-12-12/bin/windows/amd64/kubectl.exe +curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.30.14/2025-09-19/bin/windows/amd64/kubectl.exe ---- -**** [.noloc]`Kubernetes` `1.24` +**** Kubernetes `1.29` + [source,bash,subs="verbatim,attributes"] ---- -curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.24.17/2024-12-12/bin/windows/amd64/kubectl.exe +curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.29.15/2025-09-19/bin/windows/amd64/kubectl.exe ---- -**** [.noloc]`Kubernetes` `1.23` +**** Kubernetes `1.28` + [source,bash,subs="verbatim,attributes"] ---- -curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.23.17/2024-09-11/bin/windows/amd64/kubectl.exe +curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.28.15/2025-09-19/bin/windows/amd64/kubectl.exe ---- -**** [.noloc]`Kubernetes` `1.22` +**** Kubernetes `1.27` + [source,bash,subs="verbatim,attributes"] ---- -curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.22.17/2024-09-11/bin/windows/amd64/kubectl.exe +curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.27.16/2024-12-12/bin/windows/amd64/kubectl.exe ---- -**** [.noloc]`Kubernetes` `1.21` +**** Kubernetes `1.26` + [source,bash,subs="verbatim,attributes"] ---- -curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.21.14/2024-09-11/bin/windows/amd64/kubectl.exe +curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.26.15/2024-12-12/bin/windows/amd64/kubectl.exe ---- . (Optional) Verify the downloaded binary with the `SHA-256` checksum for your binary. + -.. Download the `SHA-256` checksum for your cluster's [.noloc]`Kubernetes` version for [.noloc]`Windows`. +.. Download the `SHA-256` checksum for your cluster's Kubernetes version for Windows. + -***** [.noloc]`Kubernetes` `1.32` -+ -[source,bash,subs="verbatim,attributes"] ----- -curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.32.0/2024-12-20/bin/windows/amd64/kubectl.exe.sha256 ----- -***** [.noloc]`Kubernetes` `1.31` +***** Kubernetes `1.34` + [source,bash,subs="verbatim,attributes"] ---- -curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.31.3/2024-12-12/bin/windows/amd64/kubectl.exe.sha256 +curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.34.1/2025-09-19/bin/windows/amd64/kubectl.exe.sha256 ---- -***** [.noloc]`Kubernetes` `1.30` +***** Kubernetes `1.33` + [source,bash,subs="verbatim,attributes"] ---- -curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.30.7/2024-12-12/bin/windows/amd64/kubectl.exe.sha256 +curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.33.5/2025-09-19/bin/windows/amd64/kubectl.exe.sha256 ---- -***** [.noloc]`Kubernetes` `1.29` +***** Kubernetes `1.32` + [source,bash,subs="verbatim,attributes"] ---- -curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.29.10/2024-12-12/bin/windows/amd64/kubectl.exe.sha256 +curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.32.9/2025-09-19/bin/windows/amd64/kubectl.exe.sha256 ---- -***** [.noloc]`Kubernetes` `1.28` +***** Kubernetes `1.31` + [source,bash,subs="verbatim,attributes"] ---- -curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.28.15/2024-12-12/bin/windows/amd64/kubectl.exe.sha256 +curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.31.13/2025-09-19/bin/windows/amd64/kubectl.exe.sha256 ---- -***** [.noloc]`Kubernetes` `1.27` +***** Kubernetes `1.30` + [source,bash,subs="verbatim,attributes"] ---- -curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.27.16/2024-12-12/bin/windows/amd64/kubectl.exe.sha256 +curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.30.14/2025-09-19/bin/windows/amd64/kubectl.exe.sha256 ---- -***** [.noloc]`Kubernetes` `1.26` +***** Kubernetes `1.29` + [source,bash,subs="verbatim,attributes"] ---- -curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.26.15/2024-12-12/bin/windows/amd64/kubectl.exe.sha256 +curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.29.15/2025-09-19/bin/windows/amd64/kubectl.exe.sha256 ---- -***** [.noloc]`Kubernetes` `1.25` +***** Kubernetes `1.28` + [source,bash,subs="verbatim,attributes"] ---- -curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.25.16/2024-12-12/bin/windows/amd64/kubectl.exe.sha256 +curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.28.15/2025-09-19/bin/windows/amd64/kubectl.exe.sha256 ---- -***** [.noloc]`Kubernetes` `1.24` +***** Kubernetes `1.27` + [source,bash,subs="verbatim,attributes"] ---- -curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.24.17/2024-12-12/bin/windows/amd64/kubectl.exe.sha256 ----- -***** [.noloc]`Kubernetes` `1.23` -+ -[source,bash,subs="verbatim,attributes"] ----- -curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.23.17/2024-09-11/bin/windows/amd64/kubectl.exe.sha256 ----- -***** [.noloc]`Kubernetes` `1.22` -+ -[source,bash,subs="verbatim,attributes"] ----- -curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.22.17/2024-09-11/bin/windows/amd64/kubectl.exe.sha256 +curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.27.16/2024-12-12/bin/windows/amd64/kubectl.exe.sha256 ---- -***** [.noloc]`Kubernetes` `1.21` +***** Kubernetes `1.26` + [source,bash,subs="verbatim,attributes"] ---- -curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.21.14/2024-09-11/bin/windows/amd64/kubectl.exe.sha256 +curl.exe -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.26.15/2024-12-12/bin/windows/amd64/kubectl.exe.sha256 ---- .. Check the `SHA-256` checksum for your downloaded binary. + @@ -781,28 +676,28 @@ Get-FileHash kubectl.exe .. Create a new directory for your command line binaries, such as `C:\bin`. .. Copy the `kubectl.exe` binary to your new directory. .. Edit your user or system `PATH` environment variable to add the new directory to your `PATH`. -.. Close your [.noloc]`PowerShell` terminal and open a new one to pick up the new `PATH` variable. +.. Close your PowerShell terminal and open a new one to pick up the new `PATH` variable. . After you install `kubectl`, you can verify its version. + [source,bash,subs="verbatim,attributes"] ---- kubectl version --client ---- -. When first installing `kubectl`, it isn't yet configured to communicate with any server. We will cover this configuration as needed in other procedures. If you ever need to update the configuration to communicate with a particular cluster, you can run the following command.Replace [.replaceable]`region-code` with the {aws} Region that your cluster is in. Replace [.replaceable]`my-cluster` with the name of your cluster. +. When first installing `kubectl`, it isn't yet configured to communicate with any server. We will cover this configuration as needed in other procedures. If you ever need to update the configuration to communicate with a particular cluster, you can run the following command. Replace [.replaceable]`region-code` with the {aws} Region that your cluster is in. Replace [.replaceable]`my-cluster` with the name of your cluster. + [source,bash,subs="verbatim,attributes"] ---- aws eks update-kubeconfig --region region-code --name my-cluster ---- -. Consider configuring auto completion, which lets you use the tab key to complete `kubectl` subcommands after typing the first few letters. See https://kubernetes.io/docs/reference/kubectl/quick-reference/#kubectl-autocomplete[Kubectl autocomplete] in the [.noloc]`Kubernetes` documentation for details. +. Consider configuring auto completion, which lets you use the tab key to complete `kubectl` subcommands after typing the first few letters. See https://kubernetes.io/docs/reference/kubectl/quick-reference/#kubectl-autocomplete[Kubectl autocomplete] in the Kubernetes documentation for details. -[[eksctl-install-update,eksctl-install-update.title]] +[#eksctl-install-update] == Install `eksctl` -The `eksctl` CLI is used to work with EKS clusters. It automates many individual tasks. See https://eksctl.io/installation[Installation] in the `eksctl` documentation for instructions on installing `eksctl`. +The `eksctl` CLI is used to work with EKS clusters. It automates many individual tasks. See https://eksctl.io/installation[Installation] in the `eksctl` documentation for instructions on installing `eksctl`. For Linux, use the UNIX instructions. -When using `eksctl` the IAM security principal that you're using must have permissions to work with Amazon EKS IAM roles, service linked roles, {aws} CloudFormation, a VPC, and related resources. For more information, see link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html[Actions, resources, and condition keys for Amazon Elastic Container Service for Kubernetes,type="documentation"] and link:IAM/latest/UserGuide/using-service-linked-roles.html[Using service-linked roles,type="documentation"] in the IAM User Guide. You must complete all steps in this guide as the same user. To check the current user, run the following command: +When using `eksctl` the IAM security principal that you're using must have permissions to work with Amazon EKS IAM roles, service linked roles, {aws} CloudFormation, a VPC, and related resources. For more information, see link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html[Actions, resources, and condition keys for Amazon Elastic Container Service for Kubernetes,type="documentation"] and link:IAM/latest/UserGuide/using-service-linked-roles.html[Using service-linked roles,type="documentation"] in the IAM User Guide. You must complete all steps in this guide as the same user. To check the current user, run the following command: [source,bash,subs="verbatim,attributes"] ---- @@ -810,7 +705,7 @@ aws sts get-caller-identity ---- -[[install-kubectl-next-steps,install-kubectl-next-steps.title]] +[#install-kubectl-next-steps] == Next steps * <> diff --git a/latest/ug/getting-started/learn-eks.adoc b/latest/ug/getting-started/learn-eks.adoc index 1631c12e8..f815ebdee 100644 --- a/latest/ug/getting-started/learn-eks.adoc +++ b/latest/ug/getting-started/learn-eks.adoc @@ -1,31 +1,19 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[learn-eks,learn-eks.title]] +[#learn-eks] = Learn Amazon EKS by example -:info_doctype: chapter -:info_title: Learn Amazon EKS by example :info_titleabbrev: Learn Amazon EKS -:keywords: tutorial, workshop, developer, learn -:info_abstract: Find learning paths to extend your knowledge of Amazon EKS. -:sectnums: -:toc: left -:icons: font -:experimental: -:idprefix: -:idseparator: - -:sourcedir: . [abstract] -- Find learning paths to extend your knowledge of Amazon EKS. -- -[[overview,overview.title]] +[#overview] == Overview -This Amazon EKS User Guide contains general-purpose procedures to create your first EKS cluster from the <> or <> and a solid reference for all major Amazon EKS components. However, as an Amazon EKS cluster administrator or developer, you can gain a deeper understanding of Amazon EKS by following learning paths that exist in sites outside of this guide. These sites can help you: +This Amazon EKS User Guide contains general-purpose procedures to create your first EKS cluster from the <> or <> and a solid reference for all major Amazon EKS components. However, as an Amazon EKS cluster administrator or developer, you can gain a deeper understanding of Amazon EKS by following learning paths that exist in sites outside of this guide. These sites can help you: @@ -36,7 +24,7 @@ This Amazon EKS User Guide contains general-purpose procedures to create your fi To start out on your Amazon EKS learning path, I recommend that you visit some of the sites described on this page. If you run into problems along the way, there are also resources to help you get through them. For example, the https://repost.aws/search/content?globalSearch=EKS[Re:post Knowledge Center] lets you search the support database for Amazon EKS-related support issues. Also the https://aws.github.io/aws-eks-best-practices/[Amazon EKS Best Practices Guide] offers tips on the best ways to set up your production-grade clusters. -[[eks-workshop,eks-workshop.title]] +[#eks-workshop] == Amazon EKS Workshop Starting with a basic understanding of Kubernetes and containers, the https://www.eksworkshop.com/[Amazon EKS workshop] is a learning platform for walking a cluster administrator through important features of Amazon EKS. Here are ways you can engage with the Amazon EKS workshop: @@ -56,7 +44,7 @@ Starting with a basic understanding of Kubernetes and containers, the https://ww ** Automation: Labs on https://www.eksworkshop.com/docs/automation/[Automation] step you through https://www.eksworkshop.com/docs/automation/gitops/[GitOps] methods of managing your clusters and projects like https://www.eksworkshop.com/docs/automation/controlplanes/ack/[{aws} Controllers for Kubernetes] and https://www.eksworkshop.com/docs/automation/controlplanes/crossplane/[Crossplane] for managing Amazon EKS control planes. -[[eks-hands-on-cluster-setup-tutorials,eks-hands-on-cluster-setup-tutorials.title]] +[#eks-hands-on-cluster-setup-tutorials] == Amazon EKS hands-on cluster setup tutorials A set of https://community.aws/tags/eks-cluster-setup[Amazon EKS Cluster Setup tutorials] on the {aws} Community site can help you create special-purpose Amazon EKS clusters and enhance those clusters in various ways. The tutorials are divided into three different types: @@ -98,7 +86,7 @@ Using these tutorials, you can better integrate your clusters with {aws} service * https://community.aws/tutorials/navigating-amazon-eks/eks-fargate-mtls-nginx-controller[Set up mTLS with Fargate, NGINX, and ACM PCA] -[[eks-samples,eks-samples.title]] +[#eks-samples] == Amazon EKS Samples The https://github.com/aws-samples/aws-eks-se-samples[Amazon EKS Samples] repository stores manifests to use with Amazon EKS. These manifests give you the opportunity to try out different kinds of applications in Amazon EKS or create specific types of Amazon EKS clusters. Samples include manifests to: @@ -119,7 +107,7 @@ The https://github.com/aws-samples/aws-eks-se-samples[Amazon EKS Samples] reposi Keep in mind that these samples are for learning and testing purposes only and are not intended to be used in production. -[[aws-tutorials,aws-tutorials.title]] +[#aws-tutorials] == {aws} Tutorials The link:tutorials[{aws} Tutorials,type="marketing"] site publishes a few Amazon EKS tutorials, but also offers a search tool to find other tutorials published on {aws} sites (such as the {aws} Community site). Amazon EKS tutorials published directly on this site include: @@ -131,19 +119,19 @@ The link:tutorials[{aws} Tutorials,type="marketing"] site publishes a few Amazon * link:tutorials/cost-optimize-jenkins/[How to cost optimize Jenkins jobs on Kubernetes,type="marketing"] -[[developers-workshop,developers-workshop.title]] +[#developers-workshop] == Developers Workshop If you are a software developer, looking to create or refactor applications to run on Amazon EKS, the http://developers.eksworkshop.com[Amazon EKS Developers workshop]is a good place to start. The workshop not only helps you build containerized applications, but also helps you deploy those containers to a container registry (link:ecr/[ECR,type="marketing"]) and from there to an Amazon EKS cluster. Start with the https://developers.eksworkshop.com/docs/python/[Amazon EKS Python Workshop] to go through the process of refactoring a python application, then set up your development environment to prepare for deploying the application. Step through sections on Containers, Kubernetes, and Amazon EKS to prepare to run your containerized applications in those environments. -[[terraform-workshop,terraform-workshop.title]] +[#terraform-workshop] == Terraform Workshop While `eksctl` is a simple tool for creating a cluster, for more complex infrastructure-as-code types of Amazon EKS deployments, https://www.terraform.io/[Terraform] is a popular Amazon EKS cluster creation and management tool. The https://catalog.us-east-1.prod.workshops.aws/workshops/afee4679-89af-408b-8108-44f5b1065cc7/en-US[Terraform Amazon EKS Workshop] teaches how to use Terraform to build an {aws} VPC, create Amazon EKS clusters, and add optional enhancements to your cluster. In particular, there is a section for creating a https://catalog.us-east-1.prod.workshops.aws/workshops/afee4679-89af-408b-8108-44f5b1065cc7/en-US/500-eks-terraform-workshop[private Amazon EKS cluster] -[[aws-eks-training,aws-eks-training.title]] +[#aws-eks-training] == {aws} Amazon EKS Training {aws} offers formal training for learning about Amazon EKS. A three-day training course entitled link:training/classroom/running-containers-on-amazon-elastic-kubernetes-service-amazon-eks/[Running Containers on Amazon Elastic Kubernetes Service,type="marketing"] teaches: @@ -155,4 +143,4 @@ While `eksctl` is a simple tool for creating a cluster, for more complex infrast * Securing Amazon EKS with {aws} IAM and Kubernetes RBAC authorization * GitOps automation tools * Monitoring tools -* Techniques for improving cost, efficiency, and resiliency +* Techniques for improving cost, efficiency, and resiliency \ No newline at end of file diff --git a/latest/ug/getting-started/setting-up.adoc b/latest/ug/getting-started/setting-up.adoc index 036f89865..a9a2dacbb 100644 --- a/latest/ug/getting-started/setting-up.adoc +++ b/latest/ug/getting-started/setting-up.adoc @@ -1,20 +1,8 @@ -//!!NODE_ROOT include::../attributes.txt[] -[[setting-up,setting-up.title]] + +[#setting-up] = Set up to use Amazon EKS -:doctype: book -:sectnums: -:toc: left -:icons: font -:experimental: -:idprefix: -:idseparator: - -:sourcedir: . -:info_doctype: chapter -:info_title: Set up to use Amazon EKS :info_titleabbrev: Set up -:keywords: setting up, setup -:info_abstract: Set up the tools needed for creating and working with an Amazon EKS cluster. [abstract] -- @@ -29,11 +17,11 @@ To prepare for the command-line management of your Amazon EKS clusters, you need * <> – The `eksctl` CLI interacts with {aws} to create, modify, and delete Amazon EKS clusters. Once a cluster is up, use the open source `kubectl` command to manage Kubernetes objects within your Amazon EKS clusters. * Set up a development environment (optional)– Consider adding the following tools: + -** *Local deployment tool* – If you're new to [.noloc]`Kubernetes`, consider installing a local deployment tool like https://minikube.sigs.k8s.io/docs/[minikube] or https://kind.sigs.k8s.io/[kind]. These tools allow you to have an Amazon EKS cluster on your local machine for testing applications. -** *Package manager* – https://helm.sh/docs/intro/install/[Helm] is a popular package manager for [.noloc]`Kubernetes` that simplifies the installation and management of complex packages. With [.noloc]`Helm`, it's easier to install and manage packages like the {aws} Load Balancer Controller on your Amazon EKS cluster. +** *Local deployment tool* – If you're new to Kubernetes, consider installing a local deployment tool like https://minikube.sigs.k8s.io/docs/[minikube] or https://kind.sigs.k8s.io/[kind]. These tools allow you to have an Amazon EKS cluster on your local machine for testing applications. +** *Package manager* – <> is a popular package manager for Kubernetes that simplifies the installation and management of complex packages. With <>, it's easier to install and manage packages like the {aws} Load Balancer Controller on your Amazon EKS cluster. -[[setting-up-next-steps,setting-up-next-steps.title]] +[#setting-up-next-steps] == Next steps * <> @@ -44,4 +32,4 @@ To prepare for the command-line management of your Amazon EKS clusters, you need include::install-awscli.adoc[leveloffset=+1] -include::install-kubectl.adoc[leveloffset=+1] +include::install-kubectl.adoc[leveloffset=+1] \ No newline at end of file diff --git a/latest/ug/iam_policies/AmazonEKS_CNI_IPv6_Policy.json b/latest/ug/iam_policies/AmazonEKS_CNI_IPv6_Policy.json index 23d72e4e2..ae68c611e 100644 --- a/latest/ug/iam_policies/AmazonEKS_CNI_IPv6_Policy.json +++ b/latest/ug/iam_policies/AmazonEKS_CNI_IPv6_Policy.json @@ -18,7 +18,7 @@ "ec2:CreateTags" ], "Resource": [ - "arn:aws:ec2:*:*:network-interface/*" + "{arn-aws}ec2:*:*:network-interface/*" ] } ] diff --git a/latest/ug/images/cn-image.png b/latest/ug/images/cn-image.png new file mode 100644 index 000000000..90f8d479c Binary files /dev/null and b/latest/ug/images/cn-image.png differ diff --git a/latest/ug/images/contribute-pr.png b/latest/ug/images/contribute-pr.png new file mode 100644 index 000000000..5b812a1fe Binary files /dev/null and b/latest/ug/images/contribute-pr.png differ diff --git a/latest/ug/images/contribute-preview.png b/latest/ug/images/contribute-preview.png new file mode 100644 index 000000000..16e971f16 Binary files /dev/null and b/latest/ug/images/contribute-preview.png differ diff --git a/latest/ug/images/contribute-q.png b/latest/ug/images/contribute-q.png new file mode 100644 index 000000000..15db73350 Binary files /dev/null and b/latest/ug/images/contribute-q.png differ diff --git a/latest/ug/images/contribute-style-local.png b/latest/ug/images/contribute-style-local.png new file mode 100644 index 000000000..654364989 Binary files /dev/null and b/latest/ug/images/contribute-style-local.png differ diff --git a/latest/ug/images/contribute-style-web.png b/latest/ug/images/contribute-style-web.png new file mode 100644 index 000000000..bb7c0b8d3 Binary files /dev/null and b/latest/ug/images/contribute-style-web.png differ diff --git a/latest/ug/images/contribute-web-dev.png b/latest/ug/images/contribute-web-dev.png new file mode 100644 index 000000000..04f3de1e2 Binary files /dev/null and b/latest/ug/images/contribute-web-dev.png differ diff --git a/latest/ug/images/contribute-web-edit.png b/latest/ug/images/contribute-web-edit.png new file mode 100644 index 000000000..09f851c5e Binary files /dev/null and b/latest/ug/images/contribute-web-edit.png differ diff --git a/latest/ug/images/eks-dashboard.png b/latest/ug/images/eks-dashboard.png new file mode 100644 index 000000000..a3186dbc9 Binary files /dev/null and b/latest/ug/images/eks-dashboard.png differ diff --git a/latest/ug/images/eksautosrm.png b/latest/ug/images/eksautosrm.png new file mode 100644 index 000000000..1659a4f7e Binary files /dev/null and b/latest/ug/images/eksautosrm.png differ diff --git a/latest/ug/images/hybrid-nodes-arp-proxy.png b/latest/ug/images/hybrid-nodes-arp-proxy.png new file mode 100644 index 000000000..80cf1ce9f Binary files /dev/null and b/latest/ug/images/hybrid-nodes-arp-proxy.png differ diff --git a/latest/ug/images/hybrid-nodes-bgp.png b/latest/ug/images/hybrid-nodes-bgp.png new file mode 100644 index 000000000..3b86e611a Binary files /dev/null and b/latest/ug/images/hybrid-nodes-bgp.png differ diff --git a/latest/ug/images/hybrid-nodes-cp-to-kubelet.png b/latest/ug/images/hybrid-nodes-cp-to-kubelet.png new file mode 100644 index 000000000..b9ccccc6a Binary files /dev/null and b/latest/ug/images/hybrid-nodes-cp-to-kubelet.png differ diff --git a/latest/ug/images/hybrid-nodes-cp-to-pod.png b/latest/ug/images/hybrid-nodes-cp-to-pod.png new file mode 100644 index 000000000..81b772b20 Binary files /dev/null and b/latest/ug/images/hybrid-nodes-cp-to-pod.png differ diff --git a/latest/ug/images/hybrid-nodes-east-west.png b/latest/ug/images/hybrid-nodes-east-west.png new file mode 100644 index 000000000..b13e251f2 Binary files /dev/null and b/latest/ug/images/hybrid-nodes-east-west.png differ diff --git a/latest/ug/images/hybrid-nodes-highlevel-network.png b/latest/ug/images/hybrid-nodes-highlevel-network.png new file mode 100644 index 000000000..2b9a8a07d Binary files /dev/null and b/latest/ug/images/hybrid-nodes-highlevel-network.png differ diff --git a/latest/ug/images/hybrid-nodes-ingress.png b/latest/ug/images/hybrid-nodes-ingress.png new file mode 100644 index 000000000..08c1ad2d4 Binary files /dev/null and b/latest/ug/images/hybrid-nodes-ingress.png differ diff --git a/latest/ug/images/hybrid-nodes-kubelet-to-cp-public.png b/latest/ug/images/hybrid-nodes-kubelet-to-cp-public.png new file mode 100644 index 000000000..ea207ddeb Binary files /dev/null and b/latest/ug/images/hybrid-nodes-kubelet-to-cp-public.png differ diff --git a/latest/ug/images/hybrid-nodes-pod-to-cp.png b/latest/ug/images/hybrid-nodes-pod-to-cp.png new file mode 100644 index 000000000..c5ef196a6 Binary files /dev/null and b/latest/ug/images/hybrid-nodes-pod-to-cp.png differ diff --git a/latest/ug/images/hybrid-nodes-pod-to-pod.png b/latest/ug/images/hybrid-nodes-pod-to-pod.png new file mode 100644 index 000000000..b338fa3fd Binary files /dev/null and b/latest/ug/images/hybrid-nodes-pod-to-pod.png differ diff --git a/latest/ug/images/hybrid-nodes-remote-pod-cidrs.png b/latest/ug/images/hybrid-nodes-remote-pod-cidrs.png new file mode 100644 index 000000000..c0b5075b0 Binary files /dev/null and b/latest/ug/images/hybrid-nodes-remote-pod-cidrs.png differ diff --git a/latest/ug/images/hybrid-nodes-static-routes.png b/latest/ug/images/hybrid-nodes-static-routes.png new file mode 100644 index 000000000..e67706c51 Binary files /dev/null and b/latest/ug/images/hybrid-nodes-static-routes.png differ diff --git a/latest/ug/images/kubecost.png b/latest/ug/images/kubecost.png deleted file mode 100644 index cbb7b51de..000000000 Binary files a/latest/ug/images/kubecost.png and /dev/null differ diff --git a/latest/ug/images/multi-homed-pod.png b/latest/ug/images/multi-homed-pod.png new file mode 100644 index 000000000..50d44d5f5 Binary files /dev/null and b/latest/ug/images/multi-homed-pod.png differ diff --git a/latest/ug/images/security-encrypt-request.png b/latest/ug/images/security-encrypt-request.png new file mode 100644 index 000000000..82051a8bd Binary files /dev/null and b/latest/ug/images/security-encrypt-request.png differ diff --git a/latest/ug/images/security-generate-dek.png b/latest/ug/images/security-generate-dek.png new file mode 100644 index 000000000..c9da41c5a Binary files /dev/null and b/latest/ug/images/security-generate-dek.png differ diff --git a/latest/ug/images/whatis.png b/latest/ug/images/whatis.png new file mode 100644 index 000000000..5504ce23d Binary files /dev/null and b/latest/ug/images/whatis.png differ diff --git a/latest/ug/integrations/creating-resources-with-cloudformation.adoc b/latest/ug/integrations/creating-resources-with-cloudformation.adoc index e936c6bfb..580b3eb45 100644 --- a/latest/ug/integrations/creating-resources-with-cloudformation.adoc +++ b/latest/ug/integrations/creating-resources-with-cloudformation.adoc @@ -1,14 +1,9 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[creating-resources-with-cloudformation,creating-resources-with-cloudformation.title]] +[#creating-resources-with-cloudformation] = Create Amazon EKS resources with {aws} CloudFormation -:info_doctype: section -:info_title: Create Amazon EKS resources with \ - {aws} CloudFormation -:info_abstract: Learn about how to create resources for Amazon EKS using an {aws} CloudFormation \ - template. +:info_titleabbrev: {aws} CloudFormation [abstract] -- @@ -19,14 +14,14 @@ Amazon EKS is integrated with {aws} CloudFormation, a service that helps you mod When you use {aws} CloudFormation, you can reuse your template to set up your Amazon EKS resources consistently and repeatedly. Just describe your resources once, and then provision the same resources over and over in multiple {aws} accounts and Regions. -[[working-with-templates,working-with-templates.title]] +[#working-with-templates] == Amazon EKS and {aws} CloudFormation templates To provision and configure resources for Amazon EKS and related services, you must understand link:AWSCloudFormation/latest/UserGuide/template-guide.html[{aws} CloudFormation templates,type="documentation"]. Templates are formatted text files in JSON or YAML. These templates describe the resources that you want to provision in your {aws} CloudFormation stacks. If you're unfamiliar with JSON or YAML, you can use {aws} CloudFormation Designer to help you get started with {aws} CloudFormation templates. For more information, see link:AWSCloudFormation/latest/UserGuide/working-with-templates-cfn-designer.html[What is {aws} CloudFormation Designer?,type="documentation"] in the _{aws} CloudFormation User Guide_. -Amazon EKS supports creating clusters and node groups in {aws} CloudFormation. For more information, including examples of JSON and YAML templates for your Amazon EKS resources, see link:AWSCloudFormation/latest/UserGuide/AWS_EKS.html[Amazon EKS resource type reference,type="documentation"] in the _{aws} CloudFormation User Guide_. +Amazon EKS supports creating clusters and node groups in {aws} CloudFormation. For more information, including examples of JSON and YAML templates for your Amazon EKS resources, see link:AWSCloudFormation/latest/UserGuide/AWS_EKS.html[Amazon EKS resource type reference,type="documentation"] in the _{aws} CloudFormation User Guide_. -[[learn-more-cloudformation,learn-more-cloudformation.title]] +[#learn-more-cloudformation] == Learn more about {aws} CloudFormation To learn more about {aws} CloudFormation, see the following resources: @@ -35,4 +30,4 @@ To learn more about {aws} CloudFormation, see the following resources: * link:cloudformation/[{aws} CloudFormation,type="marketing"] * link:AWSCloudFormation/latest/UserGuide/Welcome.html[{aws} CloudFormation User Guide,type="documentation"] -* link:cloudformation-cli/latest/userguide/what-is-cloudformation-cli.html[{aws} CloudFormation Command Line Interface User Guide,type="documentation"] +* link:cloudformation-cli/latest/userguide/what-is-cloudformation-cli.html[{aws} CloudFormation Command Line Interface User Guide,type="documentation"] \ No newline at end of file diff --git a/latest/ug/integrations/eks-integrations.adoc b/latest/ug/integrations/eks-integrations.adoc index 50107196f..a4778f605 100644 --- a/latest/ug/integrations/eks-integrations.adoc +++ b/latest/ug/integrations/eks-integrations.adoc @@ -1,17 +1,7 @@ -//!!NODE_ROOT include::../attributes.txt[] -[[eks-integrations,eks-integrations.title]] + +[#eks-integrations] = Enhance EKS with integrated {aws} services -:doctype: book -:sectnums: -:toc: left -:icons: font -:experimental: -:idprefix: -:idseparator: - -:sourcedir: . -:info_doctype: chapter -:info_title: Enhance EKS with integrated {aws} services :info_titleabbrev: Working with other services In addition to the services covered in other sections, Amazon EKS works with more {aws} services to provide additional solutions. This topic identifies some of the other services that either use Amazon EKS to add functionality, or services that Amazon EKS uses to perform tasks. @@ -21,20 +11,14 @@ In addition to the services covered in other sections, Amazon EKS works with mor include::creating-resources-with-cloudformation.adoc[leveloffset=+1] - include::integration-detective.adoc[leveloffset=+1] - include::integration-guardduty.adoc[leveloffset=+1] - include::integration-resilience-hub.adoc[leveloffset=+1] - include::integration-securitylake.adoc[leveloffset=+1] - include::integration-vpc-lattice.adoc[leveloffset=+1] - -include::local-zones.adoc[leveloffset=+1] +include::local-zones.adoc[leveloffset=+1] \ No newline at end of file diff --git a/latest/ug/integrations/integration-detective.adoc b/latest/ug/integrations/integration-detective.adoc index 038992f36..27a37ed5b 100644 --- a/latest/ug/integrations/integration-detective.adoc +++ b/latest/ug/integrations/integration-detective.adoc @@ -1,14 +1,9 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[integration-detective,integration-detective.title]] +[#integration-detective] = Analyze security events on EKS with Amazon Detective -:info_doctype: section -:info_title: Analyze security events on EKS with Amazon Detective -:info_abstract: Amazon Detective helps you analyze, investigate, and quickly identify the root cause \ - of security findings or suspicious activities. -:keywords: Amazon Detective +:info_titleabbrev: Amazon Detective [abstract] -- @@ -17,23 +12,23 @@ Amazon Detective helps you analyze, investigate, and quickly identify the root c link:detective/[Amazon Detective,type="marketing"] helps you analyze, investigate, and quickly identify the root cause of security findings or suspicious activities. Detective automatically collects log data from your {aws} resources. It then uses machine learning, statistical analysis, and graph theory to generate visualizations that help you to conduct faster and more efficient security investigations. The Detective prebuilt data aggregations, summaries, and context help you to quickly analyze and determine the nature and extent of possible security issues. For more information, see the link:detective/latest/adminguide/what-is-detective.html[Amazon Detective User Guide,type="documentation"]. -Detective organizes [.noloc]`Kubernetes` and {aws} data into findings such as: +Detective organizes Kubernetes and {aws} data into findings such as: -* Amazon EKS cluster details, including the IAM identity that created the cluster and the service role of the cluster. You can investigate the {aws} and [.noloc]`Kubernetes` API activity of these IAM identities with Detective. -* Container details, such as the image and security context. You can also review details for terminated [.noloc]`Pods`. -* [.noloc]`Kubernetes` API activity, including both overall trends in API activity and details on specific API calls. For example, you can show the number of successful and failed [.noloc]`Kubernetes` API calls that were issued during a selected time range. Additionally, the section on newly observed API calls might be helpful to identify suspicious activity. +* Amazon EKS cluster details, including the IAM identity that created the cluster and the service role of the cluster. You can investigate the {aws} and Kubernetes API activity of these IAM identities with Detective. +* Container details, such as the image and security context. You can also review details for terminated Pods. +* Kubernetes API activity, including both overall trends in API activity and details on specific API calls. For example, you can show the number of successful and failed Kubernetes API calls that were issued during a selected time range. Additionally, the section on newly observed API calls might be helpful to identify suspicious activity. Amazon EKS audit logs is an optional data source package that can be added to your Detective behavior graph. You can view the available optional source packages, and their status in your account. For more information, see link:detective/latest/adminguide/source-data-types-EKS.html[Amazon EKS audit logs for Detective,type="documentation"] in the _Amazon Detective User Guide_. -[[integration-detective-use,integration-detective-use.title]] +[#integration-detective-use] == Use Amazon Detective with Amazon EKS Before you can review findings, Detective must be enabled for at least 48 hours in the same {aws} Region that your cluster is in. For more information, see link:detective/latest/adminguide/detective-setup.html[Setting up Amazon Detective,type="documentation"] in the _Amazon Detective User Guide_. . Open the Detective console at https://console.aws.amazon.com/detective/. . From the left navigation pane, select *Search*. -. Select *Choose type* and then select *EKS cluster*. +. Select *Choose type* and then select *EKS cluster*. . Enter the cluster name or ARN and then choose *Search*. -. In the search results, choose the name of the cluster that you want to view activity for. For more information about what you can view, see link:detective/latest/userguide/profile-panel-drilldown-kubernetes-api-volume.html[Overall Kubernetes API activity involving an Amazon EKS cluster,type="documentation"] in the _Amazon Detective User Guide_. +. In the search results, choose the name of the cluster that you want to view activity for. For more information about what you can view, see link:detective/latest/userguide/profile-panel-drilldown-kubernetes-api-volume.html[Overall Kubernetes API activity involving an Amazon EKS cluster,type="documentation"] in the _Amazon Detective User Guide_. \ No newline at end of file diff --git a/latest/ug/integrations/integration-guardduty.adoc b/latest/ug/integrations/integration-guardduty.adoc index e38d5df66..fc50da20f 100644 --- a/latest/ug/integrations/integration-guardduty.adoc +++ b/latest/ug/integrations/integration-guardduty.adoc @@ -1,11 +1,9 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[integration-guardduty,integration-guardduty.title]] +[#integration-guardduty] = Detect threats with Amazon GuardDuty -:info_doctype: section -:info_title: Detect threats with Amazon GuardDuty +:info_titleabbrev: Amazon GuardDuty Amazon GuardDuty is a threat detection service that helps protect you accounts, containers, workloads, and the data with your {aws} environment. Using machine learning (ML) models, and anomaly and threat detection capabilities, GuardDuty continuously monitors different log sources and runtime activity to identify and prioritize potential security risks and malicious activities in your environment. @@ -19,9 +17,9 @@ Among other features, GuardDuty offers the following two features that detect po *EKS Protection*:: -This feature provides threat detection coverage to help you protect Amazon EKS clusters by monitoring the associated [.noloc]`Kubernetes` audit logs. [.noloc]`Kubernetes` audit logs capture sequential actions within your cluster, including activities from users, applications using the [.noloc]`Kubernetes` API, and the control plane. For example, GuardDuty can identify that APIs called to potentially tamper with resources in a [.noloc]`Kubernetes` cluster were invoked by an unauthenticated user. +This feature provides threat detection coverage to help you protect Amazon EKS clusters by monitoring the associated Kubernetes audit logs. Kubernetes audit logs capture sequential actions within your cluster, including activities from users, applications using the Kubernetes API, and the control plane. For example, GuardDuty can identify that APIs called to potentially tamper with resources in a Kubernetes cluster were invoked by an unauthenticated user. + -When you enable EKS Protection, GuardDuty will be able to access your Amazon EKS audit logs only for continuous threat detection. If GuardDuty identifies a potential threat to your cluster, it generates an associated [.noloc]`Kubernetes` audit log _finding_ of a specific type. For more information about the types of findings available from [.noloc]`Kubernetes` audit logs, see link:guardduty/latest/ug/guardduty_finding-types-kubernetes.html[Kubernetes audit logs finding types,type="documentation"] in the Amazon GuardDuty User Guide. +When you enable EKS Protection, GuardDuty will be able to access your Amazon EKS audit logs only for continuous threat detection. If GuardDuty identifies a potential threat to your cluster, it generates an associated Kubernetes audit log _finding_ of a specific type. For more information about the types of findings available from Kubernetes audit logs, see link:guardduty/latest/ug/guardduty_finding-types-kubernetes.html[Kubernetes audit logs finding types,type="documentation"] in the Amazon GuardDuty User Guide. + For more information, see link:guardduty/latest/ug/kubernetes-protection.html[EKS Protection,type="documentation"] in the Amazon GuardDuty User Guide. @@ -33,4 +31,4 @@ When you enable _Runtime Monitoring_ and install the GuardDuty agent in your Ama + To configure _Runtime Monitoring_, you install the GuardDuty agent to your cluster as an _Amazon EKS add-on_. For more information the add-on, see <>. + -For more information, see link:guardduty/latest/ug/runtime-monitoring.html[Runtime Monitoring,type="documentation"] in the Amazon GuardDuty User Guide. +For more information, see link:guardduty/latest/ug/runtime-monitoring.html[Runtime Monitoring,type="documentation"] in the Amazon GuardDuty User Guide. \ No newline at end of file diff --git a/latest/ug/integrations/integration-resilience-hub.adoc b/latest/ug/integrations/integration-resilience-hub.adoc index 0d31c02bc..66414c241 100644 --- a/latest/ug/integrations/integration-resilience-hub.adoc +++ b/latest/ug/integrations/integration-resilience-hub.adoc @@ -1,10 +1,8 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[integration-resilience-hub,integration-resilience-hub.title]] +[#integration-resilience-hub] = Assess EKS cluster resiliency with {aws} Resilience Hub -:info_doctype: section -:info_title: Assess EKS cluster resiliency with {aws} Resilience Hub +:info_titleabbrev: {aws} Resilience Hub -{aws} Resilience Hub assesses the resiliency of an Amazon EKS cluster by analyzing its infrastructure. {aws} Resilience Hub uses the [.noloc]`Kubernetes` role-based access control (RBAC) configuration to assess the [.noloc]`Kubernetes` workloads deployed to your cluster. For more information, see link:resilience-hub/latest/userguide/enabling-eks-in-arh.html[Enabling {aws} Resilience Hub access to your Amazon EKS cluster,type="documentation"] in the {aws} Resilience Hub User Guide. +{aws} Resilience Hub assesses the resiliency of an Amazon EKS cluster by analyzing its infrastructure. {aws} Resilience Hub uses the Kubernetes role-based access control (RBAC) configuration to assess the Kubernetes workloads deployed to your cluster. For more information, see link:resilience-hub/latest/userguide/enabling-eks-in-arh.html[Enabling {aws} Resilience Hub access to your Amazon EKS cluster,type="documentation"] in the {aws} Resilience Hub User Guide. \ No newline at end of file diff --git a/latest/ug/integrations/integration-securitylake.adoc b/latest/ug/integrations/integration-securitylake.adoc index 40588b762..c4b1bd0db 100644 --- a/latest/ug/integrations/integration-securitylake.adoc +++ b/latest/ug/integrations/integration-securitylake.adoc @@ -1,33 +1,26 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[integration-securitylake,integration-securitylake.title]] +[#integration-securitylake] = Centralize and analyze EKS security data with Security Lake -:info_doctype: section -:info_title: Centralize and analyze EKS security data with Security Lake -:info_abstract: Amazon Security Lake integrates with Amazon EKS to provide a centralized and standardized \ - solution for collecting, storing, and analyzing security data from clusters. By \ - enabling EKS control plane logging and adding EKS logs as a source in Security Lake, \ - users can gain valuable insights, detect potential threats, and enhance the \ - security posture of their Kubernetes environments. -:keywords: Amazon EKS, Amazon Security Lake, Kubernetes security, centralized security data, threat detection +:info_titleabbrev: Amazon Security Lake + [abstract] -- -Amazon Security Lake integrates with Amazon EKS to provide a centralized and standardized solution for collecting, storing, and analyzing security data from clusters. By enabling EKS control plane logging and adding EKS logs as a source in Security Lake, users can gain valuable insights, detect potential threats, and enhance the security posture of their [.noloc]`Kubernetes` environments. +Amazon Security Lake integrates with Amazon EKS to provide a centralized and standardized solution for collecting, storing, and analyzing security data from clusters. By enabling EKS control plane logging and adding EKS logs as a source in Security Lake, users can gain valuable insights, detect potential threats, and enhance the security posture of their Kubernetes environments. -- -Amazon Security Lake is a fully managed security data lake service that allows you to centralize security data from various sources, including Amazon EKS. By integrating Amazon EKS with Security Lake, you can gain deeper insights into the activities performed on your [.noloc]`Kubernetes` resources and enhance the security posture of your Amazon EKS clusters. +Amazon Security Lake is a fully managed security data lake service that allows you to centralize security data from various sources, including Amazon EKS. By integrating Amazon EKS with Security Lake, you can gain deeper insights into the activities performed on your Kubernetes resources and enhance the security posture of your Amazon EKS clusters. [NOTE] ==== -For more information about using Security Lake with Amazon EKS and setting up data sources, refer to the link:security-lake/latest/userguide/internal-sources.html#eks-eudit-logs[Amazon Security Lake documentation,type="documentation"]. +For more information about using Security Lake with Amazon EKS and setting up data sources, refer to the link:security-lake/latest/userguide/internal-sources.html#eks-eudit-logs[Amazon Security Lake documentation,type="documentation"]. ==== -[[sl-benefits,sl-benefits.title]] +[#sl-benefits] == Benefits of using Security Lake with Amazon Amazon EKS *Centralized security data* -- Security Lake automatically collects and centralizes security data from your Amazon EKS clusters, along with data from other {aws} services, SaaS providers, on-premises sources, and third-party sources. This provides a comprehensive view of your security posture across your entire organization. @@ -38,17 +31,17 @@ For more information about using Security Lake with Amazon EKS and setting up da *Simplified data management* -- Security Lake manages the lifecycle of your security data with customizable retention and replication settings. This simplifies data management tasks and ensures that you retain the necessary data for compliance and auditing purposes. -[[sl-enable,sl-enable.title]] +[#sl-enable] == Enabling Security Lake for Amazon EKS . Enable Amazon EKS control plane logging for your EKS clusters. Refer to <> for detailed instructions. -. link:security-lake/latest/userguide/internal-sources.html#add-internal-sources[Add Amazon EKS Audit Logs as a source in Security Lake.,type="documentation"] Security Lake will then start collecting in-depth information about the activities performed on the Kubernetes resources running in your EKS clusters. +. link:security-lake/latest/userguide/internal-sources.html#add-internal-sources[Add Amazon EKS Audit Logs as a source in Security Lake.,type="documentation"] Security Lake will then start collecting in-depth information about the activities performed on the Kubernetes resources running in your EKS clusters. . link:security-lake/latest/userguide/lifecycle-management.html[Configure retention and replication settings,type="documentation"] for your security data in Security Lake based on your requirements. . Use the normalized OCSF data stored in Security Lake for incident response, security analytics, and integration with other {aws} services or third-party tools. For example, you can link:big-data/generate-security-insights-from-amazon-security-lake-data-using-amazon-opensearch-ingestion[Generate security insights from Amazon Security Lake data using Amazon OpenSearch Ingestion,type="blog"]. -[[sl-format,sl-format.title]] +[#sl-format] == Analyzing EKS Logs in Security Lake Security Lake normalizes EKS log events to the OCSF format, making it easier to analyze and correlate the data with other security events. You can use various tools and services, such as Amazon Athena, Amazon QuickSight, or third-party security analytics tools, to query and visualize the normalized data. -For more information about the OCSF mapping for EKS log events, refer to the https://github.com/ocsf/examples/tree/main/mappings/markdown/{aws}/v1.1.0/EKS Audit Logs[mapping reference] in the OCSF GitHub repository. +For more information about the OCSF mapping for EKS log events, refer to the https://github.com/ocsf/examples/tree/main/mappings/markdown/{aws}/v1.1.0/EKS Audit Logs[mapping reference] in the OCSF GitHub repository. \ No newline at end of file diff --git a/latest/ug/integrations/integration-vpc-lattice.adoc b/latest/ug/integrations/integration-vpc-lattice.adoc index 84bc58b1f..140232d4d 100644 --- a/latest/ug/integrations/integration-vpc-lattice.adoc +++ b/latest/ug/integrations/integration-vpc-lattice.adoc @@ -1,10 +1,8 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[integration-vpc-lattice,integration-vpc-lattice.title]] +[#integration-vpc-lattice] = Enable secure cross-cluster connectivity with Amazon VPC Lattice -:info_doctype: section -:info_title: Enable secure cross-cluster connectivity with Amazon VPC Lattice +:info_titleabbrev: Amazon VPC Lattice -Amazon VPC Lattice is a fully managed application networking service built directly into the {aws} networking infrastructure that you can use to connect, secure, and monitor your services across multiple accounts and Virtual Private Clouds (VPCs). With Amazon EKS, you can leverage Amazon VPC Lattice through the use of the {aws} Gateway API Controller, an implementation of the Kubernetes https://gateway-api.sigs.k8s.io/[Gateway API]. Using Amazon VPC Lattice, you can set up cross-cluster connectivity with standard [.noloc]`Kubernetes` semantics in a simple and consistent manner. To get started using Amazon VPC Lattice with Amazon EKS see the https://www.gateway-api-controller.eks.aws.dev/[{aws} Gateway API Controller User Guide]. +Amazon VPC Lattice is a fully managed application networking service built directly into the {aws} networking infrastructure that you can use to connect, secure, and monitor your services across multiple accounts and Virtual Private Clouds (VPCs). With Amazon EKS, you can leverage Amazon VPC Lattice through the use of the {aws} Gateway API Controller, an implementation of the Kubernetes https://gateway-api.sigs.k8s.io/[Gateway API]. Using Amazon VPC Lattice, you can set up cross-cluster connectivity with standard Kubernetes semantics in a simple and consistent manner. To get started using Amazon VPC Lattice with Amazon EKS see the https://www.gateway-api-controller.eks.aws.dev/[{aws} Gateway API Controller User Guide]. \ No newline at end of file diff --git a/latest/ug/integrations/local-zones.adoc b/latest/ug/integrations/local-zones.adoc index 4df1cf272..ce9f3b2d7 100644 --- a/latest/ug/integrations/local-zones.adoc +++ b/latest/ug/integrations/local-zones.adoc @@ -1,17 +1,15 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[local-zones,local-zones.title]] +[#local-zones] = Launch low-latency EKS clusters with {aws} Local Zones -:info_doctype: section -:info_title: Launch low-latency EKS clusters with {aws} Local Zones +:info_titleabbrev: {aws} Local Zones An link:about-aws/global-infrastructure/localzones/[{aws} Local Zone,type="marketing"] is an extension of an {aws} Region in geographic proximity to your users. Local Zones have their own connections to the internet and support link:directconnect/[{aws} Direct Connect,type="marketing"]. Resources created in a Local Zone can serve local users with low-latency communications. For more information, see the link:local-zones/latest/ug/what-is-aws-local-zones.html[{aws} Local Zones User Guide,type="documentation"] and link:AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-local-zones[Local Zones,type="documentation"] in the _Amazon EC2 User Guide_. Amazon EKS supports certain resources in Local Zones. This includes <>, <>, Amazon EBS volumes, and Application Load Balancers (ALBs). We recommend that you consider the following when using Local Zones as part of your Amazon EKS cluster. * You can't create Fargate nodes in Local Zones with Amazon EKS. -* The Amazon EKS managed [.noloc]`Kubernetes` control plane always runs in the {aws} Region. The Amazon EKS managed [.noloc]`Kubernetes` control plane can't run in the Local Zone. Because Local Zones appear as a subnet within your VPC, [.noloc]`Kubernetes` sees your Local Zone resources as part of that subnet. -* The Amazon EKS [.noloc]`Kubernetes` cluster communicates with the Amazon EC2 instances you run in the {aws} Region or Local Zone using Amazon EKS managed link:AWSEC2/latest/UserGuide/using-eni.html[elastic network interfaces,type="documentation"]. To learn more about Amazon EKS networking architecture, see <>. -* Unlike regional subnets, Amazon EKS can't place network interfaces into your Local Zone subnets. This means that you must not specify Local Zone subnets when you create your cluster. +* The Amazon EKS managed Kubernetes control plane always runs in the {aws} Region. The Amazon EKS managed Kubernetes control plane can't run in the Local Zone. Because Local Zones appear as a subnet within your VPC, Kubernetes sees your Local Zone resources as part of that subnet. +* The Amazon EKS Kubernetes cluster communicates with the Amazon EC2 instances you run in the {aws} Region or Local Zone using Amazon EKS managed link:AWSEC2/latest/UserGuide/using-eni.html[elastic network interfaces,type="documentation"]. To learn more about Amazon EKS networking architecture, see <>. +* Unlike regional subnets, Amazon EKS can't place network interfaces into your Local Zone subnets. This means that you must not specify Local Zone subnets when you create your cluster. However, you can have worker nodes in different multiple Local Zones connected to the same cluster. diff --git a/latest/ug/manage-access/aws-access/associate-service-account-role.adoc b/latest/ug/manage-access/aws-access/associate-service-account-role.adoc new file mode 100644 index 000000000..1595ab23c --- /dev/null +++ b/latest/ug/manage-access/aws-access/associate-service-account-role.adoc @@ -0,0 +1,221 @@ +include::../../attributes.txt[] + +[.topic] +[#associate-service-account-role] += Assign IAM roles to Kubernetes service accounts +:info_titleabbrev: Assign IAM role + +[abstract] +-- +Discover how to configure a Kubernetes service account to assume an IAM role, enabling Pods to securely access {aws} services with granular permissions. +-- + +This topic covers how to configure a Kubernetes service account to assume an {aws} Identity and Access Management (IAM) role. Any Pods that are configured to use the service account can then access any {aws} service that the role has permissions to access. + +== Prerequisites + +* An existing cluster. If you don't have one, you can create one by following one of the guides in <>. +* An existing IAM OpenID Connect (OIDC) provider for your cluster. To learn if you already have one or how to create one, see <>. +* Version `2.12.3` or later or version `1.27.160` or later of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device or {aws} CloudShell. To check your current version, use `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or Homebrew for macOS are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the _{aws} Command Line Interface User Guide_. The {aws} CLI version that is installed in {aws} CloudShell might also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the _{aws} CloudShell User Guide_. +* The `kubectl` command line tool is installed on your device or {aws} CloudShell. The version can be the same as or up to one minor version earlier or later than the Kubernetes version of your cluster. For example, if your cluster version is `1.29`, you can use `kubectl` version `1.28`, `1.29`, or `1.30` with it. To install or upgrade `kubectl`, see <>. +* An existing `kubectl` `config` file that contains your cluster configuration. To create a `kubectl` `config` file, see <>. + + +[#irsa-associate-role-procedure] +== Step 1: Create IAM Policy + +If you want to associate an existing IAM policy to your IAM role, skip to the next step. + + +. Create an IAM policy. You can create your own policy, or copy an {aws} managed policy that already grants some of the permissions that you need and customize it to your specific requirements. For more information, see link:IAM/latest/UserGuide/access_policies_create.html[Creating IAM policies,type="documentation"] in the _IAM User Guide_. ++ +. Create a file that includes the permissions for the {aws} services that you want your Pods to access. For a list of all actions for all {aws} services, see the link:service-authorization/latest/reference/[Service Authorization Reference,type="documentation"]. ++ +You can run the following command to create an example policy file that allows read-only access to an Amazon S3 bucket. You can optionally store configuration information or a bootstrap script in this bucket, and the containers in your Pod can read the file from the bucket and load it into your application. If you want to create this example policy, copy the following contents to your device. Replace [.replaceable]`my-pod-secrets-bucket` with your bucket name and run the command. ++ +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:8bd1c472-a436-45e3-9074-1a8c53fd2ef0[] +---- +. Create the IAM policy. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam create-policy --policy-name my-policy --policy-document file://my-policy.json +---- + +== Step 2: Create and associate IAM Role + +Create an IAM role and associate it with a Kubernetes service account. You can use either `eksctl` or the {aws} CLI. + +=== Create and associate role (eksctl) + +This `eksctl` command creates a Kubernetes service account in the specified namespace, creates an IAM role (if it doesn't exist) with the specified name, attaches an existing IAM policy ARN to the role, and annotates the service account with the IAM role ARN. Be sure to replace the sample placeholder values in this command with your specific values. To install or update `eksctl`, see https://eksctl.io/installation[Installation] in the `eksctl` documentation. + +[source,bash,subs="verbatim,attributes"] +---- +eksctl create iamserviceaccount --name my-service-account --namespace default --cluster my-cluster --role-name my-role \ + --attach-policy-arn {arn-aws}iam::111122223333:policy/my-policy --approve +---- + +IMPORTANT: If the role or service account already exist, the previous command might fail. `eksctl` has different options that you can provide in those situations. For more information run `eksctl create iamserviceaccount --help`. + + +=== Create and associate role ({aws} CLI) + +If you have an existing Kubernetes service account that you want to assume an IAM role, then you can skip this step. + +. Create a Kubernetes service account. Copy the following contents to your device. Replace [.replaceable]`my-service-account` with your desired name and [.replaceable]`default` with a different namespace, if necessary. If you change [.replaceable]`default`, the namespace must already exist. ++ +[source,yaml,subs="verbatim,attributes"] +---- +cat >my-service-account.yaml <> for more information. ++ +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:5f6a2e0b-00ab-4b9b-aa41-e700048754d7[] +---- +. Create the role. Replace [.replaceable]`my-role` with a name for your IAM role, and [.replaceable]`my-role-description` with a description for your role. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam create-role --role-name my-role --assume-role-policy-document file://trust-relationship.json --description "my-role-description" +---- +. Attach an IAM policy to your role. Replace [.replaceable]`my-role` with the name of your IAM role and [.replaceable]`my-policy` with the name of an existing policy that you created. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam attach-role-policy --role-name my-role --policy-arn={arn-aws}iam::$account_id:policy/my-policy +---- +. Annotate your service account with the Amazon Resource Name (ARN) of the IAM role that you want the service account to assume. Replace [.replaceable]`my-role` with the name of your existing IAM role. Suppose that you allowed a role from a different {aws} account than the account that your cluster is in to assume the role in a previous step. Then, make sure to specify the {aws} account and role from the other account. For more information, see <>. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl annotate serviceaccount -n $namespace $service_account eks.amazonaws.com/role-arn={arn-aws}iam::$account_id:role/my-role +---- +. (Optional) <>. {aws} recommends using a regional {aws} STS endpoint instead of the global endpoint. This reduces latency, provides built-in redundancy, and increases session token validity. + + +[#irsa-confirm-role-configuration] +== Step 3: Confirm configuration +. Confirm that the IAM role's trust policy is configured correctly. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam get-role --role-name my-role --query Role.AssumeRolePolicyDocument +---- ++ +An example output is as follows. ++ +[source,json,subs="verbatim,attributes",role="nocopy"] +---- +include::iam_snippet:ccc47547-ea4d-4a0e-b3fa-2a68aa171d30[] +---- +. Confirm that the policy that you attached to your role in a previous step is attached to the role. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam list-attached-role-policies --role-name my-role --query "AttachedPolicies[].PolicyArn" --output text +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes",role="nocopy"] +---- +{arn-aws}iam::111122223333:policy/my-policy +---- +. Set a variable to store the Amazon Resource Name (ARN) of the policy that you want to use. Replace [.replaceable]`my-policy` with the name of the policy that you want to confirm permissions for. ++ +[source,bash,subs="verbatim,attributes"] +---- +export policy_arn={arn-aws}iam::111122223333:policy/my-policy +---- +. View the default version of the policy. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam get-policy --policy-arn $policy_arn +---- ++ +An example output is as follows. ++ +[source,json,subs="verbatim,attributes",role="nocopy"] +---- +{ + "Policy": { + "PolicyName": "my-policy", + "PolicyId": "EXAMPLEBIOWGLDEXAMPLE", + "Arn": "{arn-aws}iam::111122223333:policy/my-policy", + "Path": "/", + "DefaultVersionId": "v1", + [...] + } +} +---- +. View the policy contents to make sure that the policy includes all the permissions that your Pod needs. If necessary, replace [.replaceable]`1` in the following command with the version that's returned in the previous output. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam get-policy-version --policy-arn $policy_arn --version-id v1 +---- ++ +An example output is as follows. ++ +[source,json,subs="verbatim,attributes",role="nocopy"] +---- +include::iam_snippet:f4c9d9e0-bbed-485d-baf3-15faa91a796a[] +---- ++ +If you created the example policy in a previous step, then your output is the same. If you created a different policy, then the [.replaceable]`example` content is different. +. Confirm that the Kubernetes service account is annotated with the role. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl describe serviceaccount my-service-account -n default +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes",role="nocopy"] +---- +Name: my-service-account +Namespace: default +Annotations: eks.amazonaws.com/role-arn: {arn-aws}iam::111122223333:role/my-role +Image pull secrets: +Mountable secrets: my-service-account-token-qqjfl +Tokens: my-service-account-token-qqjfl +[...] +---- + + +== Next steps + +* <> diff --git a/latest/ug/manage-access/aws-access/configure-sts-endpoint.adoc b/latest/ug/manage-access/aws-access/configure-sts-endpoint.adoc new file mode 100644 index 000000000..c8d1b353f --- /dev/null +++ b/latest/ug/manage-access/aws-access/configure-sts-endpoint.adoc @@ -0,0 +1,109 @@ +include::../../attributes.txt[] + +[.topic] +[#configure-sts-endpoint] += Configure the {aws} Security Token Service endpoint for a service account +:info_titleabbrev: STS endpoints + +If you're using a Kubernetes service account with <>, then you can configure the type of {aws} Security Token Service endpoint that's used by the service account. + +{aws} recommends using the regional {aws} STS endpoints instead of the global endpoint. This reduces latency, provides built-in redundancy, and increases session token validity. The {aws} Security Token Service must be active in the {aws} Region where the Pod is running. Moreover, your application must have built-in redundancy for a different {aws} Region in the event of a failure of the service in the {aws} Region. For more information, see link:IAM/latest/UserGuide/id_credentials_temp_enable-regions.html[Managing {aws} STS in an {aws} Region,type="documentation"] in the IAM User Guide. + + + +* An existing cluster. If you don't have one, you can create one using one of the guides in <>. +* An existing IAM OIDC provider for your cluster. For more information, see <>. +* An existing Kubernetes service account configured for use with the <> feature. + +The following examples all use the aws-node Kubernetes service account used by the <>. You can replace the [.replaceable]`example values` with your own service accounts, Pods, namespaces, and other resources. + +. Select a Pod that uses a service account that you want to change the endpoint for. Determine which {aws} Region that the Pod runs in. Replace [.replaceable]`aws-node-6mfgv` with your Pod name and [.replaceable]`kube-system` with your Pod's namespace. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl describe pod aws-node-6mfgv -n kube-system |grep Node: +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +ip-192-168-79-166.us-west-2/192.168.79.166 +---- ++ +In the previous output, the Pod is running on a node in the us-west-2 {aws} Region. +. Determine the endpoint type that the Pod's service account is using. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl describe pod aws-node-6mfgv -n kube-system |grep AWS_STS_REGIONAL_ENDPOINTS +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +AWS_STS_REGIONAL_ENDPOINTS: regional +---- ++ +If the current endpoint is global, then `global` is returned in the output. If no output is returned, then the default endpoint type is in use and has not been overridden. +. If your cluster or platform version are the same or later than those listed in the table, then you can change the endpoint type used by your service account from the default type to a different type with one of the following commands. Replace [.replaceable]`aws-node` with the name of your service account and [.replaceable]`kube-system` with the namespace for your service account. ++ +** If your default or current endpoint type is global and you want to change it to regional: ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl annotate serviceaccount -n kube-system aws-node eks.amazonaws.com/sts-regional-endpoints=true +---- +// Not using Pods' because the ' character seems to mess up the processing. ++ +If you're using <> to generate pre-signed S3 URLs in your application running in Pods' containers, the format of the URL for regional endpoints is similar to the following example: ++ +[source,none,subs="verbatim,attributes"] +---- +https://bucket.s3.us-west-2.amazonaws.com/path?...&X-Amz-Credential=your-access-key-id/date/us-west-2/s3/aws4_request&... +---- +** If your default or current endpoint type is regional and you want to change it to global: ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl annotate serviceaccount -n kube-system aws-node eks.amazonaws.com/sts-regional-endpoints=false +---- ++ +If your application is explicitly making requests to {aws} STS global endpoints and you don't override the default behavior of using regional endpoints in Amazon EKS clusters, then requests will fail with an error. For more information, see <>. +// Not using Pods' because the ' character seems to mess up the processing. ++ +If you're using <> to generate pre-signed S3 URLs in your application running in Pods' containers, the format of the URL for global endpoints is similar to the following example: ++ +[source,none,subs="verbatim,attributes"] +---- +https://bucket.s3.amazonaws.com/path?...&X-Amz-Credential=your-access-key-id/date/us-west-2/s3/aws4_request&... +---- + ++ +If you have automation that expects the pre-signed URL in a certain format or if your application or downstream dependencies that use pre-signed URLs have expectations for the {aws} Region targeted, then make the necessary changes to use the appropriate {aws} STS endpoint. +. Delete and re-create any existing Pods that are associated with the service account to apply the credential environment variables. The mutating web hook doesn't apply them to Pods that are already running. You can replace [.replaceable]`Pods`, [.replaceable]`kube-system`, and [.replaceable]`-l k8s-app=aws-node` with the information for the Pods that you set your annotation for. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl delete Pods -n kube-system -l k8s-app=aws-node +---- +. Confirm that the all Pods restarted. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get Pods -n kube-system -l k8s-app=aws-node +---- +. View the environment variables for one of the Pods. Verify that the `AWS_STS_REGIONAL_ENDPOINTS` value is what you set it to in a previous step. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl describe pod aws-node-kzbtr -n kube-system |grep AWS_STS_REGIONAL_ENDPOINTS +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +AWS_STS_REGIONAL_ENDPOINTS=regional +---- \ No newline at end of file diff --git a/latest/ug/manage-access/aws-access/cross-account-access.adoc b/latest/ug/manage-access/aws-access/cross-account-access.adoc new file mode 100644 index 000000000..8ffdfe0d3 --- /dev/null +++ b/latest/ug/manage-access/aws-access/cross-account-access.adoc @@ -0,0 +1,78 @@ +include::../../attributes.txt[] + +[.topic] +[#cross-account-access] += Authenticate to another account with IRSA +:info_titleabbrev: Cross-account IAM + +[abstract] +-- +Learn how to configure cross-account IAM permissions for Amazon EKS clusters by creating an identity provider from another account's cluster or using chained AssumeRole operations, enabling secure access to {aws} resources across multiple accounts. +-- + +You can configure cross-account IAM permissions either by creating an identity provider from another account's cluster or by using chained `AssumeRole` operations. In the following examples, _Account A_ owns an Amazon EKS cluster that supports IAM roles for service accounts. Pods that are running on that cluster must assume IAM permissions from _Account B_. + +.Create an identity provider from another account's cluster +==== + +==== + +==== + +In this example, Account A provides Account B with the OpenID Connect (OIDC) issuer URL from their cluster. Account B follows the instructions in <> and <> using the OIDC issuer URL from Account A's cluster. Then, a cluster administrator annotates the service account in Account A's cluster to use the role from Account B ([.replaceable]`444455556666`). + +[source,yaml,subs="verbatim,attributes"] +---- +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: + eks.amazonaws.com/role-arn: {arn-aws}iam::444455556666:role/account-b-role +---- + +==== + +.Use chained `AssumeRole` operations +==== + +==== + +==== + +In this example, Account B creates an IAM policy with the permissions to give to Pods in Account A's cluster. Account B ([.replaceable]`444455556666`) attaches that policy to an IAM role with a trust relationship that allows `AssumeRole` permissions to Account A ([.replaceable]`111122223333`). + +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:539114bb-b8e0-44ba-b4cc-2dd62108ef64[] +---- + +Account A creates a role with a trust policy that gets credentials from the identity provider created with the cluster's OIDC issuer address. + +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:0436d765-c970-4491-887b-65c0f585250e[] +---- + +Account A attaches a policy to that role with the following permissions to assume the role that Account B created. + +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:01ba2728-97d1-4049-a0eb-fa44b1f63ca6[] +---- + +The application code for Pods to assume Account B's role uses two profiles: `account_b_role` and `account_a_role`. The `account_b_role` profile uses the `account_a_role` profile as its source. For the {aws} CLI, the `~/.aws/config` file is similar to the following. + +[source,none,subs="verbatim,attributes"] +---- +[profile account_b_role] +source_profile = account_a_role +role_arn={arn-aws}iam::444455556666:role/account-b-role + +[profile account_a_role] +web_identity_token_file = /var/run/secrets/eks.amazonaws.com/serviceaccount/token +role_arn={arn-aws}iam::111122223333:role/account-a-role +---- + +To specify chained profiles for other {aws} SDKs, consult the documentation for the SDK that you're using. For more information, see link:developer/tools/[Tools to Build on {aws},type="marketing"]. + +==== diff --git a/latest/ug/manage-access/aws-access/enable-iam-roles-for-service-accounts.adoc b/latest/ug/manage-access/aws-access/enable-iam-roles-for-service-accounts.adoc new file mode 100644 index 000000000..d0952380e --- /dev/null +++ b/latest/ug/manage-access/aws-access/enable-iam-roles-for-service-accounts.adoc @@ -0,0 +1,80 @@ +include::../../attributes.txt[] + +[.topic] +[#enable-iam-roles-for-service-accounts] += Create an IAM OIDC provider for your cluster +:info_titleabbrev: IAM OIDC provider + +[abstract] +-- +Learn how to create an {aws} Identity and Access Management OpenID Connect provider for your cluster. +-- + +Your cluster has an https://openid.net/connect/[OpenID Connect] (OIDC) issuer URL associated with it. To use {aws} Identity and Access Management (IAM) roles for service accounts, an IAM OIDC provider must exist for your cluster's OIDC issuer URL. + +== Prerequisites + +* An existing Amazon EKS cluster. To deploy one, see <>. +* Version `2.12.3` or later or version `1.27.160` or later of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device or {aws} CloudShell. To check your current version, use `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or Homebrew for macOS are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the _{aws} Command Line Interface User Guide_. The {aws} CLI version that is installed in {aws} CloudShell might also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the _{aws} CloudShell User Guide_. +* The `kubectl` command line tool is installed on your device or {aws} CloudShell. The version can be the same as or up to one minor version earlier or later than the Kubernetes version of your cluster. For example, if your cluster version is `1.29`, you can use `kubectl` version `1.28`, `1.29`, or `1.30` with it. To install or upgrade `kubectl`, see <>. +* An existing `kubectl` `config` file that contains your cluster configuration. To create a `kubectl` `config` file, see <>. + +You can create an IAM OIDC provider for your cluster using `eksctl` or the {aws-management-console}. + +== Create OIDC provider (eksctl) + +. Version `{eksctl-min-version}` or later of the `eksctl` command line tool installed on your device or {aws} CloudShell. To install or update `eksctl`, see https://eksctl.io/installation[Installation] in the `eksctl` documentation. ++ +. Determine the OIDC issuer ID for your cluster. ++ +Retrieve your cluster's OIDC issuer ID and store it in a variable. Replace `` with your own value. ++ +[source,bash,subs="verbatim,attributes"] +---- +cluster_name= +oidc_id=$(aws eks describe-cluster --name $cluster_name --query "cluster.identity.oidc.issuer" --output text | cut -d '/' -f 5) +echo $oidc_id +---- +. Determine whether an IAM OIDC provider with your cluster's issuer ID is already in your account. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam list-open-id-connect-providers | grep $oidc_id | cut -d "/" -f4 +---- ++ +If output is returned, then you already have an IAM OIDC provider for your cluster and you can skip the next step. If no output is returned, then you must create an IAM OIDC provider for your cluster. +. Create an IAM OIDC identity provider for your cluster with the following command. ++ +[source,bash,subs="verbatim,attributes"] +---- +eksctl utils associate-iam-oidc-provider --cluster $cluster_name --approve +---- ++ +[NOTE] +==== +If you enabled the EKS VPC endpoint, the EKS OIDC service endpoint couldn't be accessed from inside that VPC. Consequently, your operations such as creating an OIDC provider with `eksctl` in the VPC will not work and will result in a timeout. An example error message follows: +---- +** server cant find oidc.eks..amazonaws.com: NXDOMAIN +---- +==== ++ +To complete this step, you can run the command outside the VPC, for example in {aws} CloudShell or on a computer connected to the internet. Alternatively, you can create a split-horizon conditional resolver in the VPC, such as Route 53 Resolver to use a different resolver for the OIDC Issuer URL and not use the VPC DNS for it. For an example of conditional forwarding in CoreDNS, see the https://github.com/aws/containers-roadmap/issues/2038[Amazon EKS feature request] on GitHub. + + +== Create OIDC provider ({aws} Console) + +. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. +. In the left pane, select *Clusters*, and then select the name of your cluster on the *Clusters* page. +. In the *Details* section on the *Overview* tab, note the value of the *OpenID Connect provider URL*. +. Open the IAM console at https://console.aws.amazon.com/iam/. +. In the left navigation pane, choose *Identity Providers* under *Access management*. If a *Provider* is listed that matches the URL for your cluster, then you already have a provider for your cluster. If a provider isn't listed that matches the URL for your cluster, then you must create one. +. To create a provider, choose *Add provider*. +. For *Provider type*, select *OpenID Connect*. +. For *Provider URL*, enter the OIDC provider URL for your cluster. +. For *Audience*, enter `sts.amazonaws.com`. +. (Optional) Add any tags, for example a tag to identify which cluster is for this provider. +. Choose *Add provider*. + + +Next step: +<> diff --git a/latest/ug/manage-access/aws-access/iam-roles-for-service-accounts-minimum-sdk.adoc b/latest/ug/manage-access/aws-access/iam-roles-for-service-accounts-minimum-sdk.adoc new file mode 100644 index 000000000..6fb9d0803 --- /dev/null +++ b/latest/ug/manage-access/aws-access/iam-roles-for-service-accounts-minimum-sdk.adoc @@ -0,0 +1,38 @@ +include::../../attributes.txt[] + +[.topic] +[#iam-roles-for-service-accounts-minimum-sdk] += Use IRSA with the {aws} SDK +:info_titleabbrev: Supported SDKs + +.Using the credentials +To use the credentials from IAM roles for service accounts (IRSA), your code can use any {aws} SDK to create a client for an {aws} service with an SDK, and by default the SDK searches in a chain of locations for {aws} Identity and Access Management credentials to use. The IAM roles for service accounts credentials will be used if you don't specify a credential provider when you create the client or otherwise initialized the SDK. + +This works because IAM roles for service accounts have been added as a step in the default credential chain. If your workloads currently use credentials that are earlier in the chain of credentials, those credentials will continue to be used even if you configure an IAM roles for service accounts for the same workload. + +The SDK automatically exchanges the service account OIDC token for temporary credentials from {aws} Security Token Service by using the `AssumeRoleWithWebIdentity` action. Amazon EKS and this SDK action continue to rotate the temporary credentials by renewing them before they expire. + +When using <>, the containers in your Pods must use an {aws} SDK version that supports assuming an IAM role through an OpenID Connect web identity token file. Make sure that you're using the following versions, or later, for your {aws} SDK: + +* Java (Version 2) – https://github.com/aws/aws-sdk-java-v2/releases/tag/2.10.11[2.10.11] +* Java – https://github.com/aws/aws-sdk-java/releases/tag/1.12.782[1.12.782] +* {aws} SDK for Go v1 – https://github.com/aws/aws-sdk-go/releases/tag/v1.23.13[1.23.13] +* {aws} SDK for Go v2 – All versions support +* Python (Boto3) – https://github.com/boto/boto3/releases/tag/1.9.220[1.9.220] +* Python (botocore) – https://github.com/boto/botocore/releases/tag/1.12.200[1.12.200] +* {aws} CLI – https://github.com/aws/aws-cli/releases/tag/1.16.232[1.16.232] +* Node – https://github.com/aws/aws-sdk-js/releases/tag/v2.525.0[2.525.0] and https://github.com/aws/aws-sdk-js-v3/releases/tag/v3.27.0[3.27.0] +* Ruby – https://github.com/aws/aws-sdk-ruby/blob/version-3/gems/aws-sdk-core/CHANGELOG.md#3580-2019-07-01[3.58.0] +* {cpp} – https://github.com/aws/aws-sdk-cpp/releases/tag/1.7.174[1.7.174] +* .NET – https://github.com/aws/aws-sdk-net/releases/tag/3.3.659.1[3.3.659.1] – You must also include `AWSSDK.SecurityToken`. +* PHP – https://github.com/aws/aws-sdk-php/releases/tag/3.110.7[3.110.7] + +Many popular Kubernetes add-ons, such as the https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler[Cluster Autoscaler], the <>, and the <> support IAM roles for service accounts. + +To ensure that you're using a supported SDK, follow the installation instructions for your preferred SDK at link:tools/[Tools to Build on {aws},type="marketing"] when you build your containers. + +== Considerations + +=== Java + +When using Java, you _must_ include the `sts` module on the classpath. For more information, see https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/auth/credentials/WebIdentityTokenFileCredentialsProvider.html[WebIdentityTokenFileCredentialsProvider] in the Java SDK docs. diff --git a/latest/ug/manage-access/aws-access/iam-roles-for-service-accounts.adoc b/latest/ug/manage-access/aws-access/iam-roles-for-service-accounts.adoc new file mode 100644 index 000000000..666f5c69c --- /dev/null +++ b/latest/ug/manage-access/aws-access/iam-roles-for-service-accounts.adoc @@ -0,0 +1,78 @@ +include::../../attributes.txt[] + +[.topic] +[#iam-roles-for-service-accounts] += IAM roles for service accounts +:info_titleabbrev: Credentials with IRSA + +[abstract] +-- +Learn how applications in your Pods can access {aws} services. +-- + +Applications in a Pod's containers can use an {aws} SDK or the {aws} CLI to make API requests to {aws} services using {aws} Identity and Access Management (IAM) permissions. Applications must sign their {aws} API requests with {aws} credentials. *IAM roles for service accounts (IRSA)* provide the ability to manage credentials for your applications, similar to the way that Amazon EC2 instance profiles provide credentials to Amazon EC2 instances. Instead of creating and distributing your {aws} credentials to the containers or using the Amazon EC2 instance's role, you associate an IAM role with a Kubernetes service account and configure your Pods to use the service account. You can't use IAM roles for service accounts with <>. + +IAM roles for service accounts provide the following benefits: + +* *Least privilege* + – You can scope IAM permissions to a service account, and only Pods that use that service account have access to those permissions. This feature also eliminates the need for third-party solutions such as `kiam` or `kube2iam`. +* *Credential isolation* + – When access to the link:AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html[Amazon EC2 Instance Metadata Service (IMDS),type="documentation"] is restricted, a Pod's containers can only retrieve credentials for the IAM role that's associated with the service account that the container uses. A container never has access to credentials that are used by other containers in other Pods. If IMDS is not restricted, the Pod's containers also have access to the <> and the containers may be able to gain access to credentials of IAM roles of other Pods on the same node. For more information, see link:eks/latest/best-practices/identity-and-access-management.html#_identities_and_credentials_for_eks_pods_recommendations[Restrict access to the instance profile assigned to the worker node,type="documentation"]. + +[NOTE] +==== +Pods configured with `hostNetwork: true` will always have IMDS access, but the {aws} SDKs and CLI will use IRSA credentials when enabled. +==== + +* *Auditability* + – Access and event logging is available through {aws} CloudTrail to help ensure retrospective auditing. + +IMPORTANT: Containers are not a security boundary, and the use of IAM roles for service accounts does not change this. Pods assigned to the same node will share a kernel and potentially other resources depending on your Pod configuration. While Pods running on separate nodes will be isolated at the compute layer, there are node applications that have additional permissions in the Kubernetes API beyond the scope of an individual instance. Some examples are `kubelet`, `kube-proxy`, CSI storage drivers, or your own Kubernetes applications. + +Enable IAM roles for service accounts by completing the following procedures: + +. <> – You only complete this procedure once for each cluster. ++ +[NOTE] +==== +If you enabled the EKS VPC endpoint, the EKS OIDC service endpoint couldn't be accessed from inside that VPC. Consequently, your operations such as creating an OIDC provider with `eksctl` in the VPC will not work and will result in a timeout when attempting to request `https://oidc.eks.[.replaceable]``region``.amazonaws.com`. An example error message follows: + +[source,bash,subs="verbatim,attributes"] +---- +server cant find oidc.eks.region.amazonaws.com: NXDOMAIN +---- + +To complete this step, you can run the command outside the VPC, for example in {aws} CloudShell or on a computer connected to the internet. Alternatively, you can create a split-horizon conditional resolver in the VPC, such as Route 53 Resolver to use a different resolver for the OIDC Issuer URL and not use the VPC DNS for it. For an example of conditional forwarding in CoreDNS, see the https://github.com/aws/containers-roadmap/issues/2038[Amazon EKS feature request] on GitHub. +==== + +. <> – Complete this procedure for each unique set of permissions that you want an application to have. + +. <> – Complete this procedure for each Pod that needs access to {aws} services. + +. <> – Confirm that the workload uses an {aws} SDK of a supported version and that the workload uses the default credential chain. + + +[#irsa-oidc-background] +== IAM, Kubernetes, and OpenID Connect (OIDC) background information + +In 2014, {aws} Identity and Access Management added support for federated identities using OpenID Connect (OIDC). This feature allows you to authenticate {aws} API calls with supported identity providers and receive a valid OIDC JSON web token (JWT). You can pass this token to the {aws} STS `AssumeRoleWithWebIdentity` API operation and receive IAM temporary role credentials. You can use these credentials to interact with any {aws} service, including Amazon S3 and DynamoDB. + +Each JWT token is signed by a signing key pair. The keys are served on the OIDC provider managed by Amazon EKS and the private key rotates every 7 days. Amazon EKS keeps the public keys until they expire. If you connect external OIDC clients, be aware that you need to refresh the signing keys before the public key expires. Learn how to <>. + +Kubernetes has long used service accounts as its own internal identity system. Pods can authenticate with the Kubernetes API server using an auto-mounted token (which was a non-OIDC JWT) that only the Kubernetes API server could validate. These legacy service account tokens don't expire, and rotating the signing key is a difficult process. In Kubernetes version `1.12`, support was added for a new `ProjectedServiceAccountToken` feature. This feature is an OIDC JSON web token that also contains the service account identity and supports a configurable audience. + +Amazon EKS hosts a public OIDC discovery endpoint for each cluster that contains the signing keys for the `ProjectedServiceAccountToken` JSON web tokens so external systems, such as IAM, can validate and accept the OIDC tokens that are issued by Kubernetes. + +include::enable-iam-roles-for-service-accounts.adoc[leveloffset=+1] + +include::associate-service-account-role.adoc[leveloffset=+1] + +include::pod-configuration.adoc[leveloffset=+1] + +include::configure-sts-endpoint.adoc[leveloffset=+1] + +include::cross-account-access.adoc[leveloffset=+1] + +include::iam-roles-for-service-accounts-minimum-sdk.adoc[leveloffset=+1] + +include::irsa-fetch-keys.adoc[leveloffset=+1] diff --git a/latest/ug/clusters/authentication/images b/latest/ug/manage-access/aws-access/images similarity index 100% rename from latest/ug/clusters/authentication/images rename to latest/ug/manage-access/aws-access/images diff --git a/latest/ug/manage-access/aws-access/irsa-fetch-keys.adoc b/latest/ug/manage-access/aws-access/irsa-fetch-keys.adoc new file mode 100644 index 000000000..8803870a3 --- /dev/null +++ b/latest/ug/manage-access/aws-access/irsa-fetch-keys.adoc @@ -0,0 +1,41 @@ +include::../../attributes.txt[] + +[.topic] +[#irsa-fetch-keys] += Fetch signing keys to validate OIDC tokens +:info_titleabbrev: Fetch signing keys + +[abstract] +-- +Discover how to fetch the OIDC public signing keys (JSON Web Key Set) required to validate the ProjectedServiceAccountToken for Amazon EKS clusters, enabling external systems to authenticate with IAM roles for Kubernetes service accounts. +-- + +Kubernetes issues a `ProjectedServiceAccountToken` to each Kubernetes Service Account. This token is an OIDC token, which is further a type of JSON web token (JWT). Amazon EKS hosts a public OIDC endpoint for each cluster that contains the signing keys for the token so external systems can validate it. + +To validate a `ProjectedServiceAccountToken`, you need to fetch the OIDC public signing keys, also called the JSON Web Key Set (JWKS). Use these keys in your application to validate the token. For example, you can use the https://pyjwt.readthedocs.io/en/latest/[PyJWT Python library] to validate tokens using these keys. For more information on the `ProjectedServiceAccountToken`, see <>. + +== Prerequisites + +* An existing {aws} Identity and Access Management (IAM) OpenID Connect (OIDC) provider for your cluster. To determine whether you already have one, or to create one, see <>. +* *{aws} CLI* -- A command line tool for working with {aws} services, including Amazon EKS. For more information, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] in the {aws} Command Line Interface User Guide. After installing the {aws} CLI, we recommend that you also configure it. For more information, see link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the {aws} Command Line Interface User Guide. + +== Procedure + +. Retrieve the OIDC URL for your Amazon EKS cluster using the {aws} CLI. ++ +[source,bash,subs="verbatim,attributes"] +---- +$ aws eks describe-cluster --name my-cluster --query 'cluster.identity.oidc.issuer' +"https://oidc.eks.us-west-2.amazonaws.com/id/8EBDXXXX00BAE" +---- +. Retrieve the public signing key using curl, or a similar tool. The result is a https://www.rfc-editor.org/rfc/rfc7517#section-5[JSON Web Key Set (JWKS)]. ++ +IMPORTANT: Amazon EKS throttles calls to the OIDC endpoint. You should cache the public signing key. Respect the `cache-control` header included in the response. ++ +IMPORTANT: Amazon EKS rotates the OIDC signing key every seven days. ++ +[source,bash,subs="verbatim,attributes"] +---- +$ curl https://oidc.eks.us-west-2.amazonaws.com/id/8EBDXXXX00BAE/keys +{"keys":[{"kty":"RSA","kid":"2284XXXX4a40","use":"sig","alg":"RS256","n":"wklbXXXXMVfQ","e":"AQAB"}]} +---- \ No newline at end of file diff --git a/latest/ug/manage-access/aws-access/pod-configuration.adoc b/latest/ug/manage-access/aws-access/pod-configuration.adoc new file mode 100644 index 000000000..10177039b --- /dev/null +++ b/latest/ug/manage-access/aws-access/pod-configuration.adoc @@ -0,0 +1,127 @@ +include::../../attributes.txt[] + +[.topic] +[#pod-configuration] += Configure Pods to use a Kubernetes service account +:info_titleabbrev: Assign to Pod + +[abstract] +-- +Learn how to configure your Pods to use a Kubernetes service account that you allowed to assume an {aws} Identity and Access Management role. +-- + +If a Pod needs to access {aws} services, then you must configure it to use a Kubernetes service account. The service account must be associated to an {aws} Identity and Access Management (IAM) role that has permissions to access the {aws} services. + + + +* An existing cluster. If you don't have one, you can create one using one of the guides in <>. +* An existing IAM OpenID Connect (OIDC) provider for your cluster. To learn if you already have one or how to create one, see <>. +* An existing Kubernetes service account that's associated with an IAM role. The service account must be annotated with the Amazon Resource Name (ARN) of the IAM role. The role must have an associated IAM policy that contains the permissions that you want your Pods to have to use {aws} services. For more information about how to create the service account and role, and configure them, see <>. +* Version `2.12.3` or later or version `1.27.160` or later of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device or {aws} CloudShell. To check your current version, use `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or Homebrew for macOS are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the _{aws} Command Line Interface User Guide_. The {aws} CLI version that is installed in {aws} CloudShell might also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the _{aws} CloudShell User Guide_. +* The `kubectl` command line tool is installed on your device or {aws} CloudShell. The version can be the same as or up to one minor version earlier or later than the Kubernetes version of your cluster. For example, if your cluster version is `1.29`, you can use `kubectl` version `1.28`, `1.29`, or `1.30` with it. To install or upgrade `kubectl`, see <>. +* An existing `kubectl` `config` file that contains your cluster configuration. To create a `kubectl` `config` file, see <>. +. Use the following command to create a deployment manifest that you can deploy a Pod to confirm configuration with. Replace the example values with your own values. ++ +[source,yaml,subs="verbatim,attributes"] +---- +cat >my-deployment.yaml <>. +.. Confirm that the Pod has a web identity token file mount. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl describe pod my-app-6f4dfff6cb-76cv9 | grep AWS_WEB_IDENTITY_TOKEN_FILE: +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +AWS_WEB_IDENTITY_TOKEN_FILE: /var/run/secrets/eks.amazonaws.com/serviceaccount/token +---- ++ +The `kubelet` requests and stores the token on behalf of the Pod. By default, the `kubelet` refreshes the token if the token is older than 80 percent of its total time to live or older than 24 hours. You can modify the expiration duration for any account other than the default service account by using the settings in your Pod spec. For more information, see https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/#serviceaccount-token-volume-projection[Service Account Token Volume Projection] in the Kubernetes documentation. ++ +The https://github.com/aws/amazon-eks-pod-identity-webhook#amazon-eks-pod-identity-webhook[Amazon EKS Pod Identity Webhook] on the cluster watches for Pods that use a service account with the following annotation: ++ +[source,bash,subs="verbatim,attributes"] +---- +eks.amazonaws.com/role-arn: {arn-aws}iam::111122223333:role/my-role +---- ++ +The webhook applies the previous environment variables to those Pods. Your cluster doesn't need to use the webhook to configure the environment variables and token file mounts. You can manually configure Pods to have these environment variables. The <> look for these environment variables first in the credential chain provider. The role credentials are used for Pods that meet this criteria. +. Confirm that your Pods can interact with the {aws} services using the permissions that you assigned in the IAM policy attached to your role. ++ +NOTE: When a Pod uses {aws} credentials from an IAM role that's associated with a service account, the {aws} CLI or other SDKs in the containers for that Pod use the credentials that are provided by that role. If you don't restrict access to the credentials that are provided to the <>, the Pod still has access to these credentials. For more information, see https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node[Restrict access to the instance profile assigned to the worker node]. ++ +If your Pods can't interact with the services as you expected, complete the following steps to confirm that everything is properly configured. ++ +.. Confirm that your Pods use an {aws} SDK version that supports assuming an IAM role through an OpenID Connect web identity token file. For more information, see <>. +.. Confirm that the deployment is using the service account. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl describe deployment my-app | grep "Service Account" +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +Service Account: my-service-account +---- +.. If your Pods still can't access services, review the <> that are described in <> to confirm that your role and service account are configured properly. \ No newline at end of file diff --git a/latest/ug/manage-access/aws-access/pod-id-abac.adoc b/latest/ug/manage-access/aws-access/pod-id-abac.adoc new file mode 100644 index 000000000..9c79bb14b --- /dev/null +++ b/latest/ug/manage-access/aws-access/pod-id-abac.adoc @@ -0,0 +1,73 @@ +include::../../attributes.txt[] + +[.topic] +[#pod-id-abac] += Grant Pods access to {aws} resources based on tags +:info_titleabbrev: Grant Pods access + +[abstract] +-- +Learn how to use Amazon EKS Pod Identity to attach tags for cluster, namespace, and service account to temporary credentials, enabling attribute-based access control (ABAC) for EKS Pods to {aws} resources based on matching tags. +-- + +Attribute-based access control (ABAC) grants rights to users through policies which combine attributes together. EKS Pod Identity attaches tags to the temporary credentials to each Pod with attributes such as cluster name, namespace, and service account name. These role session tags enable administrators to author a single role that can work across service accounts by allowing access to {aws} resources based on matching tags. By adding support for role session tags, you can enforce tighter security boundaries between clusters, and workloads within clusters, while reusing the same IAM roles and IAM policies. + +== Sample policy with tags +Below is an IAM policy example that grants `s3:GetObject` permissions when the corresponding object is tagged with the EKS cluster name. + +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:ab95d3dc-9d65-470f-9455-53a520c2abe3[] +---- + + +[#pod-id-abac-tags] +== Enable or disable session tags + +EKS Pod Identity adds a pre-defined set of session tags when it assumes the role. These session tags enable administrators to author a single role that can work across resources by allowing access to {aws} resources based on matching tags. + +=== Enable session tags + +Session tags are automatically enabled with EKS Pod Identity--no action is required on your part. By default, EKS Pod Identity attaches a set of predefined tags to your session. To reference these tags in policies, use the syntax `${aws:PrincipalTag/` followed by the tag key. For example, `${aws:PrincipalTag/kubernetes-namespace}`. + +* `eks-cluster-arn` +* `eks-cluster-name` +* `kubernetes-namespace` +* `kubernetes-service-account` +* `kubernetes-pod-name` +* `kubernetes-pod-uid` + +=== Disable session tags + +{aws} compresses inline session policies, managed policy ARNs, and session tags into a packed binary format that has a separate limit. If you receive a `PackedPolicyTooLarge` error indicating the packed binary format has exceeded the size limit, you can attempt to reduce the size by disabling the session tags added by EKS Pod Identity. To disable these session tags, follow these steps: + +. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. +. In the left navigation pane, select *Clusters*, and then select the name of the cluster that you want to modify. +. Choose the *Access* tab. +. In the *Pod Identity associations*, choose the association ID you would like to modify in *Association ID*, then choose *Edit*. +. Under *Session tags*, choose *Disable session tags*. +. Choose *Save changes*. + +[#pod-id-abac-chaining] +== Cross-account tags + +All of the session tags that are added by EKS Pod Identity are _transitive_; the tag keys and values are passed to any `AssumeRole` actions that your workloads use to switch roles into another account. You can use these tags in policies in other accounts to limit access in cross-account scenarios. For more infromation, see link:IAM/latest/UserGuide/id_session-tags.html#id_session-tags_role-chaining[Chaining roles with session tags,type="documentation"] in the _IAM User Guide_. + +[#pod-id-abac-custom-tags] +== Custom tags + +EKS Pod Identity can't add additional custom tags to the `AssumeRole` action that it performs. However, tags that you apply to the IAM role are always available through the same format: `${aws:PrincipalTag/` followed by the key, for example `${aws:PrincipalTag/MyCustomTag}`. + +[NOTE] +==== + +Tags added to the session through the `sts:AssumeRole` request take precedence in the case of conflict. For example, say that: + + + +* Amazon EKS adds a key `eks-cluster-name` and value `my-cluster` to the session when EKS assumes the customer role and +* You add an `eks-cluster-name` tag to the IAM role with the value `my-own-cluster`. + +In this case, the former takes precedence and the value for the `eks-cluster-name` tag will be `my-cluster`. + +==== diff --git a/latest/ug/manage-access/aws-access/pod-id-agent-config-ipv6.adoc b/latest/ug/manage-access/aws-access/pod-id-agent-config-ipv6.adoc new file mode 100644 index 000000000..a83d2c84b --- /dev/null +++ b/latest/ug/manage-access/aws-access/pod-id-agent-config-ipv6.adoc @@ -0,0 +1,85 @@ +include::../../attributes.txt[] + +[.topic] +[#pod-id-agent-config-ipv6] += Disable `IPv6` in the EKS Pod Identity Agent +:info_titleabbrev: Disable IPv6 + +[#pod-id-console] +== {aws-management-console} +. To disable `IPv6` in the EKS Pod Identity Agent, add the following configuration to the *Optional configuration settings* of the EKS Add-on. ++ +.. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. +.. In the left navigation pane, select *Clusters*, and then select the name of the cluster that you want to configure the add-on for. +.. Choose the *Add-ons* tab. +.. Select the box in the top right of the EKS Pod Identity Agent add-on box and then choose *Edit*. +.. On the *Configure EKS Pod Identity Agent* page: ++ +... Select the *Version* that you'd like to use. We recommend that you keep the same version as the previous step, and update the version and configuration in separate actions. +... Expand the *Optional configuration settings*. +... Enter the JSON key `"agent":` and value of a nested JSON object with a key `"additionalArgs":` in *Configuration values*. The resulting text must be a valid JSON object. If this key and value are the only data in the text box, surround the key and value with curly braces `{ }`. The following example shows network policy is enabled: ++ +[source,json,subs="verbatim,attributes"] +---- +{ + "agent": { + "additionalArgs": { + "-b": "169.254.170.23" + } + } +} +---- ++ +This configuration sets the `IPv4` address to be the only address used by the agent. +.. To apply the new configuration by replacing the EKS Pod Identity Agent pods, choose *Save changes*. ++ +Amazon EKS applies changes to the EKS Add-ons by using a _rollout_ of the Kubernetes `DaemonSet` for EKS Pod Identity Agent. You can track the status of the rollout in the *Update history* of the add-on in the {aws-management-console} and with `kubectl rollout status daemonset/eks-pod-identity-agent --namespace kube-system`. ++ +`kubectl rollout` has the following commands: ++ +[source,shell,subs="verbatim,attributes"] +---- +$ kubectl rollout + +history -- View rollout history +pause -- Mark the provided resource as paused +restart -- Restart a resource +resume -- Resume a paused resource +status -- Show the status of the rollout +undo -- Undo a previous rollout +---- ++ +If the rollout takes too long, Amazon EKS will undo the rollout, and a message with the type of *Addon Update* and a status of *Failed* will be added to the *Update history* of the add-on. To investigate any issues, start from the history of the rollout, and run `kubectl logs` on a EKS Pod Identity Agent pod to see the logs of EKS Pod Identity Agent. +. If the new entry in the *Update history* has a status of *Successful*, then the rollout has completed and the add-on is using the new configuration in all of the EKS Pod Identity Agent pods. + +[#pod-id-cli] +== {aws} CLI +. To disable `IPv6` in the EKS Pod Identity Agent, add the following configuration to the *configuration values* of the EKS Add-on. ++ +Run the following {aws} CLI command. Replace `my-cluster` with the name of your cluster and the IAM role ARN with the role that you are using. ++ +[source,shell,subs="verbatim,attributes"] +---- +aws eks update-addon --cluster-name my-cluster --addon-name eks-pod-identity-agent \ + --resolve-conflicts PRESERVE --configuration-values '{"agent":{"additionalArgs": { "-b": "169.254.170.23"}}}' +---- ++ +This configuration sets the `IPv4` address to be the only address used by the agent. ++ +Amazon EKS applies changes to the EKS Add-ons by using a _rollout_ of the Kubernetes DaemonSet for EKS Pod Identity Agent. You can track the status of the rollout in the *Update history* of the add-on in the {aws-management-console} and with `kubectl rollout status daemonset/eks-pod-identity-agent --namespace kube-system`. ++ +`kubectl rollout` has the following commands: ++ +[source,shell,subs="verbatim,attributes"] +---- +kubectl rollout + +history -- View rollout history +pause -- Mark the provided resource as paused +restart -- Restart a resource +resume -- Resume a paused resource +status -- Show the status of the rollout +undo -- Undo a previous rollout +---- ++ +If the rollout takes too long, Amazon EKS will undo the rollout, and a message with the type of *Addon Update* and a status of *Failed* will be added to the *Update history* of the add-on. To investigate any issues, start from the history of the rollout, and run `kubectl logs` on a EKS Pod Identity Agent pod to see the logs of EKS Pod Identity Agent. \ No newline at end of file diff --git a/latest/ug/manage-access/aws-access/pod-id-agent-setup.adoc b/latest/ug/manage-access/aws-access/pod-id-agent-setup.adoc new file mode 100644 index 000000000..65ab06630 --- /dev/null +++ b/latest/ug/manage-access/aws-access/pod-id-agent-setup.adoc @@ -0,0 +1,105 @@ +include::../../attributes.txt[] + +[.topic] +[#pod-id-agent-setup] += Set up the Amazon EKS Pod Identity Agent +:info_titleabbrev: Set up the Agent + +[abstract] +-- +Learn how to set up the EKS Pod Identity Agent for your cluster. +-- + +Amazon EKS Pod Identity associations provide the ability to manage credentials for your applications, similar to the way that Amazon EC2 instance profiles provide credentials to Amazon EC2 instances. + +Amazon EKS Pod Identity provides credentials to your workloads with an additional _EKS Auth_ API and an agent pod that runs on each node. + +[TIP] +==== +You do not need to install the EKS Pod Identity Agent on EKS Auto Mode Clusters. This capability is built into EKS Auto Mode. +==== + + +[#pod-id-agent-considerations] +== Considerations + +* By default, the EKS Pod Identity Agent is pre-installed on EKS Auto Mode clusters. To learn more, see <>. +* By default, the EKS Pod Identity Agent listens on an `IPv4` and `IPv6` address for pods to request credentials. The agent uses the loopback (localhost) IP address `169.254.170.23` for `IPv4` and the localhost IP address `[fd00:ec2::23]` for `IPv6`. +* If you disable `IPv6` addresses, or otherwise prevent localhost `IPv6` IP addresses, the agent can't start. To start the agent on nodes that can't use `IPv6`, follow the steps in <> to disable the `IPv6` configuration. + + +[#pod-id-agent-add-on-create] +== Creating the Amazon EKS Pod Identity Agent + +[#pod-id-agent-prereqs] +=== Agent prerequisites + +* An existing Amazon EKS cluster. To deploy one, see <>. The cluster version and platform version must be the same or later than the versions listed in <>. +* The node role has permissions for the agent to do the `AssumeRoleForPodIdentity` action in the EKS Auth API. You can use the <> or add a custom policy similar to the following: ++ +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:485a9c6f-1ee7-4cec-98cf-0feca8c7576f[] +---- ++ +This action can be limited by tags to restrict which roles can be assumed by pods that use the agent. +* The nodes can reach and download images from Amazon ECR. The container image for the add-on is in the registries listed in <>. ++ +Note that you can change the image location and provide `imagePullSecrets` for EKS add-ons in the *Optional configuration settings* in the {aws-management-console}, and in the `--configuration-values` in the {aws} CLI. +* The nodes can reach the Amazon EKS Auth API. For private clusters, the `eks-auth` endpoint in {aws} PrivateLink is required. + + +=== Setup agent with {aws} console +. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. +. In the left navigation pane, select *Clusters*, and then select the name of the cluster that you want to configure the EKS Pod Identity Agent add-on for. +. Choose the *Add-ons* tab. +. Choose *Get more add-ons*. +. Select the box in the top right of the add-on box for EKS Pod Identity Agent and then choose *Next*. +. On the *Configure selected add-ons settings* page, select any version in the *Version* dropdown list. +. (Optional) Expand *Optional configuration settings* to enter additional configuration. For example, you can provide an alternative container image location and `ImagePullSecrets`. The JSON Schema with accepted keys is shown in *Add-on configuration schema*. ++ +Enter the configuration keys and values in *Configuration values*. +. Choose *Next*. +. Confirm that the EKS Pod Identity Agent pods are running on your cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get pods -n kube-system | grep 'eks-pod-identity-agent' +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +eks-pod-identity-agent-gmqp7 1/1 Running 1 (24h ago) 24h +eks-pod-identity-agent-prnsh 1/1 Running 1 (24h ago) 24h +---- ++ +You can now use EKS Pod Identity associations in your cluster. For more information, see <>. + + +=== Setup agent with {aws} CLI +. Run the following {aws} CLI command. Replace `my-cluster` with the name of your cluster. ++ +[source,shell,subs="verbatim,attributes"] +---- +aws eks create-addon --cluster-name my-cluster --addon-name eks-pod-identity-agent --addon-version v1.0.0-eksbuild.1 +---- ++ +NOTE: The EKS Pod Identity Agent doesn't use the `service-account-role-arn` for _IAM roles for service accounts_. You must provide the EKS Pod Identity Agent with permissions in the node role. +. Confirm that the EKS Pod Identity Agent pods are running on your cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get pods -n kube-system | grep 'eks-pod-identity-agent' +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +eks-pod-identity-agent-gmqp7 1/1 Running 1 (24h ago) 24h +eks-pod-identity-agent-prnsh 1/1 Running 1 (24h ago) 24h +---- ++ +You can now use EKS Pod Identity associations in your cluster. For more information, see <>. \ No newline at end of file diff --git a/latest/ug/manage-access/aws-access/pod-id-assign-target-role.adoc b/latest/ug/manage-access/aws-access/pod-id-assign-target-role.adoc new file mode 100644 index 000000000..4a2bbff10 --- /dev/null +++ b/latest/ug/manage-access/aws-access/pod-id-assign-target-role.adoc @@ -0,0 +1,95 @@ +include::../../attributes.txt[] + +[.topic] +[#pod-id-assign-target-role] += Access {aws} Resources using EKS Pod Identity Target IAM Roles +:info_titleabbrev: Assign Target IAM role + +[abstract] +-- +Learn how to configure account role access for Amazon EKS workloads using Pod Identity. +-- + +When running applications on Amazon Elastic Kubernetes Service (Amazon EKS), you might need to access {aws} resources that exist in different {aws} accounts. This guide shows you how to set up cross account access using EKS Pod Identity, which enables your Kubernetes pods to access other {aws} resources using target roles. + +== Prerequisites + +Before you begin, ensure you have completed the following steps: + +* https://docs.aws.amazon.com/eks/latest/userguide/pod-id-agent-setup.html[Set up the Amazon EKS Pod Identity Agent] +* https://docs.aws.amazon.com/eks/latest/userguide/pod-id-role.html[Create an EKS Pod Identity role] + +== How It Works + +Pod Identity enables applications in your EKS cluster to access {aws} resources across accounts through a process called role chaining. + +When creating a Pod Identity association, you can provide two IAM roles: an link:eks/latest/userguide/pod-id-role.html["EKS Pod Identity role",type="documentation"] in the same account as your EKS cluster and a Target IAM Role from the account containing your {aws} resources you wish to access, (like S3 buckets or RDS Databases). The link:eks/latest/userguide/pod-id-role.html["EKS Pod Identity role",type="documentation"] must be in your EKS cluster's account due to https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_iam-passrole-service.html[IAM PassRole] requirements, while the Target IAM Role can be in any {aws} account. PassRole enables an {aws} entity to delegate role assumption to another service. EKS Pod Identity uses PassRole to connect a role to a Kubernetes service account, requiring both the role and the identity passing it to be in the same {aws} account as the EKS cluster. When your application pod needs to access {aws} resources, it requests credentials from Pod Identity. Pod Identity then automatically performs two role assumptions in sequence: first assuming the link:eks/latest/userguide/pod-id-role.html["EKS Pod Identity role",type="documentation"], then using those credentials to assume the Target IAM Role. This process provides your pod with temporary credentials that have the permissions defined in the target role, allowing secure access to resources in other {aws} accounts. + +== Caching considerations + +Due to caching mechanisms, updates to an IAM role in an existing Pod Identity association may not take effect immediately in the pods running on your EKS cluster. The Pod Identity Agent caches IAM credentials based on the association's configuration at the time the credentials are fetched. If the association includes only an link:eks/latest/userguide/pod-id-role.html["EKS Pod Identity role",type="documentation"] and no Target IAM Role, the cached credentials last for 6 hours. If the association includes both the link:eks/latest/userguide/pod-id-role.html["EKS Pod Identity role",type="documentation"] ARN and a Target IAM Role, the cached credentials last for 59 minutes. Modifying an existing association, such as updating the link:eks/latest/userguide/pod-id-role.html["EKS Pod Identity role",type="documentation"] ARN or adding a Target IAM Role, does not reset the existing cache. As a result, the agent will not recognize updates until the cached credentials refresh. To apply changes sooner, you can recreate the existing pods; otherwise, you will need to wait for the cache to expire. + +== Step 1: Create and associate a Target IAM Role + +In this step, you will establish a secure trust chain by creating and configuring a Target IAM Role. For demonstration, we will create a new Target IAM Role to establish a trust chain between two {aws} accounts: the link:eks/latest/userguide/pod-id-role.html["EKS Pod Identity role",type="documentation"] (e.g., `eks-pod-identity-primary-role`) in the EKS cluster's {aws} account gains permission to assume the Target IAM Role (e.g. `eks-pod-identity-aws-resources`) in your target account, enabling access to {aws} resources like Amazon S3 buckets. + +=== Create the Target IAM Role + +1. Open the link:iam/home[Amazon IAM console,type="console"]. +2. In the top navigation bar, verify that you are signed into the account containing the {aws} resources (like S3 buckets or DynamoDB tables) for your Target IAM Role. +3. In the left navigation pane, choose *Roles*. +4. Choose the *Create role* button, then *{aws} account* under "Trusted entity type." +5. Choose *Another {aws} account*, enter your {aws} account number (the account where your link:eks/latest/userguide/pod-id-role.html["EKS Pod Identity role",type="documentation"] exists), then choose *Next*. +6. Add the permission policies you would like to associate to the role (e.g., AmazonS3FullAccess), then choose *Next*. +7. Enter a role name, such as `MyCustomIAMTargetRole`, then choose *Create role*. + +=== Update the Target IAM Role trust policy + +1. After creating the role, you'll be returned to the *Roles* list. Find and select the new role you created in the previous step (e.g., `MyCustomIAMTargetRole`). +2. Select the *Trust relationships* tab. +3. Click *Edit trust policy* on the right side. +4. In the policy editor, replace the default JSON with your trust policy. Replace the placeholder values for role name and `111122223333` in the IAM role ARN with the {aws} account ID hosting your EKS cluster. You can also optionally use PrincipalTags in the role trust policy to authorize only specific service accounts from a given cluster and namespace to assume your target role . For example: + +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:b376e151-c50d-4924-a12f-839792777e7b[] +---- +The above policy policy lets the role `eks-pod-identeity-primary-role` from {aws} account 111122223333 with the relevant link:eks/latest/userguide/pod-id-abac.html["EKS Pod Identity Session Tags",type="documentation"] assume this role. + +If you link:eks/latest/userguide/pod-id-abac.html#pod-id-abac-tags["Disabled Session Tags",type="documentation"] in your EKS Pod Identity, EKS Pod Identity also sets the `sts:ExternalId` with information about the cluster, namespace, and service account of a pod when assuming a target role. +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:c7e8f150-426a-4a9a-89bc-ff510779774a[] +---- +The above policy helps ensure that only the expected cluster, namespace and service account can assume the target role. + +=== Update the permission policy for EKS Pod Identity role + +In this step, you will update the permission policy of the link:eks/latest/userguide/pod-id-role.html["EKS Pod Identity role",type="documentation"] associated with your Amazon EKS cluster by adding the Target IAM Role ARN as a resource. + +1. Open the https://console.aws.amazon.com/eks/home#/clusters[Amazon EKS console]. +2. In the left navigation pane, select *Clusters*, and then select the name of your EKS cluster. +3. Choose the *Access* tab. +4. Under *Pod Identity associations*, select your link:eks/latest/userguide/pod-id-role.html["EKS Pod Identity role",type="documentation"]. +5. Choose *Permissions*, *Add permissions*, then *Create inline policy*. +6. Choose *JSON* on the right side. +7. In the policy editor, replace the default JSON with your permission policy. Replace the placeholder value for role name and `222233334444` in the IAM role ARN with your Target IAM Role. For example: + +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:8e7b4bdd-52ec-4b08-bd87-b0570c6a0d5c[] +---- + +== Step 2: Associate the Target IAM Role to a Kubernetes service account + +In this step, you will create an association between the Target IAM role and the Kubernetes service account in your EKS cluster. + +1. Open the https://console.aws.amazon.com/eks/home#/clusters[Amazon EKS console]. +2. In the left navigation pane, select *Clusters*, and then select the name of the cluster that you want to add the association to. +3. Choose the *Access* tab. +4. In the *Pod Identity associations*, choose *Create*. +5. Choose the link:eks/latest/userguide/pod-id-role.html["EKS Pod Identity role",type="documentation"] in *IAM role* for your workloads to assume. +6. Choose the Target IAM role in *Target IAM role* that will be assumed by the link:eks/latest/userguide/pod-id-role.html["EKS Pod Identity role",type="documentation"]. +7. In the *Kubernetes namespace* field, enter the name of the namespace where you want to create the association (e.g., `my-app-namespace`). This defines where the service account resides. +8. In the *Kubernetes service account* field, enter the name of the service account (e.g., `my-service-account`) that will use the IAM credentials. This links the IAM role to the service account. +9. Choose *Create* to create the association. diff --git a/latest/ug/manage-access/aws-access/pod-id-association.adoc b/latest/ug/manage-access/aws-access/pod-id-association.adoc new file mode 100644 index 000000000..186653115 --- /dev/null +++ b/latest/ug/manage-access/aws-access/pod-id-association.adoc @@ -0,0 +1,223 @@ +include::../../attributes.txt[] + +[.topic] +[#pod-id-association] += Assign an IAM role to a Kubernetes service account +:info_titleabbrev: Assign IAM role + +[abstract] +-- +Learn how to configure a Kubernetes service account to assume an {aws} IAM role with Amazon EKS Pod Identity for securely accessing {aws} services from your pods. +-- + +This topic covers how to configure a Kubernetes service account to assume an {aws} Identity and Access Management (IAM) role with EKS Pod Identity. Any Pods that are configured to use the service account can then access any {aws} service that the role has permissions to access. + +To create an EKS Pod Identity association, there is only a single step; you create the association in EKS through the {aws-management-console}, {aws} CLI, {aws} SDKs, {aws} CloudFormation and other tools. There isn't any data or metadata about the associations inside the cluster in any Kubernetes objects and you don't add any annotations to the service accounts. + +*Prerequisites* + +* An existing cluster. If you don't have one, you can create one by following one of the guides in <>. +* The IAM principal that is creating the association must have `iam:PassRole`. +* The latest version of the {aws} CLI installed and configured on your device or {aws} CloudShell. You can check your current version with `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or Homebrew for macOS are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the {aws} Command Line Interface User Guide. The {aws} CLI version installed in the {aws} CloudShell may also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the {aws} CloudShell User Guide. +* The `kubectl` command line tool is installed on your device or {aws} CloudShell. The version can be the same as or up to one minor version earlier or later than the Kubernetes version of your cluster. For example, if your cluster version is `1.29`, you can use `kubectl` version `1.28`, `1.29`, or `1.30` with it. To install or upgrade `kubectl`, see <>. +* An existing `kubectl` `config` file that contains your cluster configuration. To create a `kubectl` `config` file, see <>. + + +[#pod-id-association-create] +== Create a Pod Identity association ({aws} Console) + +. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. +. In the left navigation pane, select *Clusters*, and then select the name of the cluster that you want to configure the EKS Pod Identity Agent add-on for. +. Choose the *Access* tab. +. In the *Pod Identity associations*, choose *Create*. +. For the *IAM role*, select the IAM role with the permissions that you want the workload to have. ++ +NOTE: The list only contains roles that have the following trust policy which allows EKS Pod Identity to use them. ++ +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:5e1c4467-9416-42e0-88d1-4133499289b8[] +---- ++ +`sts:AssumeRole` -- EKS Pod Identity uses `AssumeRole` to assume the IAM role before passing the temporary credentials to your pods. ++ +`sts:TagSession` -- EKS Pod Identity uses `TagSession` to include _session tags_ in the requests to {aws} STS. ++ +You can use these tags in the _condition keys_ in the trust policy to restrict which service accounts, namespaces, and clusters can use this role. ++ +For a list of Amazon EKS condition keys, see link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html#amazonelastickubernetesservice-policy-keys[Conditions defined by Amazon Elastic Kubernetes Service,type="documentation"] in the _Service Authorization Reference_. To learn which actions and resources you can use a condition key with, see link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html#amazonelastickubernetesservice-actions-as-permissions[Actions defined by Amazon Elastic Kubernetes Service,type="documentation"]. +. For the *Kubernetes namespace*, select the Kubernetes namespace that contains the service account and workload. Optionally, you can specify a namespace by name that doesn't exist in the cluster. +. For the *Kubernetes service account*, select the Kubernetes service account to use. The manifest for your Kubernetes workload must specify this service account. Optionally, you can specify a service account by name that doesn't exist in the cluster. +. (Optional) For the *Tags*, choose *Add tag* to add metadata in a key and value pair. These tags are applied to the association and can be used in IAM policies. ++ +You can repeat this step to add multiple tags. +. Choose *Create*. + + +== Create a Pod Identity association ({aws} CLI) +. If you want to associate an existing IAM policy to your IAM role, skip to the next step. ++ +Create an IAM policy. You can create your own policy, or copy an {aws} managed policy that already grants some of the permissions that you need and customize it to your specific requirements. For more information, see link:IAM/latest/UserGuide/access_policies_create.html[Creating IAM policies,type="documentation"] in the _IAM User Guide_. ++ +.. Create a file that includes the permissions for the {aws} services that you want your Pods to access. For a list of all actions for all {aws} services, see the link:service-authorization/latest/reference/[Service Authorization Reference,type="documentation"]. ++ +You can run the following command to create an example policy file that allows read-only access to an Amazon S3 bucket. You can optionally store configuration information or a bootstrap script in this bucket, and the containers in your Pod can read the file from the bucket and load it into your application. If you want to create this example policy, copy the following contents to your device. Replace [.replaceable]`my-pod-secrets-bucket` with your bucket name and run the command. ++ +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:1ac501bc-092b-4a4c-8fc9-e802ea247027[] +---- +.. Create the IAM policy. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam create-policy --policy-name my-policy --policy-document file://my-policy.json +---- +. Create an IAM role and associate it with a Kubernetes service account. ++ +.. If you have an existing Kubernetes service account that you want to assume an IAM role, then you can skip this step. ++ +Create a Kubernetes service account. Copy the following contents to your device. Replace [.replaceable]`my-service-account` with your desired name and [.replaceable]`default` with a different namespace, if necessary. If you change [.replaceable]`default`, the namespace must already exist. ++ +[source,yaml,subs="verbatim,attributes"] +---- +cat >my-service-account.yaml <> diff --git a/latest/ug/manage-access/aws-access/pod-id-configure-pods.adoc b/latest/ug/manage-access/aws-access/pod-id-configure-pods.adoc new file mode 100644 index 000000000..459c6daef --- /dev/null +++ b/latest/ug/manage-access/aws-access/pod-id-configure-pods.adoc @@ -0,0 +1,97 @@ +include::../../attributes.txt[] + +[.topic] +[#pod-id-configure-pods] += Configure Pods to access {aws} services with service accounts +:info_titleabbrev: Pod service account + +[abstract] +-- +Learn how to configure Pods to use a Kubernetes service account with an associated IAM role for accessing {aws} services on Amazon EKS. +-- + +If a Pod needs to access {aws} services, then you must configure it to use a Kubernetes service account. The service account must be associated to an {aws} Identity and Access Management (IAM) role that has permissions to access the {aws} services. + +* An existing cluster. If you don't have one, you can create one using one of the guides in <>. +* An existing Kubernetes service account and an EKS Pod Identity association that associates the service account with an IAM role. The role must have an associated IAM policy that contains the permissions that you want your Pods to have to use {aws} services. For more information about how to create the service account and role, and configure them, see <>. +* The latest version of the {aws} CLI installed and configured on your device or {aws} CloudShell. You can check your current version with `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or Homebrew for macOS are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the {aws} Command Line Interface User Guide. The {aws} CLI version installed in the {aws} CloudShell may also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the {aws} CloudShell User Guide. +* The `kubectl` command line tool is installed on your device or {aws} CloudShell. The version can be the same as or up to one minor version earlier or later than the Kubernetes version of your cluster. For example, if your cluster version is `1.29`, you can use `kubectl` version `1.28`, `1.29`, or `1.30` with it. To install or upgrade `kubectl`, see <>. +* An existing `kubectl` `config` file that contains your cluster configuration. To create a `kubectl` `config` file, see <>. +. Use the following command to create a deployment manifest that you can deploy a Pod to confirm configuration with. Replace the example values with your own values. ++ +[source,yaml,subs="verbatim,attributes"] +---- +cat >my-deployment.yaml <>, the Pod still has access to these credentials. For more information, see https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node[Restrict access to the instance profile assigned to the worker node]. ++ +If your Pods can't interact with the services as you expected, complete the following steps to confirm that everything is properly configured. ++ +.. Confirm that your Pods use an {aws} SDK version that supports assuming an IAM role through an EKS Pod Identity association. For more information, see <>. +.. Confirm that the deployment is using the service account. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl describe deployment my-app | grep "Service Account" +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +Service Account: my-service-account +---- \ No newline at end of file diff --git a/latest/ug/manage-access/aws-access/pod-id-how-it-works.adoc b/latest/ug/manage-access/aws-access/pod-id-how-it-works.adoc new file mode 100644 index 000000000..7d5e04846 --- /dev/null +++ b/latest/ug/manage-access/aws-access/pod-id-how-it-works.adoc @@ -0,0 +1,59 @@ +include::../../attributes.txt[] + +[.topic] +[#pod-id-how-it-works] += Understand how EKS Pod Identity works +:info_titleabbrev: How it works + +[abstract] +-- +Learn how Amazon EKS Pod Identity works to provide temporary credentials to your Kubernetes workloads, using an agent running on each node and the {aws} SDKs. +-- + +Amazon EKS Pod Identity associations provide the ability to manage credentials for your applications, similar to the way that Amazon EC2 instance profiles provide credentials to Amazon EC2 instances. + +Amazon EKS Pod Identity provides credentials to your workloads with an additional _EKS Auth_ API and an agent pod that runs on each node. + +In your add-ons, such as _Amazon EKS add-ons_ and self-managed controller, operators, and other add-ons, the author needs to update their software to use the latest {aws} SDKs. For the list of compatibility between EKS Pod Identity and the add-ons produced by Amazon EKS, see the previous section <>. + +[#pod-id-credentials] +== Using EKS Pod Identities in your code + +In your code, you can use the {aws} SDKs to access {aws} services. You write code to create a client for an {aws} service with an SDK, and by default the SDK searches in a chain of locations for {aws} Identity and Access Management credentials to use. After valid credentials are found, the search is stopped. For more information about the default locations used, see the link:sdkref/latest/guide/standardized-credentials.html#credentialProviderChain[Credential provider chain,type="documentation"] in the {aws} SDKs and Tools Reference Guide. + +EKS Pod Identities have been added to the _Container credential provider_ which is searched in a step in the default credential chain. If your workloads currently use credentials that are earlier in the chain of credentials, those credentials will continue to be used even if you configure an EKS Pod Identity association for the same workload. This way you can safely migrate from other types of credentials by creating the association first, before removing the old credentials. + +The container credentials provider provides temporary credentials from an agent that runs on each node. In Amazon EKS, the agent is the Amazon EKS Pod Identity Agent and on Amazon Elastic Container Service the agent is the `amazon-ecs-agent`. The SDKs use environment variables to locate the agent to connect to. + +In contrast, _IAM roles for service accounts_ provides a _web identity_ token that the {aws} SDK must exchange with {aws} Security Token Service by using `AssumeRoleWithWebIdentity`. + +[#pod-id-agent-pod] +== How EKS Pod Identity Agent works with a Pod +. When Amazon EKS starts a new pod that uses a service account with an EKS Pod Identity association, the cluster adds the following content to the Pod manifest: ++ +[source,yaml,subs="verbatim,attributes"] +---- + env: + - name: AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE + value: "/var/run/secrets/pods.eks.amazonaws.com/serviceaccount/eks-pod-identity-token" + - name: AWS_CONTAINER_CREDENTIALS_FULL_URI + value: "http://169.254.170.23/v1/credentials" + volumeMounts: + - mountPath: "/var/run/secrets/pods.eks.amazonaws.com/serviceaccount/" + name: eks-pod-identity-token + volumes: + - name: eks-pod-identity-token + projected: + defaultMode: 420 + sources: + - serviceAccountToken: + audience: pods.eks.amazonaws.com + expirationSeconds: 86400 # 24 hours + path: eks-pod-identity-token +---- +. Kubernetes selects which node to run the pod on. Then, the Amazon EKS Pod Identity Agent on the node uses the link:eks/latest/APIReference/API_auth_AssumeRoleForPodIdentity.html[AssumeRoleForPodIdentity,type="documentation"] action to retrieve temporary credentials from the EKS Auth API. +. The EKS Pod Identity Agent makes these credentials available for the {aws} SDKs that you run inside your containers. +. You use the SDK in your application without specifying a credential provider to use the default credential chain. Or, you specify the container credential provider. For more information about the default locations used, see the link:sdkref/latest/guide/standardized-credentials.html#credentialProviderChain[Credential provider chain,type="documentation"] in the {aws} SDKs and Tools Reference Guide. +. The SDK uses the environment variables to connect to the EKS Pod Identity Agent and retrieve the credentials. ++ +NOTE: If your workloads currently use credentials that are earlier in the chain of credentials, those credentials will continue to be used even if you configure an EKS Pod Identity association for the same workload. \ No newline at end of file diff --git a/latest/ug/manage-access/aws-access/pod-id-minimum-sdk.adoc b/latest/ug/manage-access/aws-access/pod-id-minimum-sdk.adoc new file mode 100644 index 000000000..eda260efb --- /dev/null +++ b/latest/ug/manage-access/aws-access/pod-id-minimum-sdk.adoc @@ -0,0 +1,42 @@ +include::../../attributes.txt[] + +[.topic] +[#pod-id-minimum-sdk] += Use pod identity with the {aws} SDK +:info_titleabbrev: Supported SDKs + +[#pod-id-using-creds] +== Using EKS Pod Identity credentials + +To use the credentials from a EKS Pod Identity association, your code can use any {aws} SDK to create a client for an {aws} service with an SDK, and by default the SDK searches in a chain of locations for {aws} Identity and Access Management credentials to use. The EKS Pod Identity credentials will be used if you don't specify a credential provider when you create the client or otherwise initialized the SDK. + +This works because EKS Pod Identities have been added to the _Container credential provider_ which is searched in a step in the default credential chain. If your workloads currently use credentials that are earlier in the chain of credentials, those credentials will continue to be used even if you configure an EKS Pod Identity association for the same workload. + +For more information about how EKS Pod Identities work, see <>. + +When using <>, the containers in your Pods must use an {aws} SDK version that supports assuming an IAM role from the EKS Pod Identity Agent. Make sure that you're using the following versions, or later, for your {aws} SDK: + + + +* Java (Version 2) – https://github.com/aws/aws-sdk-java-v2/releases/tag/2.21.30[2.21.30] +* Java – https://github.com/aws/aws-sdk-java/releases/tag/1.12.746[1.12.746] +* Go v1 – https://github.com/aws/aws-sdk-go/releases/tag/v1.47.11[v1.47.11] +* Go v2 – https://github.com/aws/aws-sdk-go-v2/releases/tag/release-2023-11-14[release-2023-11-14] +* Python (Boto3) – https://github.com/boto/boto3/releases/tag/1.34.41[1.34.41] +* Python (botocore) – https://github.com/boto/botocore/releases/tag/1.34.41[1.34.41] +* {aws} CLI – https://github.com/aws/aws-cli/releases/tag/1.30.0[1.30.0] ++ +{aws} CLI – https://github.com/aws/aws-cli/releases/tag/2.15.0[2.15.0] +* JavaScript v2 – https://github.com/aws/aws-sdk-js/releases/tag/v2.1550.0[2.1550.0] +* JavaScript v3 – https://github.com/aws/aws-sdk-js-v3/releases/tag/v3.458.0[v3.458.0] +* Kotlin – https://github.com/awslabs/aws-sdk-kotlin/releases/tag/v1.0.1[v1.0.1] +* Ruby – https://github.com/aws/aws-sdk-ruby/blob/version-3/gems/aws-sdk-core/CHANGELOG.md#31880-2023-11-22[3.188.0] +* Rust – https://github.com/awslabs/aws-sdk-rust/releases/tag/release-2024-03-13[release-2024-03-13] +* {cpp} – https://github.com/aws/aws-sdk-cpp/releases/tag/1.11.263[1.11.263] +* .NET – https://github.com/aws/aws-sdk-net/releases/tag/3.7.734.0[3.7.734.0] +* PowerShell – https://www.powershellgallery.com/packages/AWS.Tools.Common/4.1.502[4.1.502] +* PHP – https://github.com/aws/aws-sdk-php/releases/tag/3.287.1[3.289.0] + +To ensure that you're using a supported SDK, follow the installation instructions for your preferred SDK at link:tools/[Tools to Build on {aws},type="marketing"] when you build your containers. + +For a list of add-ons that support EKS Pod Identity, see <>. diff --git a/latest/ug/manage-access/aws-access/pod-id-role.adoc b/latest/ug/manage-access/aws-access/pod-id-role.adoc new file mode 100644 index 000000000..b94fdfa6d --- /dev/null +++ b/latest/ug/manage-access/aws-access/pod-id-role.adoc @@ -0,0 +1,34 @@ +include::../../attributes.txt[] + +[.topic] +[#pod-id-role] += Create IAM role with trust policy required by EKS Pod Identity +:info_titleabbrev: EKS Pod Identity role + +[abstract] +-- +Learn how to configure the IAM trust policy for Amazon EKS Pod Identity to allow Kubernetes pods to assume IAM roles and access {aws} resources securely using Amazon EKS condition keys. +-- + +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:fd9b4d3f-f5db-4638-a213-2a29dc8f391e[] +---- + +*`sts:AssumeRole`*:: +EKS Pod Identity uses `AssumeRole` to assume the IAM role before passing the temporary credentials to your pods. + + +*`sts:TagSession`*:: +EKS Pod Identity uses `TagSession` to include _session tags_ in the requests to {aws} STS. + +*Setting Conditions*:: +You can use these tags in the _condition keys_ in the trust policy to restrict which service accounts, namespaces, and clusters can use this role. For the list of request tags that Pod Identity adds, see <>. ++ +For example, you can restrict which pods can assume the role a Pod Identity IAM Role to a specific `ServiceAccount` and `Namespace` with the following Trust Policy with the added `Condition`: +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:064584d9-099b-4928-a2c8-51558015f494[] +---- + +For a list of Amazon EKS condition keys, see link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html#amazonelastickubernetesservice-policy-keys[Conditions defined by Amazon Elastic Kubernetes Service,type="documentation"] in the _Service Authorization Reference_. To learn which actions and resources you can use a condition key with, see link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html#amazonelastickubernetesservice-actions-as-permissions[Actions defined by Amazon Elastic Kubernetes Service,type="documentation"]. diff --git a/latest/ug/manage-access/aws-access/pod-identities.adoc b/latest/ug/manage-access/aws-access/pod-identities.adoc new file mode 100644 index 000000000..4aea85873 --- /dev/null +++ b/latest/ug/manage-access/aws-access/pod-identities.adoc @@ -0,0 +1,147 @@ +include::../../attributes.txt[] + +[.topic] +[#pod-identities] += Learn how EKS Pod Identity grants pods access to {aws} services +:info_titleabbrev: Pod Identity + +include::pod-id-how-it-works.adoc[leveloffset=+1] + +include::pod-id-agent-setup.adoc[leveloffset=+1] + +include::pod-id-association.adoc[leveloffset=+1] + +include::pod-id-assign-target-role.adoc[leveloffset=+1] + +include::pod-id-configure-pods.adoc[leveloffset=+1] + +include::pod-id-abac.adoc[leveloffset=+1] + +include::pod-id-minimum-sdk.adoc[leveloffset=+1] + +include::pod-id-agent-config-ipv6.adoc[leveloffset=+1] + +include::pod-id-role.adoc[leveloffset=+1] + +[abstract] +-- +Learn how to provide {aws} service access to your Kubernetes workloads with Amazon EKS Pod Identities, offering least privilege access, credential isolation, and auditability for enhanced security. Discover the benefits and considerations of this identity management solution for your Amazon EKS clusters. +-- + +Applications in a Pod's containers can use an {aws} SDK or the {aws} CLI to make API requests to {aws} services using {aws} Identity and Access Management (IAM) permissions. Applications must sign their {aws} API requests with {aws} credentials. + +_EKS Pod Identities_ provide the ability to manage credentials for your applications, similar to the way that Amazon EC2 instance profiles provide credentials to Amazon EC2 instances. Instead of creating and distributing your {aws} credentials to the containers or using the Amazon EC2 instance's role, you associate an IAM role with a Kubernetes service account and configure your Pods to use the service account. + +video::aUjJSorBE70[youtube,align = center,height = 405,fileref = https://www.youtube.com/embed/aUjJSorBE70,width = 720] + +Each EKS Pod Identity association maps a role to a service account in a namespace in the specified cluster. If you have the same application in multiple clusters, you can make identical associations in each cluster without modifying the trust policy of the role. + +If a pod uses a service account that has an association, Amazon EKS sets environment variables in the containers of the pod. The environment variables configure the {aws} SDKs, including the {aws} CLI, to use the EKS Pod Identity credentials. + +[#pod-id-benefits] +== Benefits of EKS Pod Identities + +EKS Pod Identities provide the following benefits: + + + +* *Least privilege* + – You can scope IAM permissions to a service account, and only Pods that use that service account have access to those permissions. This feature also eliminates the need for third-party solutions such as `kiam` or `kube2iam`. +* *Credential isolation* + – When access to the link:AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html[Amazon EC2 Instance Metadata Service (IMDS),type="documentation"] is restricted, a Pod's containers can only retrieve credentials for the IAM role that's associated with the service account that the container uses. A container never has access to credentials that are used by other containers in other Pods. If IMDS is not restricted, the Pod's containers also have access to the <> and the containers may be able to gain access to credentials of IAM roles of other Pods on the same node. For more information, see link:eks/latest/best-practices/identity-and-access-management.html#_identities_and_credentials_for_eks_pods_recommendations[Restrict access to the instance profile assigned to the worker node,type="documentation"]. + +[NOTE] +==== +Pods configured with `hostNetwork: true` will always have IMDS access, but the {aws} SDKs and CLI will use Pod Identity credentials when enabled. +==== + +* *Auditability* + – Access and event logging is available through {aws} CloudTrail to help facilitate retrospective auditing. + +IMPORTANT: Containers are not a security boundary, and the use of Pod Identity does not change this. Pods assigned to the same node will share a kernel and potentially other resources depending on your Pod configuration. While Pods running on separate nodes will be isolated at the compute layer, there are node applications that have additional permissions in the Kubernetes API beyond the scope of an individual instance. Some examples are `kubelet`, `kube-proxy`, CSI storage drivers, or your own Kubernetes applications. + +EKS Pod Identity is a simpler method than <>, as this method doesn't use OIDC identity providers. EKS Pod Identity has the following enhancements: + + + +* *Independent operations* + – In many organizations, creating OIDC identity providers is a responsibility of different teams than administering the Kubernetes clusters. EKS Pod Identity has clean separation of duties, where all configuration of EKS Pod Identity associations is done in Amazon EKS and all configuration of the IAM permissions is done in IAM. +* *Reusability* + – EKS Pod Identity uses a single IAM principal instead of the separate principals for each cluster that IAM roles for service accounts use. Your IAM administrator adds the following principal to the trust policy of any role to make it usable by EKS Pod Identities. ++ +[source,json,subs="verbatim,attributes"] +---- + "Principal": { + "Service": "pods.eks.amazonaws.com" + } +---- +* *Scalability* + -- Each set of temporary credentials are assumed by the EKS Auth service in EKS Pod Identity, instead of each {aws} SDK that you run in each pod. Then, the Amazon EKS Pod Identity Agent that runs on each node issues the credentials to the SDKs. Thus the load is reduced to once for each node and isn't duplicated in each pod. For more details of the process, see <>. + +For more information to compare the two alternatives, see <>. + +[#pod-id-setup-overview] +== Overview of setting up EKS Pod Identities + +Turn on EKS Pod Identities by completing the following procedures: + +. <> -- You only complete this procedure once for each cluster. You do not need to complete this step if EKS Auto Mode is enabled on your cluster. +. <> -- Complete this procedure for each unique set of permissions that you want an application to have. ++ +. <> -- Complete this procedure for each Pod that needs access to {aws} services. +. <> -- Confirm that the workload uses an {aws} SDK of a supported version and that the workload uses the default credential chain. + +[#pod-id-limits] +== Limits + +* Up to 5,000 EKS Pod Identity associations per cluster to map IAM roles to Kubernetes service accounts are supported. + +[#pod-id-considerations] +== Considerations + +* *IAM Role Association*: Each Kubernetes service account in a cluster can be associated with one IAM role from the same {aws} account as the cluster. To change the role, edit the EKS Pod Identity association. For cross-account access, delegate access to the role using IAM roles. To learn more, see link:https://docs.aws.amazon.com/IAM/latest/UserGuide/tutorial_cross-account-with-roles.html[Delegate access across {aws} accounts using IAM roles] in the _IAM User Guide_. +* *EKS Pod Identity Agent*: The Pod Identity Agent is required to use EKS Pod Identity. The agent runs as a Kubernetes `DaemonSet` on cluster nodes, providing credentials only to pods on the same node. It uses the node’s `hostNetwork`, occupying port `80` and `2703` on the link-local address (`169.254.170.23` for IPv4, `[fd00:ec2::23]` for IPv6). If IPv6 is disabled in your cluster, disable IPv6 for the Pod Identity Agent. To learn more, see link:https://docs.aws.amazon.com/eks/latest/userguide/pod-id-agent-config-ipv6.html[Disable IPv6 in the EKS Pod Identity Agent]. +* *Eventual Consistency*: EKS Pod Identity associations are eventually consistent, with potential delays of several seconds after API calls. Avoid creating or updating associations in critical, high-availability code paths. Instead, perform these actions in separate, less frequent initialization or setup routines. To learn more, see link:https://docs.aws.amazon.com/eks/latest/best-practices/sgpp.html[Security Groups Per Pod] in the _EKS Best Practices Guide_. +* *Proxy and Security Group Considerations*: For pods using a proxy, add `169.254.170.23` (IPv4) and `[fd00:ec2::23]` (IPv6) to the `no_proxy/NO_PROXY` environment variables to prevent failed requests to the EKS Pod Identity Agent. If using Security Groups for Pods with the {aws} VPC CNI, set the `ENABLE_POD_ENI` flag to ‘true’ and the `POD_SECURITY_GROUP_ENFORCING_MODE` flag to ‘standard’. To learn more, see link:https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html[Assign security groups to individual Pods]. + +[#pod-id-cluster-versions] +=== EKS Pod Identity cluster versions + +To use EKS Pod Identity, the cluster must have a platform version that is the same or later than the version listed in the following table, or a Kubernetes version that is later than the versions listed in the table. To find the suggested version of the Amazon EKS Pod Identity Agent for a Kubernetes version, see <>. + +[%header,cols="2"] +|=== +|Kubernetes version +|Platform version + +|Kubernetes versions not listed +|All platform versions support + +|`1.28` +|`eks.4` + +|=== + +[#pod-id-restrictions] +=== EKS Pod Identity restrictions + +EKS Pod Identities are available on the following: + + + +* Amazon EKS cluster versions listed in the previous topic <>. +* Worker nodes in the cluster that are Linux Amazon EC2 instances. + +EKS Pod Identities aren't available on the following: + + + +* {aws} Outposts. +* Amazon EKS Anywhere. +* Kubernetes clusters that you create and run on Amazon EC2. The EKS Pod Identity components are only available on Amazon EKS. + +You can't use EKS Pod Identities with: + + +* Pods that run anywhere except Linux Amazon EC2 instances. Linux and Windows pods that run on {aws} Fargate (Fargate) aren't supported. Pods that run on Windows Amazon EC2 instances aren't supported. + diff --git a/latest/ug/manage-access/aws-access/process_all.sh b/latest/ug/manage-access/aws-access/process_all.sh new file mode 100755 index 000000000..4239c9c34 --- /dev/null +++ b/latest/ug/manage-access/aws-access/process_all.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +FILES=( + "pod-id-how-it-works.adoc" + "pod-id-agent-setup.adoc" + "pod-id-association.adoc" + "pod-id-configure-pods.adoc" + "pod-id-abac.adoc" + "pod-id-minimum-sdk.adoc" + "pod-id-agent-config-ipv6.adoc" + "pod-id-role.adoc" +) + + +for file in "${FILES[@]}"; do + if [ -f "$file" ]; then + echo "Processing $file..." + python3 process_doc.py "$file" + else + echo "Warning: $file not found" + fi +done diff --git a/latest/ug/manage-access/aws-access/process_doc.py b/latest/ug/manage-access/aws-access/process_doc.py new file mode 100644 index 000000000..7aa55e909 --- /dev/null +++ b/latest/ug/manage-access/aws-access/process_doc.py @@ -0,0 +1,49 @@ +import sys +import re + +def process_file(filename): + # Read the file + with open(filename, 'r') as f: + lines = f.readlines() + + # Remove first = from lines starting with == but not ==== + processed_lines = [] + for line in lines: + if re.match(r'^==(?!==$).*$', line): + line = re.sub(r'^=', '', line) + processed_lines.append(line) + + # Find the index of the line starting with "= " + section_index = -1 + for i, line in enumerate(processed_lines): + if line.lstrip().startswith('= '): + section_index = i + break + + if section_index == -1: + print(f"Error: No line starting with '= ' found in {filename}") + return + + # Insert all new content at once to avoid index shifting issues + new_lines = processed_lines[:section_index + 1] # Everything up to and including the heading + new_lines.insert(0, "//!!NODE_ROOT
\n") # Add ROOT at start + + # Add the new lines after the heading + new_lines.extend([ + "\n:info_doctype: section\n", + "\ninclude::../../attributes.txt[]\n" + ]) + + # Add the rest of the original content + new_lines.extend(processed_lines[section_index + 1:]) + + # Write back to file + with open(filename, 'w') as f: + f.writelines(new_lines) + +if __name__ == "__main__": + if len(sys.argv) != 2: + print("Usage: python script.py filename") + sys.exit(1) + + process_file(sys.argv[1]) diff --git a/latest/ug/manage-access/aws-access/service-accounts.adoc b/latest/ug/manage-access/aws-access/service-accounts.adoc index 25b1d1160..00c618256 100644 --- a/latest/ug/manage-access/aws-access/service-accounts.adoc +++ b/latest/ug/manage-access/aws-access/service-accounts.adoc @@ -1,19 +1,20 @@ -//!!NODE_ROOT
include::../../attributes.txt[] [.topic] -[[service-accounts,service-accounts.title]] -= Grant Kubernetes workloads access to {aws} using [.noloc]`Kubernetes` Service Accounts -:info_doctype: section -:info_title: Grant Kubernetes workloads access to {aws} using Kubernetes Service Accounts -:info_titleabbrev: Grant workloads access to {aws} +[#service-accounts] += Grant Kubernetes workloads access to {aws} using Kubernetes Service Accounts +:info_titleabbrev: Workload access to {aws} -A [.noloc]`Kubernetes` service account provides an identity for processes that run in a [.noloc]`Pod`. For more information see https://kubernetes.io/docs/reference/access-authn-authz/service-accounts-admin[Managing Service Accounts] in the [.noloc]`Kubernetes` documentation. If your [.noloc]`Pod` needs access to {aws} services, you can map the service account to an {aws} Identity and Access Management identity to grant that access. For more information, see <>. +include::iam-roles-for-service-accounts.adoc[leveloffset=+1] -[[service-account-tokens,service-account-tokens.title]] +include::pod-identities.adoc[leveloffset=+1] + +A Kubernetes service account provides an identity for processes that run in a Pod. For more information see https://kubernetes.io/docs/reference/access-authn-authz/service-accounts-admin[Managing Service Accounts] in the Kubernetes documentation. If your Pod needs access to {aws} services, you can map the service account to an {aws} Identity and Access Management identity to grant that access. For more information, see <> or <>. + +[#service-account-tokens] == Service account tokens -The https://kubernetes.io/docs/reference/access-authn-authz/service-accounts-admin/#bound-service-account-token-volume[BoundServiceAccountTokenVolume] feature is enabled by default in [.noloc]`Kubernetes` versions. This feature improves the security of service account tokens by allowing workloads running on [.noloc]`Kubernetes` to request JSON web tokens that are audience, time, and key bound. Service account tokens have an expiration of one hour. In earlier [.noloc]`Kubernetes` versions, the tokens didn't have an expiration. This means that clients that rely on these tokens must refresh the tokens within an hour. The following https://kubernetes.io/docs/reference/using-api/client-libraries/[Kubernetes client SDKs] refresh tokens automatically within the required time frame: +The https://kubernetes.io/docs/reference/access-authn-authz/service-accounts-admin/#bound-service-account-token-volume[BoundServiceAccountTokenVolume] feature is enabled by default in Kubernetes versions. This feature improves the security of service account tokens by allowing workloads running on Kubernetes to request JSON web tokens that are audience, time, and key bound. Service account tokens have an expiration of one hour. In earlier Kubernetes versions, the tokens didn't have an expiration. This means that clients that rely on these tokens must refresh the tokens within an hour. The following https://kubernetes.io/docs/reference/using-api/client-libraries/[Kubernetes client SDKs] refresh tokens automatically within the required time frame: @@ -23,9 +24,9 @@ The https://kubernetes.io/docs/reference/access-authn-authz/service-accounts-adm * JavaScript version `0.10.3` and later * Ruby `master` branch * Haskell version `0.3.0.0` -* [.noloc]`C#` version `7.0.5` and later +* C# version `7.0.5` and later -If your workload is using an earlier client version, then you must update it. To enable a smooth migration of clients to the newer time-bound service account tokens, [.noloc]`Kubernetes` adds an extended expiry period to the service account token over the default one hour. For Amazon EKS clusters, the extended expiry period is 90 days. Your Amazon EKS cluster's [.noloc]`Kubernetes` API server rejects requests with tokens that are greater than 90 days old. We recommend that you check your applications and their dependencies to make sure that the Kubernetes client SDKs are the same or later than the versions listed previously. +If your workload is using an earlier client version, then you must update it. To enable a smooth migration of clients to the newer time-bound service account tokens, Kubernetes adds an extended expiry period to the service account token over the default one hour. For Amazon EKS clusters, the extended expiry period is 90 days. Your Amazon EKS cluster's Kubernetes API server rejects requests with tokens that are greater than 90 days old. We recommend that you check your applications and their dependencies to make sure that the Kubernetes client SDKs are the same or later than the versions listed previously. When the API server receives requests with tokens that are greater than one hour old, it annotates the API audit log event with `annotations.authentication.k8s.io/stale-token`. The value of the annotation looks like the following example: @@ -34,7 +35,7 @@ When the API server receives requests with tokens that are greater than one hour subject: system:serviceaccount:common:fluent-bit, seconds after warning threshold: 4185802. ---- -If your cluster has <> enabled, then the annotations are in the audit logs. You can use the following link:AmazonCloudWatch/latest/logs/AnalyzingLogData.html[CloudWatch Logs Insights,type="documentation"] query to identify all the [.noloc]`Pods` in your Amazon EKS cluster that are using stale tokens: +If your cluster has <> enabled, then the annotations are in the audit logs. You can use the following link:AmazonCloudWatch/latest/logs/AnalyzingLogData.html[CloudWatch Logs Insights,type="documentation"] query to identify all the Pods in your Amazon EKS cluster that are using stale tokens: [source,bash,subs="verbatim,attributes"] ---- @@ -44,9 +45,9 @@ fields @timestamp |parse @message "subject: *, seconds after warning threshold:*\"" as subject, elapsedtime ---- -The `subject` refers to the service account that the [.noloc]`Pod` used. The `elapsedtime` indicates the elapsed time (in seconds) after reading the latest token. The requests to the API server are denied when the `elapsedtime` exceeds 90 days (7,776,000 seconds). You should proactively update your applications' [.noloc]`Kubernetes` client SDK to use one of the version listed previously that automatically refresh the token. If the service account token used is close to 90 days and you don't have sufficient time to update your client SDK versions before token expiration, then you can terminate existing [.noloc]`Pods` and create new ones. This results in refetching of the service account token, giving you an additional 90 days to update your client version SDKs. +The `subject` refers to the service account that the Pod used. The `elapsedtime` indicates the elapsed time (in seconds) after reading the latest token. The requests to the API server are denied when the `elapsedtime` exceeds 90 days (7,776,000 seconds). You should proactively update your applications' Kubernetes client SDK to use one of the version listed previously that automatically refresh the token. If the service account token used is close to 90 days and you don't have sufficient time to update your client SDK versions before token expiration, then you can terminate existing Pods and create new ones. This results in refetching of the service account token, giving you an additional 90 days to update your client version SDKs. -If the [.noloc]`Pod` is part of a deployment, the suggested way to terminate [.noloc]`Pods` while keeping high availability is to perform a roll out with the following command. Replace [.replaceable]`my-deployment` with the name of your deployment. +If the Pod is part of a deployment, the suggested way to terminate Pods while keeping high availability is to perform a roll out with the following command. Replace [.replaceable]`my-deployment` with the name of your deployment. [source,bash,subs="verbatim,attributes"] ---- @@ -54,42 +55,47 @@ kubectl rollout restart deployment/my-deployment ---- -[[boundserviceaccounttoken-validated-add-on-versions,boundserviceaccounttoken-validated-add-on-versions.title]] +[#boundserviceaccounttoken-validated-add-on-versions] == Cluster add-ons -The following cluster add-ons have been updated to use the [.noloc]`Kubernetes` client SDKs that automatically refetch service account tokens. We recommend making sure that the listed versions, or later versions, are installed on your cluster. +The following cluster add-ons have been updated to use the Kubernetes client SDKs that automatically refetch service account tokens. We recommend making sure that the listed versions, or later versions, are installed on your cluster. -* [.noloc]`Amazon VPC CNI plugin for Kubernetes` and metrics helper plugins version `1.8.0` and later. To check your current version or update it, see <> and https://github.com/aws/amazon-vpc-cni-k8s/blob/master/cmd/cni-metrics-helper/README.md[cni-metrics-helper]. -* [.noloc]`CoreDNS` version `1.8.4` and later. To check your current version or update it, see <>. -* [.noloc]`{aws} Load Balancer Controller` version `2.0.0` and later. To check your current version or update it, see <>. +* Amazon VPC CNI plugin for Kubernetes and metrics helper plugins version `1.8.0` and later. To check your current version or update it, see <> and https://github.com/aws/amazon-vpc-cni-k8s/blob/master/cmd/cni-metrics-helper/README.md[cni-metrics-helper]. +* CoreDNS version `1.8.4` and later. To check your current version or update it, see <>. +* {aws} Load Balancer Controller version `2.0.0` and later. To check your current version or update it, see <>. * A current `kube-proxy` version. To check your current version or update it, see <>. -* {aws} for Fluent Bit version `2.25.0` or later. To update your current version, see https://github.com/aws/aws-for-fluent-bit/releases[Releases] on [.noloc]`GitHub`. +* {aws} for Fluent Bit version `2.25.0` or later. To update your current version, see https://github.com/aws/aws-for-fluent-bit/releases[Releases] on GitHub. * Fluentd image version https://hub.docker.com/r/fluent/fluentd/tags?page=1&name=v1.14.6-1.2[1.14.6-1.2] or later and Fluentd filter plugin for Kubernetes metadata version https://rubygems.org/gems/fluent-plugin-kubernetes_metadata_filter/versions/2.11.1[2.11.1] or later. -[[service-accounts-iam,service-accounts-iam.title]] +[#service-accounts-iam] == Granting {aws} Identity and Access Management permissions to workloads on Amazon Elastic Kubernetes Service clusters -Amazon EKS provides two ways to grant {aws} Identity and Access Management permissions to workloads that run in Amazon EKS clusters: _IAM roles for service accounts_, and _EKS Pod Identities_. +Amazon EKS provides two ways to grant {aws} Identity and Access Management permissions to workloads that run in Amazon EKS clusters: _IAM roles for service accounts_, and _EKS Pod Identities_. *IAM roles for service accounts*:: -_IAM roles for service accounts (IRSA)_ configures Kubernetes applications running on {aws} with fine-grained IAM permissions to access various other {aws} resources such as Amazon S3 buckets, Amazon DynamoDB tables, and more. You can run multiple applications together in the same Amazon EKS cluster, and ensure each application has only the minimum set of permissions that it needs. IRSA was build to support various [.noloc]`Kubernetes` deployment options supported by {aws} such as Amazon EKS, Amazon EKS Anywhere, Red Hat OpenShift Service on {aws}, and self managed [.noloc]`Kubernetes` clusters on Amazon EC2 instances. Thus, IRSA was build using foundational {aws} service like IAM, and did not take any direct dependency on the Amazon EKS service and the EKS API. For more information, see <>. +_IAM roles for service accounts (IRSA)_ configures Kubernetes applications running on {aws} with fine-grained IAM permissions to access various other {aws} resources such as Amazon S3 buckets, Amazon DynamoDB tables, and more. You can run multiple applications together in the same Amazon EKS cluster, and ensure each application has only the minimum set of permissions that it needs. IRSA was build to support various Kubernetes deployment options supported by {aws} such as Amazon EKS, Amazon EKS Anywhere, Red Hat OpenShift Service on {aws}, and self managed Kubernetes clusters on Amazon EC2 instances. Thus, IRSA was build using foundational {aws} service like IAM, and did not take any direct dependency on the Amazon EKS service and the EKS API. For more information, see <>. *EKS Pod Identities*:: -EKS Pod Identity offers cluster administrators a simplified workflow for authenticating applications to access various other {aws} resources such as Amazon S3 buckets, Amazon DynamoDB tables, and more. EKS Pod Identity is for EKS only, and as a result, it simplifies how cluster administrators can configure Kubernetes applications to obtain IAM permissions. These permissions can now be easily configured with fewer steps directly through {aws-management-console}, EKS API, and {aws} CLI, and there isn't any action to take inside the cluster in any [.noloc]`Kubernetes` objects. Cluster administrators don't need to switch between the EKS and IAM services, or use privileged IAM operations to configure permissions required by your applications. IAM roles can now be used across multiple clusters without the need to update the role trust policy when creating new clusters. IAM credentials supplied by EKS Pod Identity include role session tags, with attributes such as cluster name, namespace, service account name. Role session tags enable administrators to author a single role that can work across service accounts by allowing access to {aws} resources based on matching tags. For more information, see <>. +EKS Pod Identity offers cluster administrators a simplified workflow for authenticating applications to access various other {aws} resources such as Amazon S3 buckets, Amazon DynamoDB tables, and more. EKS Pod Identity is for EKS only, and as a result, it simplifies how cluster administrators can configure Kubernetes applications to obtain IAM permissions. These permissions can now be easily configured with fewer steps directly through {aws-management-console}, EKS API, and {aws} CLI, and there isn't any action to take inside the cluster in any Kubernetes objects. Cluster administrators don't need to switch between the EKS and IAM services, or use privileged IAM operations to configure permissions required by your applications. IAM roles can now be used across multiple clusters without the need to update the role trust policy when creating new clusters. IAM credentials supplied by EKS Pod Identity include role session tags, with attributes such as cluster name, namespace, service account name. Role session tags enable administrators to author a single role that can work across service accounts by allowing access to {aws} resources based on matching tags. For more information, see <>. -[[service-accounts-iam-compare,service-accounts-iam-compare.title]] +[#service-accounts-iam-compare] === Comparing EKS Pod Identity and IRSA At a high level, both EKS Pod Identity and IRSA enables you to grant IAM permissions to applications running on Kubernetes clusters. But they are fundamentally different in how you configure them, the limits supported, and features enabled. Below, we compare some of the key facets of both solutions. -[cols="1,1,1", options="header"] +[NOTE] +==== +{aws} recommends using EKS Pod Identities to grant access to {aws} resources to your pods whenever possible. For more information, see <>. +==== + +[%header,cols="3"] |=== |Attribute |EKS Pod Identity @@ -98,1840 +104,29 @@ At a high level, both EKS Pod Identity and IRSA enables you to grant IAM permiss |Role extensibility |You have to setup each role once to establish trust with the newly-introduced Amazon EKS service principal `pods.eks.amazonaws.com`. After this one-time step, you don't need to update the role's trust policy each time that it is used in a new cluster. -|You have to update the IAM role's trust policy with the new EKS cluster [.noloc]`OIDC` provider endpoint each time you want to use the role in a new cluster. +|You have to update the IAM role's trust policy with the new EKS cluster OIDC provider endpoint each time you want to use the role in a new cluster. |Cluster scalability |EKS Pod Identity doesn't require users to setup IAM OIDC provider, so this limit doesn't apply. -|Each EKS cluster has an [.noloc]`OpenID Connect` ([.noloc]`OIDC`) issuer URL associated with it. To use IRSA, a unique [.noloc]`OpenID Connect` provider needs to be created for each EKS cluster in IAM. IAM has a default global limit of 100 [.noloc]`OIDC` providers for each {aws} account. If you plan to have more than 100 EKS clusters for each {aws} account with IRSA, then you will reach the IAM [.noloc]`OIDC` provider limit. +|Each EKS cluster has an OpenID Connect (OIDC) issuer URL associated with it. To use IRSA, a unique OpenID Connect provider needs to be created for each EKS cluster in IAM. IAM has a default global limit of 100 OIDC providers for each {aws} account. If you plan to have more than 100 EKS clusters for each {aws} account with IRSA, then you will reach the IAM OIDC provider limit. |Role scalability |EKS Pod Identity doesn't require users to define trust relationship between IAM role and service account in the trust policy, so this limit doesn't apply. |In IRSA, you define the trust relationship between an IAM role and service account in the role's trust policy. By default, the length of trust policy size is `2048`. This means that you can typically define 4 trust relationships in a single trust policy. While you can get the trust policy length limit increased, you are typically limited to a max of 8 trust relationships within a single trust policy. +|STS API Quota Usage +|EKS Pod Identity simplifies delivery of {aws} credentials to your pods, and does not require your code make calls with the {aws} Security Token Service (STS) directly. The EKS service handles role assumption, and delivers credentials to applications written using the {aws} SDK in your pods without your pods communicating with {aws} STS or using STS API Quota. +|In IRSA, applications written using the {aws} SDK in your pods use tokens to call the `AssumeRoleWithWebIdentity` API on the {aws} Security Token Service (STS). Depending on the logic of your code on the {aws} SDK, it is possible for your code to make unneccesarry calls to {aws} STS and receive throttling errors. + |Role reusability |{aws} STS temporary credentials supplied by EKS Pod Identity include role session tags, such as cluster name, namespace, service account name. Role session tags enable administrators to author a single IAM role that can be used with multiple service accounts, with different effective permission, by allowing access to {aws} resources based on tags attached to them. This is also called attribute-based access control (ABAC). For more information, see <>. |{aws} STS session tags are not supported. You can reuse a role between clusters but every pod receives all of the permissions of the role. |Environments supported |EKS Pod Identity is only available on Amazon EKS. -|IRSA can be used such as Amazon EKS, Amazon EKS Anywhere, Red Hat OpenShift Service on {aws}, and self managed [.noloc]`Kubernetes` clusters on Amazon EC2 instances. +|IRSA can be used such as Amazon EKS, Amazon EKS Anywhere, Red Hat OpenShift Service on {aws}, and self managed Kubernetes clusters on Amazon EC2 instances. |EKS versions supported -|EKS [.noloc]`Kubernetes` versions `1.24` or later. For the specific platform versions, see <>. +|All of the supported EKS cluster versions. For the specific platform versions, see <>. |All of the supported EKS cluster versions. |=== - -[.topic] -[[pod-identities,pod-identities.title]] -== Learn how [.noloc]`EKS Pod Identity` grants pods access to {aws} services - -[abstract] --- -Learn how to provide {aws} service access to your Kubernetes workloads with Amazon EKS Pod Identities, offering least privilege access, credential isolation, and auditability for enhanced security. Discover the benefits and considerations of this identity management solution for your Amazon EKS clusters. --- - -Applications in a Pod's containers can use an {aws} SDK or the {aws} CLI to make API requests to {aws} services using {aws} Identity and Access Management (IAM) permissions. Applications must sign their {aws} API requests with {aws} credentials. - -_EKS Pod Identities_ provide the ability to manage credentials for your applications, similar to the way that Amazon EC2 instance profiles provide credentials to Amazon EC2 instances. Instead of creating and distributing your {aws} credentials to the containers or using the Amazon EC2 instance's role, you associate an IAM role with a [.noloc]`Kubernetes` service account and configure your [.noloc]`Pods` to use the service account. - -video::aUjJSorBE70[youtube,align = center,height = 405,fileref = https://www.youtube.com/embed/aUjJSorBE70,width = 720] - -Each EKS Pod Identity association maps a role to a service account in a namespace in the specified cluster. If you have the same application in multiple clusters, you can make identical associations in each cluster without modifying the trust policy of the role. - -If a pod uses a service account that has an association, Amazon EKS sets environment variables in the containers of the pod. The environment variables configure the {aws} SDKs, including the {aws} CLI, to use the EKS Pod Identity credentials. - -[[pod-id-benefits,pod-id-benefits.title]] -=== Benefits of EKS Pod Identities - -EKS Pod Identities provide the following benefits: - - - -* *Least privilege* - – You can scope IAM permissions to a service account, and only [.noloc]`Pods` that use that service account have access to those permissions. This feature also eliminates the need for third-party solutions such as `kiam` or `kube2iam`. -* *Credential isolation* - – A [.noloc]`Pod's` containers can only retrieve credentials for the IAM role that's associated with the service account that the container uses. A container never has access to credentials that are used by other containers in other [.noloc]`Pods`. When using Pod Identities, the [.noloc]`Pod's` containers also have the permissions assigned to the <>, unless you block [.noloc]`Pod` access to the link:AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html["Amazon EC2 Instance Metadata Service (IMDS)", type="documentation"]. For more information, see https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node[Restrict access to the instance profile assigned to the worker node]. -* *Auditability* - – Access and event logging is available through {aws} CloudTrail to help facilitate retrospective auditing. - -EKS Pod Identity is a simpler method than <>, as this method doesn't use [.noloc]`OIDC` identity providers. EKS Pod Identity has the following enhancements: - - - -* *Independent operations* - – In many organizations, creating [.noloc]`OIDC` identity providers is a responsibility of different teams than administering the [.noloc]`Kubernetes` clusters. EKS Pod Identity has clean separation of duties, where all configuration of EKS Pod Identity associations is done in Amazon EKS and all configuration of the IAM permissions is done in IAM. -* *Reusability* - – EKS Pod Identity uses a single IAM principal instead of the separate principals for each cluster that IAM roles for service accounts use. Your IAM administrator adds the following principal to the trust policy of any role to make it usable by EKS Pod Identities. -+ -[source,json,subs="verbatim,attributes"] ----- - "Principal": { - "Service": "pods.eks.amazonaws.com" - } ----- -* *Scalability* - -- Each set of temporary credentials are assumed by the [.noloc]`EKS Auth` service in EKS Pod Identity, instead of each {aws} SDK that you run in each pod. Then, the Amazon EKS Pod Identity Agent that runs on each node issues the credentials to the SDKs. Thus the load is reduced to once for each node and isn't duplicated in each pod. For more details of the process, see <>. - -For more information to compare the two alternatives, see <>. - -[[pod-id-setup-overview,pod-id-setup-overview.title]] -=== Overview of setting up EKS Pod Identities - -Turn on EKS Pod Identities by completing the following procedures: - -. <> -- You only complete this procedure once for each cluster. You do not need to complete this step if EKS Auto Mode is enabled on your cluster. -. <> -- Complete this procedure for each unique set of permissions that you want an application to have. -+ -. <> -- Complete this procedure for each [.noloc]`Pod` that needs access to {aws} services. -. <> -- Confirm that the workload uses an {aws} SDK of a supported version and that the workload uses the default credential chain. - - -[[pod-id-considerations,pod-id-considerations.title]] -=== EKS Pod Identity considerations - -* You can associate one IAM role to each [.noloc]`Kubernetes` service account in each cluster. You can change which role is mapped to the service account by editing the EKS Pod Identity association. -* You can only associate roles that are in the same {aws} account as the cluster. You can delegate access from another account to the role in this account that you configure for EKS Pod Identities to use. For a tutorial about delegating access and `AssumeRole`, see link:IAM/latest/UserGuide/tutorial_cross-account-with-roles.html[Delegate access across {aws} accounts using IAM roles,type="documentation"] in the _IAM User Guide_. -* The EKS Pod Identity Agent is required. It runs as a [.noloc]`Kubernetes` `DaemonSet` on your nodes and only provides credentials to pods on the node that it runs on. For more information about EKS Pod Identity Agent compatibility, see the following section <>. -* If you are using Security Group for Pods along with Pod Identity Agent, you may need to set the `POD_SECURITY_GROUP_ENFORCING_MODE` Flag for the {aws} VPC CNI. For more information on security group for pods considerations, see <>. -* The EKS Pod Identity Agent uses the `hostNetwork` of the node and it uses port `80` and port `2703` on a link-local address on the node. This address is `169.254.170.23` for [.noloc]`IPv4` and `[fd00:ec2::23]` for [.noloc]`IPv6` clusters. -+ -If you disable `IPv6` addresses, or otherwise prevent localhost `IPv6` IP addresses, the agent can't start. To start the agent on nodes that can't use `IPv6`, follow the steps in <> to disable the `IPv6` configuration. - - -[[pod-id-cluster-versions,pod-id-cluster-versions.title]] -==== EKS Pod Identity cluster versions - -To use EKS Pod Identities, the cluster must have a platform version that is the same or later than the version listed in the following table, or a [.noloc]`Kubernetes` version that is later than the versions listed in the table. - -[cols="1,1", options="header"] -|=== -|Kubernetes version -|Platform version - - -|`1.31` -|`eks.4` - -|`1.30` -|`eks.2` - -|`1.29` -|`eks.1` - -|`1.28` -|`eks.4` - -|`1.27` -|`eks.8` - -|`1.26` -|`eks.9` - -|`1.25` -|`eks.10` - -|`1.24` -|`eks.13` -|=== - -[[pod-id-restrictions,pod-id-restrictions.title]] -==== EKS Pod Identity restrictions - -EKS Pod Identities are available on the following: - - - -* Amazon EKS cluster versions listed in the previous topic <>. -* Worker nodes in the cluster that are Linux Amazon EC2 instances. - -EKS Pod Identities aren't available on the following: - - - -* {aws} Outposts. -* Amazon EKS Anywhere. -* [.noloc]`Kubernetes` clusters that you create and run on Amazon EC2. The EKS Pod Identity components are only available on Amazon EKS. - -You can't use EKS Pod Identities with: - - - -* Pods that run anywhere except Linux Amazon EC2 instances. Linux and Windows pods that run on {aws} Fargate (Fargate) aren't supported. Pods that run on Windows Amazon EC2 instances aren't supported. - - - - -[.topic] -[[pod-id-how-it-works,pod-id-how-it-works.title]] -=== Understand how [.noloc]`EKS Pod Identity` works - -[abstract] --- -Learn how Amazon EKS Pod Identity works to provide temporary credentials to your Kubernetes workloads, using an agent running on each node and the {aws} SDKs. --- - -Amazon EKS Pod Identity associations provide the ability to manage credentials for your applications, similar to the way that Amazon EC2 instance profiles provide credentials to Amazon EC2 instances. - -Amazon EKS Pod Identity provides credentials to your workloads with an additional _EKS Auth_ API and an agent pod that runs on each node. - -In your add-ons, such as _Amazon EKS add-ons_ and self-managed controller, operators, and other add-ons, the author needs to update their software to use the latest {aws} SDKs. For the list of compatibility between EKS Pod Identity and the add-ons produced by Amazon EKS, see the previous section <>. - -[[pod-id-credentials,pod-id-credentials.title]] -==== Using EKS Pod Identities in your code - -In your code, you can use the {aws} SDKs to access {aws} services. You write code to create a client for an {aws} service with an SDK, and by default the SDK searches in a chain of locations for {aws} Identity and Access Management credentials to use. After valid credentials are found, the search is stopped. For more information about the default locations used, see the link:sdkref/latest/guide/standardized-credentials.html#credentialProviderChain[Credential provider chain,type="documentation"] in the {aws} SDKs and Tools Reference Guide. - -EKS Pod Identities have been added to the _Container credential provider_ which is searched in a step in the default credential chain. If your workloads currently use credentials that are earlier in the chain of credentials, those credentials will continue to be used even if you configure an EKS Pod Identity association for the same workload. This way you can safely migrate from other types of credentials by creating the association first, before removing the old credentials. - -The container credentials provider provides temporary credentials from an agent that runs on each node. In Amazon EKS, the agent is the Amazon EKS Pod Identity Agent and on Amazon Elastic Container Service the agent is the `amazon-ecs-agent`. The SDKs use environment variables to locate the agent to connect to. - -In contrast, _IAM roles for service accounts_ provides a _web identity_ token that the {aws} SDK must exchange with {aws} Security Token Service by using `AssumeRoleWithWebIdentity`. - -[[pod-id-agent-pod,pod-id-agent-pod.title]] -==== How EKS Pod Identity Agent works with a [.noloc]`Pod` -. When Amazon EKS starts a new pod that uses a service account with an EKS Pod Identity association, the cluster adds the following content to the [.noloc]`Pod` manifest: -+ -[source,yaml,subs="verbatim,attributes"] ----- - env: - - name: AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE - value: "/var/run/secrets/pods.eks.amazonaws.com/serviceaccount/eks-pod-identity-token" - - name: AWS_CONTAINER_CREDENTIALS_FULL_URI - value: "http://169.254.170.23/v1/credentials" - volumeMounts: - - mountPath: "/var/run/secrets/pods.eks.amazonaws.com/serviceaccount/" - name: eks-pod-identity-token - volumes: - - name: eks-pod-identity-token - projected: - defaultMode: 420 - sources: - - serviceAccountToken: - audience: pods.eks.amazonaws.com - expirationSeconds: 86400 # 24 hours - path: eks-pod-identity-token ----- -. [.noloc]`Kubernetes` selects which node to run the pod on. Then, the Amazon EKS Pod Identity Agent on the node uses the link:eks/latest/APIReference/API_auth_AssumeRoleForPodIdentity.html[AssumeRoleForPodIdentity,type="documentation"] action to retrieve temporary credentials from the EKS Auth API. -. The EKS Pod Identity Agent makes these credentials available for the {aws} SDKs that you run inside your containers. -. You use the SDK in your application without specifying a credential provider to use the default credential chain. Or, you specify the container credential provider. For more information about the default locations used, see the link:sdkref/latest/guide/standardized-credentials.html#credentialProviderChain[Credential provider chain,type="documentation"] in the {aws} SDKs and Tools Reference Guide. -. The SDK uses the environment variables to connect to the EKS Pod Identity Agent and retrieve the credentials. -+ -NOTE: If your workloads currently use credentials that are earlier in the chain of credentials, those credentials will continue to be used even if you configure an EKS Pod Identity association for the same workload. - - -[.topic] -[[pod-id-agent-setup,pod-id-agent-setup.title]] -=== Set up the Amazon EKS Pod Identity Agent - -[abstract] --- -Learn how to set up the EKS Pod Identity Agent for your cluster. --- - -Amazon EKS Pod Identity associations provide the ability to manage credentials for your applications, similar to the way that Amazon EC2 instance profiles provide credentials to Amazon EC2 instances. - -Amazon EKS Pod Identity provides credentials to your workloads with an additional _EKS Auth_ API and an agent pod that runs on each node. - -[TIP] -==== -You do not need to install the EKS Pod Identity Agent on EKS Auto Mode Clusters. This capability is built into EKS Auto Mode. -==== - - -[[pod-id-agent-considerations,pod-id-agent-considerations.title]] -==== Considerations - -* By default, the EKS Pod Identity Agent listens on an `IPv4` and `IPv6` address for pods to request credentials. The agent uses the loopback (localhost) IP address `169.254.170.23` for `IPv4` and the localhost IP address `[fd00:ec2::23]` for `IPv6`. -* If you disable `IPv6` addresses, or otherwise prevent localhost `IPv6` IP addresses, the agent can't start. To start the agent on nodes that can't use `IPv6`, follow the steps in <> to disable the `IPv6` configuration. - - -[[pod-id-agent-add-on-create,pod-id-agent-add-on-create.title]] -==== Creating the Amazon EKS Pod Identity Agent - -[[pod-id-agent-prereqs,pod-id-agent-prereqs.title]] -===== Agent prerequisites - -* An existing Amazon EKS cluster. To deploy one, see <>. The cluster version and platform version must be the same or later than the versions listed in <>. -* The node role has permissions for the agent to do the `AssumeRoleForPodIdentity` action in the EKS Auth API. You can use the <> or add a custom policy similar to the following: -+ -[source,json,subs="verbatim,attributes"] ----- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "eks-auth:AssumeRoleForPodIdentity" - ], - "Resource": "*" - } - ] -} ----- -+ -This action can be limited by tags to restrict which roles can be assumed by pods that use the agent. -* The nodes can reach and download images from Amazon ECR. The container image for the add-on is in the registries listed in <>. -+ -Note that you can change the image location and provide `imagePullSecrets` for EKS add-ons in the *Optional configuration settings* in the {aws-management-console}, and in the `--configuration-values` in the {aws} CLI. -* The nodes can reach the Amazon EKS Auth API. For private clusters, the `eks-auth` endpoint in {aws} PrivateLink is required. - - -===== Setup agent with {aws} console -. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. -. In the left navigation pane, select *Clusters*, and then select the name of the cluster that you want to configure the EKS Pod Identity Agent add-on for. -. Choose the *Add-ons* tab. -. Choose *Get more add-ons*. -. Select the box in the top right of the add-on box for EKS Pod Identity Agent and then choose *Next*. -. On the *Configure selected add-ons settings* page, select any version in the *Version* dropdown list. -. (Optional) Expand *Optional configuration settings* to enter additional configuration. For example, you can provide an alternative container image location and `ImagePullSecrets`. The [.noloc]`JSON Schema` with accepted keys is shown in *Add-on configuration schema*. -+ -Enter the configuration keys and values in *Configuration values*. -. Choose *Next*. -. Confirm that the EKS Pod Identity Agent pods are running on your cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl get pods -n kube-system | grep 'eks-pod-identity-agent' ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -eks-pod-identity-agent-gmqp7 1/1 Running 1 (24h ago) 24h -eks-pod-identity-agent-prnsh 1/1 Running 1 (24h ago) 24h ----- -+ -You can now use EKS Pod Identity associations in your cluster. For more information, see <>. - - -===== Setup agent with {aws} CLI -. Run the following {aws} CLI command. Replace `my-cluster` with the name of your cluster. -+ -[source,shell,subs="verbatim,attributes"] ----- -aws eks create-addon --cluster-name my-cluster --addon-name eks-pod-identity-agent --addon-version v1.0.0-eksbuild.1 ----- -+ -NOTE: The EKS Pod Identity Agent doesn't use the `service-account-role-arn` for _IAM roles for service accounts_. You must provide the EKS Pod Identity Agent with permissions in the node role. -. Confirm that the EKS Pod Identity Agent pods are running on your cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl get pods -n kube-system | grep 'eks-pod-identity-agent' ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -eks-pod-identity-agent-gmqp7 1/1 Running 1 (24h ago) 24h -eks-pod-identity-agent-prnsh 1/1 Running 1 (24h ago) 24h ----- -+ -You can now use EKS Pod Identity associations in your cluster. For more information, see <>. - - -[.topic] -[[pod-id-association,pod-id-association.title]] -=== Assign an [.noloc]`IAM` role to a [.noloc]`Kubernetes` service account - -[abstract] --- -Learn how to configure a Kubernetes service account to assume an {aws} IAM role with Amazon EKS Pod Identity for securely accessing {aws} services from your pods. --- - -This topic covers how to configure a [.noloc]`Kubernetes` service account to assume an {aws} Identity and Access Management (IAM) role with EKS Pod Identity. Any [.noloc]`Pods` that are configured to use the service account can then access any {aws} service that the role has permissions to access. - -To create an EKS Pod Identity association, there is only a single step; you create the association in EKS through the {aws-management-console}, {aws} CLI, {aws} SDKs, {aws} CloudFormation and other tools. There isn't any data or metadata about the associations inside the cluster in any [.noloc]`Kubernetes` objects and you don't add any annotations to the service accounts. - - - -* An existing cluster. If you don't have one, you can create one by following one of the guides in <>. -* The IAM principal that is creating the association must have `iam:PassRole`. -* The latest version of the {aws} CLI installed and configured on your device or {aws} CloudShell. You can check your current version with `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or [.noloc]`Homebrew` for [.noloc]`macOS` are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the {aws} Command Line Interface User Guide. The {aws} CLI version installed in the {aws} CloudShell may also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the {aws} CloudShell User Guide. -* The `kubectl` command line tool is installed on your device or {aws} CloudShell. The version can be the same as or up to one minor version earlier or later than the [.noloc]`Kubernetes` version of your cluster. For example, if your cluster version is `1.29`, you can use `kubectl` version `1.28`, `1.29`, or `1.30` with it. To install or upgrade `kubectl`, see <>. -* An existing `kubectl` `config` file that contains your cluster configuration. To create a `kubectl` `config` file, see <>. - - -[[pod-id-association-create,pod-id-association-create.title]] -==== Create a Pod Identity association ({aws} Console) - -. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. -. In the left navigation pane, select *Clusters*, and then select the name of the cluster that you want to configure the EKS Pod Identity Agent add-on for. -. Choose the *Access* tab. -. In the *Pod Identity associations*, choose *Create*. -. For the *IAM role*, select the IAM role with the permissions that you want the workload to have. -+ -NOTE: The list only contains roles that have the following trust policy which allows EKS Pod Identity to use them. -+ -[source,json,subs="verbatim,attributes"] ----- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "AllowEksAuthToAssumeRoleForPodIdentity", - "Effect": "Allow", - "Principal": { - "Service": "pods.eks.amazonaws.com" - }, - "Action": [ - "sts:AssumeRole", - "sts:TagSession" - ] - } - ] -} ----- -+ -`sts:AssumeRole` -- EKS Pod Identity uses `AssumeRole` to assume the IAM role before passing the temporary credentials to your pods. -+ -`sts:TagSession` -- EKS Pod Identity uses `TagSession` to include _session tags_ in the requests to {aws} STS. -+ -You can use these tags in the _condition keys_ in the trust policy to restrict which service accounts, namespaces, and clusters can use this role. -+ -For a list of Amazon EKS condition keys, see link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html#amazonelastickubernetesservice-policy-keys[Conditions defined by Amazon Elastic Kubernetes Service,type="documentation"] in the _Service Authorization Reference_. To learn which actions and resources you can use a condition key with, see link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html#amazonelastickubernetesservice-actions-as-permissions[Actions defined by Amazon Elastic Kubernetes Service,type="documentation"]. -. For the *[.noloc]`Kubernetes` namespace*, select the [.noloc]`Kubernetes` namespace that contains the service account and workload. Optionally, you can specify a namespace by name that doesn't exist in the cluster. -. For the *[.noloc]`Kubernetes` service account*, select the [.noloc]`Kubernetes` service account to use. The manifest for your [.noloc]`Kubernetes` workload must specify this service account. Optionally, you can specify a service account by name that doesn't exist in the cluster. -. (Optional) For the *Tags*, choose *Add tag* to add metadata in a key and value pair. These tags are applied to the association and can be used in IAM policies. -+ -You can repeat this step to add multiple tags. -. Choose *Create*. - - -==== Create a Pod Identity association ({aws} CLI) -. If you want to associate an existing IAM policy to your IAM role, skip to the next step. -+ -Create an IAM policy. You can create your own policy, or copy an {aws} managed policy that already grants some of the permissions that you need and customize it to your specific requirements. For more information, see link:IAM/latest/UserGuide/access_policies_create.html[Creating IAM policies,type="documentation"] in the _IAM User Guide_. -+ -.. Create a file that includes the permissions for the {aws} services that you want your [.noloc]`Pods` to access. For a list of all actions for all {aws} services, see the link:service-authorization/latest/reference/[Service Authorization Reference,type="documentation"]. -+ -You can run the following command to create an example policy file that allows read-only access to an Amazon S3 bucket. You can optionally store configuration information or a bootstrap script in this bucket, and the containers in your [.noloc]`Pod` can read the file from the bucket and load it into your application. If you want to create this example policy, copy the following contents to your device. Replace [.replaceable]`my-pod-secrets-bucket` with your bucket name and run the command. -+ -[source,json,subs="verbatim,attributes"] ----- -cat >my-policy.json <my-service-account.yaml <trust-relationship.json <> - -[.topic] -[[pod-id-configure-pods,pod-id-configure-pods.title]] -=== Configure [.noloc]`pods` to access {aws} services with service accounts - -[abstract] --- -Learn how to configure Pods to use a Kubernetes service account with an associated IAM role for accessing {aws} services on Amazon EKS. --- - -If a [.noloc]`Pod` needs to access {aws} services, then you must configure it to use a [.noloc]`Kubernetes` service account. The service account must be associated to an {aws} Identity and Access Management (IAM) role that has permissions to access the {aws} services. - - - -* An existing cluster. If you don't have one, you can create one using one of the guides in <>. -* An existing [.noloc]`Kubernetes` service account and an EKS Pod Identity association that associates the service account with an IAM role. The role must have an associated IAM policy that contains the permissions that you want your [.noloc]`Pods` to have to use {aws} services. For more information about how to create the service account and role, and configure them, see <>. -* The latest version of the {aws} CLI installed and configured on your device or {aws} CloudShell. You can check your current version with `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or [.noloc]`Homebrew` for [.noloc]`macOS` are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the {aws} Command Line Interface User Guide. The {aws} CLI version installed in the {aws} CloudShell may also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the {aws} CloudShell User Guide. -* The `kubectl` command line tool is installed on your device or {aws} CloudShell. The version can be the same as or up to one minor version earlier or later than the [.noloc]`Kubernetes` version of your cluster. For example, if your cluster version is `1.29`, you can use `kubectl` version `1.28`, `1.29`, or `1.30` with it. To install or upgrade `kubectl`, see <>. -* An existing `kubectl` `config` file that contains your cluster configuration. To create a `kubectl` `config` file, see <>. -. Use the following command to create a deployment manifest that you can deploy a [.noloc]`Pod` to confirm configuration with. Replace the [.replaceable]`example values` with your own values. -+ -[source,yaml,subs="verbatim,attributes"] ----- -cat >my-deployment.yaml <>, the [.noloc]`Pod` still has access to these credentials. For more information, see https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node[Restrict access to the instance profile assigned to the worker node]. -+ -If your [.noloc]`Pods` can't interact with the services as you expected, complete the following steps to confirm that everything is properly configured. -+ -.. Confirm that your [.noloc]`Pods` use an {aws} SDK version that supports assuming an IAM role through an EKS Pod Identity association. For more information, see <>. -.. Confirm that the deployment is using the service account. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl describe deployment my-app | grep "Service Account" ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -Service Account: my-service-account ----- - - -[.topic] -[[pod-id-abac,pod-id-abac.title]] -=== Grant [.noloc]`pods` access to {aws} resources based on tags - -[abstract] --- -Learn how to use Amazon EKS Pod Identity to attach tags for cluster, namespace, and service account to temporary credentials, enabling attribute-based access control (ABAC) for EKS pods to {aws} resources based on matching tags. --- - -EKS Pod Identity attaches tags to the temporary credentials to each pod with attributes such as cluster name, namespace, service account name. These role session tags enable administrators to author a single role that can work across service accounts by allowing access to {aws} resources based on matching tags. By adding support for role session tags, customers can enforce tighter security boundaries between clusters, and workloads within clusters, while reusing the same IAM roles and IAM policies. - -For example, the following policy allows the `s3:GetObject` action if the object is tagged with the name of the EKS cluster. - -[source,json,subs="verbatim,attributes"] ----- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "s3:ListBucket" - ], - "Resource": "*" - }, - { - "Effect": "Allow", - "Action": [ - "s3:GetObject", - "s3:GetObjectTagging" - ], - "Resource": "*", - "Condition": { - "StringEquals": { - "s3:ExistingObjectTag/eks-cluster-name": "${aws:PrincipalTag/eks-cluster-name}" - } - } - } - ] -} ----- - - -[[pod-id-abac-tags,pod-id-abac-tags.title]] -==== List of session tags added by EKS Pod Identity - -The following list contains all of the keys for tags that are added to the `AssumeRole` request made by Amazon EKS. To use these tags in policies, use `${aws:PrincipalTag/` followed by the key, for example `${aws:PrincipalTag/kubernetes-namespace}`. - - - -* `eks-cluster-arn` -* `eks-cluster-name` -* `kubernetes-namespace` -* `kubernetes-service-account` -* `kubernetes-pod-name` -* `kubernetes-pod-uid` - - -[[pod-id-abac-chaining,pod-id-abac-chaining.title]] -==== Cross-account tags - -All of the session tags that are added by EKS Pod Identity are _transitive_; the tag keys and values are passed to any `AssumeRole` actions that your workloads use to switch roles into another account. You can use these tags in policies in other accounts to limit access in cross-account scenarios. For more infromation, see link:IAM/latest/UserGuide/id_session-tags.html#id_session-tags_role-chaining[Chaining roles with session tags,type="documentation"] in the _IAM User Guide_. - -[[pod-id-abac-custom-tags,pod-id-abac-custom-tags.title]] -==== Custom tags - -EKS Pod Identity can't add additional custom tags to the `AssumeRole` action that it performs. However, tags that you apply to the IAM role are always available though the same format: `${aws:PrincipalTag/` followed by the key, for example `${aws:PrincipalTag/MyCustomTag}`. - -[NOTE] -==== - -Tags added to the session through the `sts:AssumeRole` request take precedence in the case of conflict. For example, say that: - - - -* Amazon EKS adds a key `eks-cluster-name` and value `my-cluster` to the session when EKS assumes the customer role and -* You add an `eks-cluster-name` tag to the IAM role with the value `my-own-cluster`. - -In this case, the former takes precedence and the value for the `eks-cluster-name` tag will be `my-cluster`. - -==== - -[.topic] -[[pod-id-minimum-sdk,pod-id-minimum-sdk.title]] -=== Use pod identity with the {aws} SDK - -[[pod-id-using-creds,pod-id-using-creds.title]] -==== Using EKS Pod Identity credentials - -To use the credentials from a EKS Pod Identity association, your code can use any {aws} SDK to create a client for an {aws} service with an SDK, and by default the SDK searches in a chain of locations for {aws} Identity and Access Management credentials to use. The EKS Pod Identity credentials will be used if you don't specify a credential provider when you create the client or otherwise initialized the SDK. - -This works because EKS Pod Identities have been added to the _Container credential provider_ which is searched in a step in the default credential chain. If your workloads currently use credentials that are earlier in the chain of credentials, those credentials will continue to be used even if you configure an EKS Pod Identity association for the same workload. - -For more information about how EKS Pod Identities work, see <>. - -When using <>, the containers in your [.noloc]`Pods` must use an {aws} SDK version that supports assuming an IAM role from the EKS Pod Identity Agent. Make sure that you're using the following versions, or later, for your {aws} SDK: - - - -* Java (Version 2) – https://github.com/aws/aws-sdk-java-v2/releases/tag/2.21.30[2.21.30] -* Java – https://github.com/aws/aws-sdk-java/releases/tag/1.12.746[1.12.746] -* Go v1 – https://github.com/aws/aws-sdk-go/releases/tag/v1.47.11[v1.47.11] -* Go v2 – https://github.com/aws/aws-sdk-go-v2/releases/tag/release-2023-11-14[release-2023-11-14] -* Python (Boto3) – https://github.com/boto/boto3/releases/tag/1.34.41[1.34.41] -* Python (botocore) – https://github.com/boto/botocore/releases/tag/1.34.41[1.34.41] -* {aws} CLI – https://github.com/aws/aws-cli/releases/tag/1.30.0[1.30.0] -+ -{aws} CLI – https://github.com/aws/aws-cli/releases/tag/2.15.0[2.15.0] -* JavaScript v2 – https://github.com/aws/aws-sdk-js/releases/tag/v2.1550.0[2.1550.0] -* JavaScript v3 – https://github.com/aws/aws-sdk-js-v3/releases/tag/v3.458.0[v3.458.0] -* Kotlin – https://github.com/awslabs/aws-sdk-kotlin/releases/tag/v1.0.1[v1.0.1] -* Ruby – https://github.com/aws/aws-sdk-ruby/blob/version-3/gems/aws-sdk-core/CHANGELOG.md#31880-2023-11-22[3.188.0] -* Rust – https://github.com/awslabs/aws-sdk-rust/releases/tag/release-2024-03-13[release-2024-03-13] -* {cpp} – https://github.com/aws/aws-sdk-cpp/releases/tag/1.11.263[1.11.263] -* .NET – https://github.com/aws/aws-sdk-net/releases/tag/3.7.734.0[3.7.734.0] -* PowerShell – https://www.powershellgallery.com/packages/{aws}.Tools.Common/4.1.502[4.1.502] -* PHP – https://github.com/aws/aws-sdk-php/releases/tag/3.287.1[3.287.1] - -To ensure that you're using a supported SDK, follow the installation instructions for your preferred SDK at link:tools/[Tools to Build on {aws},type="marketing"] when you build your containers. - -For a list of add-ons that support EKS Pod Identity, see <>. - -[.topic] -[[pod-id-agent-config-ipv6,pod-id-agent-config-ipv6.title]] -=== Disable `IPv6` in the EKS Pod Identity Agent - -[[pod-id-console,pod-id-console.title]] -==== {aws-management-console} -. To disable `IPv6` in the EKS Pod Identity Agent, add the following configuration to the *Optional configuration settings* of the EKS Add-on. -+ -.. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. -.. In the left navigation pane, select *Clusters*, and then select the name of the cluster that you want to configure the add-on for. -.. Choose the *Add-ons* tab. -.. Select the box in the top right of the EKS Pod Identity Agent add-on box and then choose *Edit*. -.. On the *Configure EKS Pod Identity Agent* page: -+ -... Select the *Version* that you'd like to use. We recommend that you keep the same version as the previous step, and update the version and configuration in separate actions. -... Expand the *Optional configuration settings*. -... Enter the JSON key `"agent":` and value of a nested JSON object with a key `"additionalArgs":` in *Configuration values*. The resulting text must be a valid JSON object. If this key and value are the only data in the text box, surround the key and value with curly braces `{ }`. The following example shows network policy is enabled: -+ -[source,json,subs="verbatim,attributes"] ----- -{ - "agent": { - "additionalArgs": { - "-b": "169.254.170.23" - } - } -} ----- -+ -This configuration sets the `IPv4` address to be the only address used by the agent. -.. To apply the new configuration by replacing the EKS Pod Identity Agent pods, choose *Save changes*. -+ -Amazon EKS applies changes to the EKS Add-ons by using a _rollout_ of the [.noloc]`Kubernetes` `DaemonSet` for EKS Pod Identity Agent. You can track the status of the rollout in the *Update history* of the add-on in the {aws-management-console} and with `kubectl rollout status daemonset/eks-pod-identity-agent --namespace kube-system`. -+ -`kubectl rollout` has the following commands: -+ -[source,shell,subs="verbatim,attributes"] ----- -$ kubectl rollout - -history -- View rollout history -pause -- Mark the provided resource as paused -restart -- Restart a resource -resume -- Resume a paused resource -status -- Show the status of the rollout -undo -- Undo a previous rollout ----- -+ -If the rollout takes too long, Amazon EKS will undo the rollout, and a message with the type of *Addon Update* and a status of *Failed* will be added to the *Update history* of the add-on. To investigate any issues, start from the history of the rollout, and run `kubectl logs` on a EKS Pod Identity Agent pod to see the logs of EKS Pod Identity Agent. -. If the new entry in the *Update history* has a status of *Successful*, then the rollout has completed and the add-on is using the new configuration in all of the EKS Pod Identity Agent pods. - -[[pod-id-cli,pod-id-cli.title]] -==== {aws} CLI -. To disable `IPv6` in the EKS Pod Identity Agent, add the following configuration to the *configuration values* of the EKS Add-on. -+ -Run the following {aws} CLI command. Replace `my-cluster` with the name of your cluster and the IAM role ARN with the role that you are using. -+ -[source,shell,subs="verbatim,attributes"] ----- -aws eks update-addon --cluster-name my-cluster --addon-name eks-pod-identity-agent \ - --resolve-conflicts PRESERVE --configuration-values '{"agent":{"additionalArgs": { "-b": "169.254.170.23"}}}' ----- -+ -This configuration sets the `IPv4` address to be the only address used by the agent. -+ -Amazon EKS applies changes to the EKS Add-ons by using a _rollout_ of the [.noloc]`Kubernetes` DaemonSet for EKS Pod Identity Agent. You can track the status of the rollout in the *Update history* of the add-on in the {aws-management-console} and with `kubectl rollout status daemonset/eks-pod-identity-agent --namespace kube-system`. -+ -`kubectl rollout` has the following commands: -+ -[source,shell,subs="verbatim,attributes"] ----- -kubectl rollout - -history -- View rollout history -pause -- Mark the provided resource as paused -restart -- Restart a resource -resume -- Resume a paused resource -status -- Show the status of the rollout -undo -- Undo a previous rollout ----- -+ -If the rollout takes too long, Amazon EKS will undo the rollout, and a message with the type of *Addon Update* and a status of *Failed* will be added to the *Update history* of the add-on. To investigate any issues, start from the history of the rollout, and run `kubectl logs` on a EKS Pod Identity Agent pod to see the logs of EKS Pod Identity Agent. - - -[.topic] -[[pod-id-role,pod-id-role.title]] -=== Create [.noloc]`IAM` role with trust policy required by [.noloc]`EKS Pod Identity` - -[abstract] --- -Learn how to configure the IAM trust policy for Amazon EKS Pod Identity to allow Kubernetes pods to assume IAM roles and access {aws} resources securely using Amazon EKS condition keys. --- - -[source,json,subs="verbatim,attributes"] ----- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "AllowEksAuthToAssumeRoleForPodIdentity", - "Effect": "Allow", - "Principal": { - "Service": "pods.eks.amazonaws.com" - }, - "Action": [ - "sts:AssumeRole", - "sts:TagSession" - ] - } - ] -} ----- - -*`sts:AssumeRole`*:: -EKS Pod Identity uses `AssumeRole` to assume the IAM role before passing the temporary credentials to your pods. - - -*`sts:TagSession`*:: -EKS Pod Identity uses `TagSession` to include _session tags_ in the requests to {aws} STS. -+ -You can use these tags in the _condition keys_ in the trust policy to restrict which service accounts, namespaces, and clusters can use this role. -+ -For a list of Amazon EKS condition keys, see link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html#amazonelastickubernetesservice-policy-keys[Conditions defined by Amazon Elastic Kubernetes Service,type="documentation"] in the _Service Authorization Reference_. To learn which actions and resources you can use a condition key with, see link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html#amazonelastickubernetesservice-actions-as-permissions[Actions defined by Amazon Elastic Kubernetes Service,type="documentation"]. - - -[.topic] -[[iam-roles-for-service-accounts,iam-roles-for-service-accounts.title]] -== IAM roles for service accounts - -[abstract] --- -Learn how applications in your [.noloc]`Pods` can access {aws} services. --- - -Applications in a [.noloc]`Pod's` containers can use an {aws} SDK or the {aws} CLI to make API requests to {aws} services using {aws} Identity and Access Management (IAM) permissions. Applications must sign their {aws} API requests with {aws} credentials. IAM roles for service accounts provide the ability to manage credentials for your applications, similar to the way that Amazon EC2 instance profiles provide credentials to Amazon EC2 instances. Instead of creating and distributing your {aws} credentials to the containers or using the Amazon EC2 instance's role, you associate an IAM role with a [.noloc]`Kubernetes` service account and configure your [.noloc]`Pods` to use the service account. You can't use IAM roles for service accounts with <>. - -IAM roles for service accounts provide the following benefits: - -* *Least privilege* - – You can scope IAM permissions to a service account, and only [.noloc]`Pods` that use that service account have access to those permissions. This feature also eliminates the need for third-party solutions such as `kiam` or `kube2iam`. -* *Credential isolation* - – A [.noloc]`Pod's` containers can only retrieve credentials for the IAM role that's associated with the service account that the container uses. A container never has access to credentials that are used by other containers in other [.noloc]`Pods`. When using IAM roles for service accounts, the [.noloc]`Pod's` containers also have the permissions assigned to the <>, unless you block [.noloc]`Pod` access to the link:AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html[Amazon EC2 Instance Metadata Service (IMDS),type="documentation"]. For more information, see https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node[Restrict access to the instance profile assigned to the worker node]. -* *Auditability* - – Access and event logging is available through {aws} CloudTrail to help ensure retrospective auditing. - -Enable IAM roles for service accounts by completing the following procedures: - -. <> – You only complete this procedure once for each cluster. -+ -[NOTE] -==== -If you enabled the EKS VPC endpoint, the EKS OIDC service endpoint couldn't be accessed from inside that VPC. Consequently, your operations such as creating an OIDC provider with `eksctl` in the VPC will not work and will result in a timeout when attempting to request `https://oidc.eks.[.replaceable]``region``.amazonaws.com`. An example error message follows: - -[source,bash,subs="verbatim,attributes"] ----- -server cant find oidc.eks.region.amazonaws.com: NXDOMAIN ----- - -To complete this step, you can run the command outside the VPC, for example in {aws} CloudShell or on a computer connected to the internet. Alternatively, you can create a split-horizon conditional resolver in the VPC, such as Route 53 Resolver to use a different resolver for the OIDC Issuer URL and not use the VPC DNS for it. For an example of conditional forwarding in [.noloc]`CoreDNS`, see the https://github.com/aws/containers-roadmap/issues/2038[Amazon EKS feature request] on [.noloc]`GitHub`. -==== - -. <> – Complete this procedure for each unique set of permissions that you want an application to have. - -. <> – Complete this procedure for each [.noloc]`Pod` that needs access to {aws} services. - -. <> – Confirm that the workload uses an {aws} SDK of a supported version and that the workload uses the default credential chain. - - -[[irsa-oidc-background,irsa-oidc-background.title]] -=== IAM, [.noloc]`Kubernetes`, and [.noloc]`OpenID Connect` ([.noloc]`OIDC`) background information - -In 2014, {aws} Identity and Access Management added support for federated identities using [.noloc]`OpenID Connect` ([.noloc]`OIDC`). This feature allows you to authenticate {aws} API calls with supported identity providers and receive a valid [.noloc]`OIDC` [.noloc]`JSON` web token ([.noloc]`JWT`). You can pass this token to the {aws} STS `AssumeRoleWithWebIdentity` API operation and receive IAM temporary role credentials. You can use these credentials to interact with any {aws} service, including Amazon S3 and DynamoDB. - -Each JWT token is signed by a signing key pair. The keys are served on the OIDC provider managed by Amazon EKS and the private key rotates every 7 days. Amazon EKS keeps the public keys until they expire. If you connect external OIDC clients, be aware that you need to refresh the signing keys before the public key expires. Learn how to <>. - -[.noloc]`Kubernetes` has long used service accounts as its own internal identity system. [.noloc]`Pods` can authenticate with the [.noloc]`Kubernetes` API server using an auto-mounted token (which was a non-[.noloc]`OIDC` [.noloc]`JWT`) that only the [.noloc]`Kubernetes` API server could validate. These legacy service account tokens don't expire, and rotating the signing key is a difficult process. In [.noloc]`Kubernetes` version `1.12`, support was added for a new `ProjectedServiceAccountToken` feature. This feature is an [.noloc]`OIDC` [.noloc]`JSON` web token that also contains the service account identity and supports a configurable audience. - -Amazon EKS hosts a public [.noloc]`OIDC` discovery endpoint for each cluster that contains the signing keys for the `ProjectedServiceAccountToken` [.noloc]`JSON` web tokens so external systems, such as IAM, can validate and accept the [.noloc]`OIDC` tokens that are issued by [.noloc]`Kubernetes`. - -[.topic] -[[enable-iam-roles-for-service-accounts,enable-iam-roles-for-service-accounts.title]] -=== Create an IAM [.noloc]`OIDC` provider for your cluster - -[abstract] --- -Learn how to create an {aws} Identity and Access Management [.noloc]`OpenID Connect` provider for your cluster. --- - -Your cluster has an https://openid.net/connect/[OpenID Connect] ([.noloc]`OIDC`) issuer URL associated with it. To use {aws} Identity and Access Management (IAM) roles for service accounts, an IAM [.noloc]`OIDC` provider must exist for your cluster's [.noloc]`OIDC` issuer URL. - - - -* An existing Amazon EKS cluster. To deploy one, see <>. -* Version `2.12.3` or later or version `1.27.160` or later of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device or {aws} CloudShell. To check your current version, use `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or [.noloc]`Homebrew` for [.noloc]`macOS` are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the _{aws} Command Line Interface User Guide_. The {aws} CLI version that is installed in {aws} CloudShell might also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the _{aws} CloudShell User Guide_. -* The `kubectl` command line tool is installed on your device or {aws} CloudShell. The version can be the same as or up to one minor version earlier or later than the [.noloc]`Kubernetes` version of your cluster. For example, if your cluster version is `1.29`, you can use `kubectl` version `1.28`, `1.29`, or `1.30` with it. To install or upgrade `kubectl`, see <>. -* An existing `kubectl` `config` file that contains your cluster configuration. To create a `kubectl` `config` file, see <>. - -You can create an IAM [.noloc]`OIDC` provider for your cluster using `eksctl` or the {aws-management-console}. - -==== Create OIDC provider (eksctl) - -. Version `{eksctl-min-version}` or later of the `eksctl` command line tool installed on your device or {aws} CloudShell. To install or update `eksctl`, see https://eksctl.io/installation[Installation] in the `eksctl` documentation. -+ -. Determine the [.noloc]`OIDC` issuer ID for your cluster. -+ -Retrieve your cluster's [.noloc]`OIDC` issuer ID and store it in a variable. Replace [.replaceable]`my-cluster` with your own value. -+ -[source,bash,subs="verbatim,attributes"] ----- -cluster_name=my-cluster ----- -[source,bash,subs="verbatim,attributes"] ----- -oidc_id=$(aws eks describe-cluster --name $cluster_name --query "cluster.identity.oidc.issuer" --output text | cut -d '/' -f 5) ----- -[source,bash,subs="verbatim,attributes"] ----- -echo $oidc_id ----- -. Determine whether an IAM [.noloc]`OIDC` provider with your cluster's issuer ID is already in your account. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws iam list-open-id-connect-providers | grep $oidc_id | cut -d "/" -f4 ----- -+ -If output is returned, then you already have an IAM [.noloc]`OIDC` provider for your cluster and you can skip the next step. If no output is returned, then you must create an IAM [.noloc]`OIDC` provider for your cluster. -. Create an IAM [.noloc]`OIDC` identity provider for your cluster with the following command. -+ -[source,bash,subs="verbatim,attributes"] ----- -eksctl utils associate-iam-oidc-provider --cluster $cluster_name --approve ----- -+ -NOTE: If you enabled the EKS VPC endpoint, the EKS OIDC service endpoint couldn't be accessed from inside that VPC. Consequently, your operations such as creating an OIDC provider with `eksctl` in the VPC will not work and will result in a timeout when attempting to request `https://oidc.eks.[.replaceable]``region``.amazonaws.com`. An example error message follows: - -[source,bash,subs="verbatim,attributes"] ----- -** server cant find oidc.eks.region.amazonaws.com: NXDOMAIN ----- - -To complete this step, you can run the command outside the VPC, for example in {aws} CloudShell or on a computer connected to the internet. Alternatively, you can create a split-horizon conditional resolver in the VPC, such as Route 53 Resolver to use a different resolver for the OIDC Issuer URL and not use the VPC DNS for it. For an example of conditional forwarding in [.noloc]`CoreDNS`, see the https://github.com/aws/containers-roadmap/issues/2038[Amazon EKS feature request] on [.noloc]`GitHub`. - - -==== Create OIDC provider ({aws} Console) - -. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. -. In the left pane, select *Clusters*, and then select the name of your cluster on the *Clusters* page. -. In the *Details* section on the *Overview* tab, note the value of the *OpenID Connect provider URL*. -. Open the IAM console at https://console.aws.amazon.com/iam/. -. In the left navigation pane, choose *Identity Providers* under *Access management*. If a *Provider* is listed that matches the URL for your cluster, then you already have a provider for your cluster. If a provider isn't listed that matches the URL for your cluster, then you must create one. -. To create a provider, choose *Add provider*. -. For *Provider type*, select *[.noloc]`OpenID Connect`*. -. For *Provider URL*, enter the [.noloc]`OIDC` provider URL for your cluster. -. For *Audience*, enter `sts.amazonaws.com`. -. (Optional) Add any tags, for example a tag to identify which cluster is for this provider. -. Choose *Add provider*. - - -Next step: -<> - -[.topic] -[[associate-service-account-role,associate-service-account-role.title]] -=== Assign [.noloc]`IAM` roles to [.noloc]`Kubernetes` service accounts - -[abstract] --- -Discover how to configure a Kubernetes service account to assume an IAM role, enabling Pods to securely access {aws} services with granular permissions. --- - -This topic covers how to configure a [.noloc]`Kubernetes` service account to assume an {aws} Identity and Access Management (IAM) role. Any [.noloc]`Pods` that are configured to use the service account can then access any {aws} service that the role has permissions to access. - -==== Prerequisites - -* An existing cluster. If you don't have one, you can create one by following one of the guides in <>. -* An existing IAM [.noloc]`OpenID Connect` ([.noloc]`OIDC`) provider for your cluster. To learn if you already have one or how to create one, see <>. -* Version `2.12.3` or later or version `1.27.160` or later of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device or {aws} CloudShell. To check your current version, use `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or [.noloc]`Homebrew` for [.noloc]`macOS` are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the _{aws} Command Line Interface User Guide_. The {aws} CLI version that is installed in {aws} CloudShell might also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the _{aws} CloudShell User Guide_. -* The `kubectl` command line tool is installed on your device or {aws} CloudShell. The version can be the same as or up to one minor version earlier or later than the [.noloc]`Kubernetes` version of your cluster. For example, if your cluster version is `1.29`, you can use `kubectl` version `1.28`, `1.29`, or `1.30` with it. To install or upgrade `kubectl`, see <>. -* An existing `kubectl` `config` file that contains your cluster configuration. To create a `kubectl` `config` file, see <>. - - -[[irsa-associate-role-procedure,irsa-associate-role-procedure.title]] -==== Step 1: Create IAM Policy - -If you want to associate an existing IAM policy to your IAM role, skip to the next step. - - -. Create an IAM policy. You can create your own policy, or copy an {aws} managed policy that already grants some of the permissions that you need and customize it to your specific requirements. For more information, see link:IAM/latest/UserGuide/access_policies_create.html[Creating IAM policies,type="documentation"] in the _IAM User Guide_. -+ -. Create a file that includes the permissions for the {aws} services that you want your [.noloc]`Pods` to access. For a list of all actions for all {aws} services, see the link:service-authorization/latest/reference/[Service Authorization Reference,type="documentation"]. -+ -You can run the following command to create an example policy file that allows read-only access to an Amazon S3 bucket. You can optionally store configuration information or a bootstrap script in this bucket, and the containers in your [.noloc]`Pod` can read the file from the bucket and load it into your application. If you want to create this example policy, copy the following contents to your device. Replace [.replaceable]`my-pod-secrets-bucket` with your bucket name and run the command. -+ -[source,json,subs="verbatim,attributes"] ----- -cat >my-policy.json <my-service-account.yaml <> for more information. -+ -[source,json,subs="verbatim,attributes"] ----- -cat >trust-relationship.json <>. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl annotate serviceaccount -n $namespace $service_account eks.amazonaws.com/role-arn={arn-aws}iam::$account_id:role/my-role ----- -. (Optional) <>. {aws} recommends using a regional {aws} STS endpoint instead of the global endpoint. This reduces latency, provides built-in redundancy, and increases session token validity. - - -[[irsa-confirm-role-configuration,irsa-confirm-role-configuration.title]] -==== Step 3: Confirm configuration -. Confirm that the IAM role's trust policy is configured correctly. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws iam get-role --role-name my-role --query Role.AssumeRolePolicyDocument ----- -+ -An example output is as follows. -+ -[source,json,subs="verbatim,attributes"] ----- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "{arn-aws}iam::111122223333:oidc-provider/oidc.eks.region-code.amazonaws.com/id/EXAMPLED539D4633E53DE1B71EXAMPLE" - }, - "Action": "sts:AssumeRoleWithWebIdentity", - "Condition": { - "StringEquals": { - "oidc.eks.region-code.amazonaws.com/id/EXAMPLED539D4633E53DE1B71EXAMPLE:sub": "system:serviceaccount:default:my-service-account", - "oidc.eks.region-code.amazonaws.com/id/EXAMPLED539D4633E53DE1B71EXAMPLE:aud": "sts.amazonaws.com" - } - } - } - ] -} ----- -. Confirm that the policy that you attached to your role in a previous step is attached to the role. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws iam list-attached-role-policies --role-name my-role --query AttachedPolicies[].PolicyArn --output text ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -{arn-aws}iam::111122223333:policy/my-policy ----- -. Set a variable to store the Amazon Resource Name (ARN) of the policy that you want to use. Replace [.replaceable]`my-policy` with the name of the policy that you want to confirm permissions for. -+ -[source,bash,subs="verbatim,attributes"] ----- -export policy_arn={arn-aws}iam::111122223333:policy/my-policy ----- -. View the default version of the policy. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws iam get-policy --policy-arn $policy_arn ----- -+ -An example output is as follows. -+ -[source,json,subs="verbatim,attributes"] ----- -{ - "Policy": { - "PolicyName": "my-policy", - "PolicyId": "EXAMPLEBIOWGLDEXAMPLE", - "Arn": "{arn-aws}iam::111122223333:policy/my-policy", - "Path": "/", - "DefaultVersionId": "v1", - [...] - } -} ----- -. View the policy contents to make sure that the policy includes all the permissions that your [.noloc]`Pod` needs. If necessary, replace [.replaceable]`1` in the following command with the version that's returned in the previous output. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws iam get-policy-version --policy-arn $policy_arn --version-id v1 ----- -+ -An example output is as follows. -+ -[source,json,subs="verbatim,attributes"] ----- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": "s3:GetObject", - "Resource": "{arn-aws}s3:::my-pod-secrets-bucket" - } - ] -} ----- -+ -If you created the example policy in a previous step, then your output is the same. If you created a different policy, then the [.replaceable]`example` content is different. -. Confirm that the [.noloc]`Kubernetes` service account is annotated with the role. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl describe serviceaccount my-service-account -n default ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -Name: my-service-account -Namespace: default -Annotations: eks.amazonaws.com/role-arn: {arn-aws}iam::111122223333:role/my-role -Image pull secrets: -Mountable secrets: my-service-account-token-qqjfl -Tokens: my-service-account-token-qqjfl -[...] ----- - - -==== Next steps - -* <> - -[.topic] -[[pod-configuration,pod-configuration.title]] -=== Configure [.noloc]`Pods` to use a [.noloc]`Kubernetes` service account - -[abstract] --- -Learn how to configure your [.noloc]`Pods` to use a [.noloc]`Kubernetes` service account that you allowed to assume an {aws} Identity and Access Management role. --- - -If a [.noloc]`Pod` needs to access {aws} services, then you must configure it to use a [.noloc]`Kubernetes` service account. The service account must be associated to an {aws} Identity and Access Management (IAM) role that has permissions to access the {aws} services. - - - -* An existing cluster. If you don't have one, you can create one using one of the guides in <>. -* An existing IAM [.noloc]`OpenID Connect` ([.noloc]`OIDC`) provider for your cluster. To learn if you already have one or how to create one, see <>. -* An existing [.noloc]`Kubernetes` service account that's associated with an IAM role. The service account must be annotated with the Amazon Resource Name (ARN) of the IAM role. The role must have an associated IAM policy that contains the permissions that you want your [.noloc]`Pods` to have to use {aws} services. For more information about how to create the service account and role, and configure them, see <>. -* Version `2.12.3` or later or version `1.27.160` or later of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device or {aws} CloudShell. To check your current version, use `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or [.noloc]`Homebrew` for [.noloc]`macOS` are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the _{aws} Command Line Interface User Guide_. The {aws} CLI version that is installed in {aws} CloudShell might also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the _{aws} CloudShell User Guide_. -* The `kubectl` command line tool is installed on your device or {aws} CloudShell. The version can be the same as or up to one minor version earlier or later than the [.noloc]`Kubernetes` version of your cluster. For example, if your cluster version is `1.29`, you can use `kubectl` version `1.28`, `1.29`, or `1.30` with it. To install or upgrade `kubectl`, see <>. -* An existing `kubectl` `config` file that contains your cluster configuration. To create a `kubectl` `config` file, see <>. -. Use the following command to create a deployment manifest that you can deploy a [.noloc]`Pod` to confirm configuration with. Replace the [.replaceable]`example values` with your own values. -+ -[source,yaml,subs="verbatim,attributes"] ----- -cat >my-deployment.yaml <>. -.. Confirm that the [.noloc]`Pod` has a web identity token file mount. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl describe pod my-app-6f4dfff6cb-76cv9 | grep AWS_WEB_IDENTITY_TOKEN_FILE: ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -AWS_WEB_IDENTITY_TOKEN_FILE: /var/run/secrets/eks.amazonaws.com/serviceaccount/token ----- -+ -The `kubelet` requests and stores the token on behalf of the [.noloc]`Pod`. By default, the `kubelet` refreshes the token if the token is older than 80 percent of its total time to live or older than 24 hours. You can modify the expiration duration for any account other than the default service account by using the settings in your [.noloc]`Pod` spec. For more information, see https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/#serviceaccount-token-volume-projection[Service Account Token Volume Projection] in the [.noloc]`Kubernetes` documentation. -+ -The https://github.com/aws/amazon-eks-pod-identity-webhook#amazon-eks-pod-identity-webhook[Amazon EKS Pod Identity Webhook] on the cluster watches for [.noloc]`Pods` that use a service account with the following annotation: -+ -[source,bash,subs="verbatim,attributes"] ----- -eks.amazonaws.com/role-arn: {arn-aws}iam::111122223333:role/my-role ----- -+ -The webhook applies the previous environment variables to those [.noloc]`Pods`. Your cluster doesn't need to use the webhook to configure the environment variables and token file mounts. You can manually configure [.noloc]`Pods` to have these environment variables. The <> look for these environment variables first in the credential chain provider. The role credentials are used for [.noloc]`Pods` that meet this criteria. -. Confirm that your [.noloc]`Pods` can interact with the {aws} services using the permissions that you assigned in the IAM policy attached to your role. -+ -NOTE: When a [.noloc]`Pod` uses {aws} credentials from an IAM role that's associated with a service account, the {aws} CLI or other SDKs in the containers for that [.noloc]`Pod` use the credentials that are provided by that role. If you don't restrict access to the credentials that are provided to the <>, the [.noloc]`Pod` still has access to these credentials. For more information, see https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node[Restrict access to the instance profile assigned to the worker node]. -+ -If your [.noloc]`Pods` can't interact with the services as you expected, complete the following steps to confirm that everything is properly configured. -+ -.. Confirm that your [.noloc]`Pods` use an {aws} SDK version that supports assuming an IAM role through an [.noloc]`OpenID Connect` web identity token file. For more information, see <>. -.. Confirm that the deployment is using the service account. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl describe deployment my-app | grep "Service Account" ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -Service Account: my-service-account ----- -.. If your [.noloc]`Pods` still can't access services, review the <> that are described in <> to confirm that your role and service account are configured properly. - - -[.topic] -[[configure-sts-endpoint,configure-sts-endpoint.title]] -=== Configure the {aws} Security Token Service endpoint for a service account - -If you're using a [.noloc]`Kubernetes` service account with <>, then you can configure the type of {aws} Security Token Service endpoint that's used by the service account if your cluster and platform version are the same or later than those listed in the following table. If your [.noloc]`Kubernetes` or platform version are earlier than those listed in the table, then your service accounts can only use the global endpoint. - -[cols="1,1,1", options="header"] -|=== -|Kubernetes version -|Platform version -|Default endpoint type - - -|`1.31` -|`eks.4` -|Regional - -|`1.30` -|`eks.2` -|Regional - -|`1.29` -|`eks.1` -|Regional - -|`1.28` -|`eks.1` -|Regional - -|`1.27` -|`eks.1` -|Regional - -|`1.26` -|`eks.1` -|Regional - -|`1.25` -|`eks.1` -|Regional - -|`1.24` -|`eks.2` -|Regional - -|`1.23` -|`eks.1` -|Regional -|=== - -{aws} recommends using the regional {aws} STS endpoints instead of the global endpoint. This reduces latency, provides built-in redundancy, and increases session token validity. The {aws} Security Token Service must be active in the {aws} Region where the [.noloc]`Pod` is running. Moreover, your application must have built-in redundancy for a different {aws} Region in the event of a failure of the service in the {aws} Region. For more information, see link:IAM/latest/UserGuide/id_credentials_temp_enable-regions.html[Managing {aws} STS in an {aws} Region,type="documentation"] in the IAM User Guide. - - - -* An existing cluster. If you don't have one, you can create one using one of the guides in <>. -* An existing IAM OIDC provider for your cluster. For more information, see <>. -* An existing [.noloc]`Kubernetes` service account configured for use with the <> feature. - -The following examples all use the aws-node [.noloc]`Kubernetes` service account used by the <>. You can replace the [.replaceable]`example values` with your own service accounts, [.noloc]`Pods`, namespaces, and other resources. - -. Select a [.noloc]`Pod` that uses a service account that you want to change the endpoint for. Determine which {aws} Region that the [.noloc]`Pod` runs in. Replace [.replaceable]`aws-node-6mfgv` with your [.noloc]`Pod` name and [.replaceable]`kube-system` with your [.noloc]`Pod's` namespace. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl describe pod aws-node-6mfgv -n kube-system |grep Node: ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -ip-192-168-79-166.us-west-2/192.168.79.166 ----- -+ -In the previous output, the [.noloc]`Pod` is running on a node in the us-west-2 {aws} Region. -. Determine the endpoint type that the [.noloc]`Pod's` service account is using. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl describe pod aws-node-6mfgv -n kube-system |grep AWS_STS_REGIONAL_ENDPOINTS ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -AWS_STS_REGIONAL_ENDPOINTS: regional ----- -+ -If the current endpoint is global, then `global` is returned in the output. If no output is returned, then the default endpoint type is in use and has not been overridden. -. If your cluster or platform version are the same or later than those listed in the table, then you can change the endpoint type used by your service account from the default type to a different type with one of the following commands. Replace [.replaceable]`aws-node` with the name of your service account and [.replaceable]`kube-system` with the namespace for your service account. -+ -** If your default or current endpoint type is global and you want to change it to regional: -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl annotate serviceaccount -n kube-system aws-node eks.amazonaws.com/sts-regional-endpoints=true ----- -// Not using [.noloc]`Pods'` because the ' character seems to mess up the processing. -+ -If you're using <> to generate pre-signed S3 URLs in your application running in Pods' containers, the format of the URL for regional endpoints is similar to the following example: -+ -[source,none,subs="verbatim,attributes"] ----- -https://bucket.s3.us-west-2.amazonaws.com/path?...&X-Amz-Credential=your-access-key-id/date/us-west-2/s3/aws4_request&... ----- -** If your default or current endpoint type is regional and you want to change it to global: -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl annotate serviceaccount -n kube-system aws-node eks.amazonaws.com/sts-regional-endpoints=false ----- -+ -If your application is explicitly making requests to {aws} STS global endpoints and you don't override the default behavior of using regional endpoints in Amazon EKS clusters, then requests will fail with an error. For more information, see <>. -// Not using [.noloc]`Pods'` because the ' character seems to mess up the processing. -+ -If you're using <> to generate pre-signed S3 URLs in your application running in Pods' containers, the format of the URL for global endpoints is similar to the following example: -+ -[source,none,subs="verbatim,attributes"] ----- -https://bucket.s3.amazonaws.com/path?...&X-Amz-Credential=your-access-key-id/date/us-west-2/s3/aws4_request&... ----- - -+ -If you have automation that expects the pre-signed URL in a certain format or if your application or downstream dependencies that use pre-signed URLs have expectations for the {aws} Region targeted, then make the necessary changes to use the appropriate {aws} STS endpoint. -. Delete and re-create any existing [.noloc]`Pods` that are associated with the service account to apply the credential environment variables. The mutating web hook doesn't apply them to [.noloc]`Pods` that are already running. You can replace [.replaceable]`Pods`, [.replaceable]`kube-system`, and [.replaceable]`-l k8s-app=aws-node` with the information for the [.noloc]`Pods` that you set your annotation for. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl delete Pods -n kube-system -l k8s-app=aws-node ----- -. Confirm that the all [.noloc]`Pods` restarted. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl get Pods -n kube-system -l k8s-app=aws-node ----- -. View the environment variables for one of the [.noloc]`Pods`. Verify that the `AWS_STS_REGIONAL_ENDPOINTS` value is what you set it to in a previous step. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl describe pod aws-node-kzbtr -n kube-system |grep AWS_STS_REGIONAL_ENDPOINTS ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -AWS_STS_REGIONAL_ENDPOINTS=regional ----- - - -[.topic] -[[cross-account-access,cross-account-access.title]] -=== Authenticate to another account with IRSA - -[abstract] --- -Learn how to configure cross-account IAM permissions for Amazon EKS clusters by creating an identity provider from another account's cluster or using chained AssumeRole operations, enabling secure access to {aws} resources across multiple accounts. --- - -You can configure cross-account IAM permissions either by creating an identity provider from another account's cluster or by using chained `AssumeRole` operations. In the following examples, _Account A_ owns an Amazon EKS cluster that supports IAM roles for service accounts. [.noloc]`Pods` that are running on that cluster must assume IAM permissions from _Account B_. - -.Create an identity provider from another account's cluster -==== - -==== - -==== - -In this example, Account A provides Account B with the OpenID Connect (OIDC) issuer URL from their cluster. Account B follows the instructions in <> and <> using the OIDC issuer URL from Account A's cluster. Then, a cluster administrator annotates the service account in Account A's cluster to use the role from Account B ([.replaceable]`444455556666`). - -[source,yaml,subs="verbatim,attributes"] ----- -apiVersion: v1 -kind: ServiceAccount -metadata: - annotations: - eks.amazonaws.com/role-arn: {arn-aws}iam::444455556666:role/account-b-role ----- - -==== - -.Use chained `AssumeRole` operations -==== - -==== - -==== - -In this example, Account B creates an IAM policy with the permissions to give to [.noloc]`Pods` in Account A's cluster. Account B ([.replaceable]`444455556666`) attaches that policy to an IAM role with a trust relationship that allows `AssumeRole` permissions to Account A ([.replaceable]`111122223333`). - -[source,json,subs="verbatim,attributes"] ----- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "{aws}": "{arn-aws}iam::111122223333:root" - }, - "Action": "sts:AssumeRole", - "Condition": {} - } - ] -} ----- - -Account A creates a role with a trust policy that gets credentials from the identity provider created with the cluster's OIDC issuer address. - -[source,json,subs="verbatim,attributes"] ----- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "{arn-aws}iam::111122223333:oidc-provider/oidc.eks.region-code.amazonaws.com/id/EXAMPLED539D4633E53DE1B71EXAMPLE" - }, - "Action": "sts:AssumeRoleWithWebIdentity" - } - ] -} ----- - -Account A attaches a policy to that role with the following permissions to assume the role that Account B created. - -[source,json,subs="verbatim,attributes"] ----- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": "sts:AssumeRole", - "Resource": "{arn-aws}iam::444455556666:role/account-b-role" - } - ] -} ----- - -The application code for [.noloc]`Pods` to assume Account B's role uses two profiles: `account_b_role` and `account_a_role`. The `account_b_role` profile uses the `account_a_role` profile as its source. For the {aws} CLI, the `~/.aws/config` file is similar to the following. - -[source,none,subs="verbatim,attributes"] ----- -[profile account_b_role] -source_profile = account_a_role -role_arn={arn-aws}iam::444455556666:role/account-b-role - -[profile account_a_role] -web_identity_token_file = /var/run/secrets/eks.amazonaws.com/serviceaccount/token -role_arn={arn-aws}iam::111122223333:role/account-a-role ----- - -To specify chained profiles for other {aws} SDKs, consult the documentation for the SDK that you're using. For more information, see link:developer/tools/[Tools to Build on {aws},type="marketing"]. - -==== - -[.topic] -[[iam-roles-for-service-accounts-minimum-sdk,iam-roles-for-service-accounts-minimum-sdk.title]] -=== Use IRSA with the {aws} SDK - -.Using the credentials -To use the credentials from IAM roles for service accounts, your code can use any {aws} SDK to create a client for an {aws} service with an SDK, and by default the SDK searches in a chain of locations for {aws} Identity and Access Management credentials to use. The IAM roles for service accounts credentials will be used if you don't specify a credential provider when you create the client or otherwise initialized the SDK. - -This works because IAM roles for service accounts have been added as a step in the default credential chain. If your workloads currently use credentials that are earlier in the chain of credentials, those credentials will continue to be used even if you configure an IAM roles for service accounts for the same workload. - -The SDK automatically exchanges the service account [.noloc]`OIDC` token for temporary credentials from {aws} Security Token Service by using the `AssumeRoleWithWebIdentity` action. Amazon EKS and this SDK action continue to rotate the temporary credentials by renewing them before they expire. - -When using <>, the containers in your [.noloc]`Pods` must use an {aws} SDK version that supports assuming an IAM role through an [.noloc]`OpenID Connect` web identity token file. Make sure that you're using the following versions, or later, for your {aws} SDK: - - - -* Java (Version 2) – https://github.com/aws/aws-sdk-java-v2/releases/tag/2.10.11[2.10.11] -* Java – https://github.com/aws/aws-sdk-java/releases/tag/1.11.704[1.11.704] -* Go – https://github.com/aws/aws-sdk-go/releases/tag/v1.23.13[1.23.13] -* Python (Boto3) – https://github.com/boto/boto3/releases/tag/1.9.220[1.9.220] -* Python (botocore) – https://github.com/boto/botocore/releases/tag/1.12.200[1.12.200] -* {aws} CLI – https://github.com/aws/aws-cli/releases/tag/1.16.232[1.16.232] -* Node – https://github.com/aws/aws-sdk-js/releases/tag/v2.525.0[2.525.0] and https://github.com/aws/aws-sdk-js-v3/releases/tag/v3.27.0[3.27.0] -* Ruby – https://github.com/aws/aws-sdk-ruby/blob/version-3/gems/aws-sdk-core/CHANGELOG.md#3580-2019-07-01[3.58.0] -* {cpp} – https://github.com/aws/aws-sdk-cpp/releases/tag/1.7.174[1.7.174] -* .NET – https://github.com/aws/aws-sdk-net/releases/tag/3.3.659.1[3.3.659.1] – You must also include `AWSSDK.SecurityToken`. -* PHP – https://github.com/aws/aws-sdk-php/releases/tag/3.110.7[3.110.7] - -Many popular [.noloc]`Kubernetes` add-ons, such as the https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler[Cluster Autoscaler], the <>, and the <> support IAM roles for service accounts. - -To ensure that you're using a supported SDK, follow the installation instructions for your preferred SDK at link:tools/[Tools to Build on {aws},type="marketing"] when you build your containers. - -[.topic] -[[irsa-fetch-keys,irsa-fetch-keys.title]] -=== Fetch signing keys to validate [.noloc]`OIDC` tokens - -[abstract] --- -Discover how to fetch the OIDC public signing keys (JSON Web Key Set) required to validate the ProjectedServiceAccountToken for Amazon EKS clusters, enabling external systems to authenticate with IAM roles for Kubernetes service accounts. --- - -[.noloc]`Kubernetes` issues a `ProjectedServiceAccountToken` to each [.noloc]`Kubernetes` [.noloc]`Service Account`. This token is an [.noloc]`OIDC` token, which is further a type of [.noloc]`JSON web token (JWT)`. Amazon EKS hosts a public [.noloc]`OIDC` endpoint for each cluster that contains the signing keys for the token so external systems can validate it. - -To validate a `ProjectedServiceAccountToken`, you need to fetch the [.noloc]`OIDC` public signing keys, also called the [.noloc]`JSON Web Key Set (JWKS)`. Use these keys in your application to validate the token. For example, you can use the https://pyjwt.readthedocs.io/en/latest/[PyJWT Python library] to validate tokens using these keys. For more information on the `ProjectedServiceAccountToken`, see <>. - -==== Prerequisites - -* An existing {aws} Identity and Access Management (IAM) [.noloc]`OpenID Connect` ([.noloc]`OIDC`) provider for your cluster. To determine whether you already have one, or to create one, see <>. -* *{aws} CLI* -- A command line tool for working with {aws} services, including Amazon EKS. For more information, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] in the {aws} Command Line Interface User Guide. After installing the {aws} CLI, we recommend that you also configure it. For more information, see link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the {aws} Command Line Interface User Guide. - -==== Procedure - -. Retrieve the [.noloc]`OIDC` URL for your Amazon EKS cluster using the {aws} CLI. -+ -[source,bash,subs="verbatim,attributes"] ----- -$ aws eks describe-cluster --name my-cluster --query 'cluster.identity.oidc.issuer' -"https://oidc.eks.us-west-2.amazonaws.com/id/8EBDXXXX00BAE" ----- -. Retrieve the public signing key using [.noloc]`curl`, or a similar tool. The result is a https://www.rfc-editor.org/rfc/rfc7517#section-5[JSON Web Key Set (JWKS)]. -+ -IMPORTANT: Amazon EKS throttles calls to the [.noloc]`OIDC` endpoint. You should cache the public signing key. Respect the `cache-control` header included in the response. -+ -IMPORTANT: Amazon EKS rotates the [.noloc]`OIDC` signing key every seven days. -+ -[source,bash,subs="verbatim,attributes"] ----- -$ curl https://oidc.eks.us-west-2.amazonaws.com/id/8EBDXXXX00BAE/keys -{"keys":[{"kty":"RSA","kid":"2284XXXX4a40","use":"sig","alg":"RS256","n":"wklbXXXXMVfQ","e":"AQAB"}]} ----- diff --git a/latest/ug/manage-access/cluster-auth.adoc b/latest/ug/manage-access/cluster-auth.adoc index af75691bb..310fbf496 100644 --- a/latest/ug/manage-access/cluster-auth.adoc +++ b/latest/ug/manage-access/cluster-auth.adoc @@ -1,55 +1,42 @@ -//!!NODE_ROOT include::../attributes.txt[] + [.topic] -[[cluster-auth,cluster-auth.title]] +[#cluster-auth] = Learn how access control works in Amazon EKS -:doctype: book -:sectnums: -:toc: left -:icons: font -:experimental: -:idprefix: -:idseparator: - -:sourcedir: . -:info_doctype: chapter -:info_title: Learn how access control works in Amazon EKS :info_titleabbrev: Manage access -:info_abstract: Learn how to manage access to your EKS cluster. First, this includes granting \ - developers or external services access to Kubernetes. Second, this includes granting \ - Kubernetes workloads access to {aws} services. [abstract] -- Learn how to manage access to your EKS cluster. First, this includes granting developers or external services access to Kubernetes. Second, this includes granting Kubernetes workloads access to {aws} services. -- -Learn how to manage access to your Amazon EKS cluster. Using Amazon EKS requires knowledge of how both [.noloc]`Kubernetes` and {aws} Identity and Access Management ({aws} IAM) handle access control. +Learn how to manage access to your Amazon EKS cluster. Using Amazon EKS requires knowledge of how both Kubernetes and {aws} Identity and Access Management ({aws} IAM) handle access control. *This section includes:* -*xref:grant-k8s-access[Grant IAM users and roles access to Kubernetes APIs,linkend=grant-k8s-access]* -- Learn how to enable applications or users to authenticate to the [.noloc]`Kubernetes` API. You can use access entries, the aws-auth ConfigMap, or an external OIDC provider. +*<>* -- Learn how to enable applications or users to authenticate to the Kubernetes API. You can use access entries, the aws-auth ConfigMap, or an external OIDC provider. -*<>* -- Learn how to configure the {aws-management-console} to communicate with your Amazon EKS cluster. Use the console to view [.noloc]`Kubernetes` resources in the cluster, such as namespaces, nodes, and [.noloc]`Pods`. +*<>* -- Learn how to configure the {aws-management-console} to communicate with your Amazon EKS cluster. Use the console to view Kubernetes resources in the cluster, such as namespaces, nodes, and Pods. -*<>* -- Learn how to configure kubectl to communicate with your Amazon EKS cluster. Use the {aws} CLI to create a kubeconfig file. +*<>* -- Learn how to configure kubectl to communicate with your Amazon EKS cluster. Use the {aws} CLI to create a kubeconfig file. -*xref:service-accounts[Grant Kubernetes workloads access to {aws} using Kubernetes Service Accounts,linkend=service-accounts]* -- Learn how to associate a [.noloc]`Kubernetes` service account with {aws} IAM Roles. You can use Pod Identity or IAM Roles for Service Accounts (IRSA). +*<>* -- Learn how to associate a Kubernetes service account with {aws} IAM Roles. You can use Pod Identity or IAM Roles for Service Accounts (IRSA). == Common Tasks -* Grant developers access to the [.noloc]`Kubernetes` API. View [.noloc]`Kubernetes` resources in the {aws-management-console}. +* Grant developers access to the Kubernetes API. View Kubernetes resources in the {aws-management-console}. + -** Solution: <> to associate [.noloc]`Kubernetes` RBAC permissions with {aws} IAM Users or Roles. +** Solution: <> to associate Kubernetes RBAC permissions with {aws} IAM Users or Roles. * Configure kubectl to talk to an Amazon EKS cluster using {aws} Credentials. + ** Solution: Use the {aws} CLI to <>. -* Use an external identity provider, such as Ping Identity, to authenticate users to the [.noloc]`Kubernetes` API. +* Use an external identity provider, such as Ping Identity, to authenticate users to the Kubernetes API. + ** Solution: <>. -* Grant workloads on your [.noloc]`Kubernetes` cluster the ability to call {aws} APIs. +* Grant workloads on your Kubernetes cluster the ability to call {aws} APIs. + -** Solution: <> to associate an {aws} IAM Role to a [.noloc]`Kubernetes` Service Account. +** Solution: <> to associate an {aws} IAM Role to a Kubernetes Service Account. == Background @@ -73,11 +60,8 @@ EKS Auto Mode integrates with EKS Pod Identity and EKS EKS access entries. include::k8s-access/grant-k8s-access.adoc[leveloffset=+1] - include::view-kubernetes-resources.adoc[leveloffset=+1] - include::create-kubeconfig.adoc[leveloffset=+1] - -include::aws-access/service-accounts.adoc[leveloffset=+1] +include::aws-access/service-accounts.adoc[leveloffset=+1] \ No newline at end of file diff --git a/latest/ug/manage-access/create-kubeconfig.adoc b/latest/ug/manage-access/create-kubeconfig.adoc index 7c480d368..931b647a1 100644 --- a/latest/ug/manage-access/create-kubeconfig.adoc +++ b/latest/ug/manage-access/create-kubeconfig.adoc @@ -1,13 +1,9 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[create-kubeconfig,create-kubeconfig.title]] -= Connect [.noloc]`kubectl` to an EKS cluster by creating a [.noloc]`kubeconfig` file -:info_doctype: section -:info_title: Connect kubectl to an EKS cluster by creating a kubeconfig file +[#create-kubeconfig] += Connect kubectl to an EKS cluster by creating a kubeconfig file :info_titleabbrev: Access cluster with kubectl -:info_abstract: Learn how to create or update a kubeconfig file for authenticating with your Amazon EKS cluster using kubectl. Follow prerequisites for required tools and permissions. [abstract] -- @@ -16,7 +12,7 @@ Learn how to create or update a kubeconfig file for authenticating with your Ama In this topic, you create a `kubeconfig` file for your cluster (or update an existing one). -The `kubectl` command-line tool uses configuration information in `kubeconfig` files to communicate with the API server of a cluster. For more information, see https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/[Organizing Cluster Access Using kubeconfig Files] in the [.noloc]`Kubernetes` documentation. +The `kubectl` command-line tool uses configuration information in `kubeconfig` files to communicate with the API server of a cluster. For more information, see https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/[Organizing Cluster Access Using kubeconfig Files] in the Kubernetes documentation. Amazon EKS uses the `aws eks get-token` command with `kubectl` for cluster authentication. By default, the {aws} CLI uses the same credentials that are returned with the following command: @@ -26,15 +22,15 @@ aws sts get-caller-identity ---- * An existing Amazon EKS cluster. To deploy one, see <>. -* The `kubectl` command line tool is installed on your device or {aws} CloudShell. The version can be the same as or up to one minor version earlier or later than the [.noloc]`Kubernetes` version of your cluster. For example, if your cluster version is `1.29`, you can use `kubectl` version `1.28`, `1.29`, or `1.30` with it. To install or upgrade `kubectl`, see <>. -* Version `2.12.3` or later or version `1.27.160` or later of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device or {aws} CloudShell. To check your current version, use `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or [.noloc]`Homebrew` for [.noloc]`macOS` are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the _{aws} Command Line Interface User Guide_. The {aws} CLI version that is installed in {aws} CloudShell might also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the _{aws} CloudShell User Guide_. -* An IAM user or role with permission to use the `eks:DescribeCluster` API action for the cluster that you specify. For more information, see <>. If you use an identity from your own [.noloc]`OpenID Connect` provider to access your cluster, then see https://kubernetes.io/docs/reference/access-authn-authz/authentication/#using-kubectl[Using kubectl] in the [.noloc]`Kubernetes` documentation to create or update your `kube config` file. +* The `kubectl` command line tool is installed on your device or {aws} CloudShell. The version can be the same as or up to one minor version earlier or later than the Kubernetes version of your cluster. For example, if your cluster version is `1.29`, you can use `kubectl` version `1.28`, `1.29`, or `1.30` with it. To install or upgrade `kubectl`, see <>. +* Version `2.12.3` or later or version `1.27.160` or later of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device or {aws} CloudShell. To check your current version, use `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or Homebrew for macOS are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the _{aws} Command Line Interface User Guide_. The {aws} CLI version that is installed in {aws} CloudShell might also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the _{aws} CloudShell User Guide_. +* An IAM user or role with permission to use the `eks:DescribeCluster` API action for the cluster that you specify. For more information, see <>. If you use an identity from your own OpenID Connect provider to access your cluster, then see https://kubernetes.io/docs/reference/access-authn-authz/authentication/#using-kubectl[Using kubectl] in the Kubernetes documentation to create or update your `kube config` file. -[[create-kubeconfig-automatically,create-kubeconfig-automatically.title]] +[#create-kubeconfig-automatically] == Create `kubeconfig` file automatically -* Version `2.12.3` or later or version `1.27.160` or later of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device or {aws} CloudShell. To check your current version, use `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or [.noloc]`Homebrew` for [.noloc]`macOS` are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the _{aws} Command Line Interface User Guide_. The {aws} CLI version that is installed in {aws} CloudShell might also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the _{aws} CloudShell User Guide_. +* Version `2.12.3` or later or version `1.27.160` or later of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device or {aws} CloudShell. To check your current version, use `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or Homebrew for macOS are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the _{aws} Command Line Interface User Guide_. The {aws} CLI version that is installed in {aws} CloudShell might also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the _{aws} CloudShell User Guide_. * Permission to use the `eks:DescribeCluster` API action for the cluster that you specify. For more information, see <>. . Create or update a `kubeconfig` file for your cluster. Replace [.replaceable]`region-code` with the {aws} Region that your cluster is in and replace [.replaceable]`my-cluster` with the name of your cluster. + @@ -45,7 +41,7 @@ aws eks update-kubeconfig --region region-code --name my-cluster + By default, the resulting configuration file is created at the default `kubeconfig` path (`.kube`) in your home directory or merged with an existing `config` file at that location. You can specify another path with the `--kubeconfig` option. + -You can specify an IAM role ARN with the `--role-arn` option to use for authentication when you issue `kubectl` commands. Otherwise, the link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] in your default {aws} CLI or SDK credential chain is used. You can view your default {aws} CLI or SDK identity by running the `aws sts get-caller-identity` command. +You can specify an IAM role ARN with the `--role-arn` option to use for authentication when you issue `kubectl` commands. Otherwise, the link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] in your default {aws} CLI or SDK credential chain is used. You can view your default {aws} CLI or SDK identity by running the `aws sts get-caller-identity` command. + For all available options, run the `aws eks update-kubeconfig help` command or see link:cli/latest/reference/eks/update-kubeconfig.html[update-kubeconfig,type="documentation"] in the _{aws} CLI Command Reference_. . Test your configuration. @@ -63,4 +59,4 @@ NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE svc/kubernetes ClusterIP 10.100.0.1 443/TCP 1m ---- + -If you receive any authorization or resource type errors, see <> in the troubleshooting topic. +If you receive any authorization or resource type errors, see <> in the troubleshooting topic. \ No newline at end of file diff --git a/latest/ug/manage-access/images b/latest/ug/manage-access/images new file mode 120000 index 000000000..5e6757319 --- /dev/null +++ b/latest/ug/manage-access/images @@ -0,0 +1 @@ +../images \ No newline at end of file diff --git a/latest/ug/manage-access/k8s-access/access-entries.adoc b/latest/ug/manage-access/k8s-access/access-entries.adoc index 7cf97db0a..3ff350db5 100644 --- a/latest/ug/manage-access/k8s-access/access-entries.adoc +++ b/latest/ug/manage-access/k8s-access/access-entries.adoc @@ -1,319 +1,74 @@ -//!!NODE_ROOT
- -[.topic] -[[access-entries,access-entries.title]] -= Grant [.noloc]`IAM` users access to [.noloc]`Kubernetes` with EKS access entries -:info_doctype: section - include::../../attributes.txt[] - -include::access-policies.adoc[leveloffset=+1] - -include::migrating-access-entries.adoc[leveloffset=+1] - -include::access-policy-reference.adoc[leveloffset=+1] +[.topic] +[#access-entries] += Grant IAM users access to Kubernetes with EKS access entries +:info_titleabbrev: Access entries [abstract] -- Learn how to manage access entries for IAM principals to your Amazon EKS cluster, including creating, updating, and deleting access entries for fine-grained authentication and authorization. -- -*What is EKS access entries?* +This section is designed to show you how to manage IAM principal access to Kubernetes clusters in Amazon Elastic Kubernetes Service (EKS) using access entries and policies. You'll find details on changing authentication modes, migrating from legacy `aws-auth` ConfigMap entries, creating, updating, and deleting access entries, associating policies with entries, reviewing predefined policy permissions, and key prerequisites and considerations for secure access management. + +== Overview + +EKS access entries are the best way to grant users access to the Kubernetes API. For example, you can use access entries to grant developers access to use kubectl. Fundamentally, an EKS access entry associates a set of Kubernetes permissions with an IAM identity, such as an IAM role. For example, a developer may assume an IAM role and use that to authenticate to an EKS Cluster. + +== Features -EKS access entries it the best way to grant users access to the Kubernetes API. For example, you can use access entries to grant developers access to use kubectl. +* **Centralized Authentication and Authorization**: Controls access to Kubernetes clusters directly via Amazon EKS APIs, eliminating the need to switch between {aws} and Kubernetes APIs for user permissions. +* **Granular Permissions Management**: Uses access entries and policies to define fine-grained permissions for {aws} IAM principals, including modifying or revoking cluster-admin access from the creator. +* **IaC Tool Integration**: Supports infrastructure as code tools like {aws} CloudFormation, Terraform, and {aws} CDK to define access configurations during cluster creation. +* **Misconfiguration Recovery**: Allows restoring cluster access through the Amazon EKS API without direct Kubernetes API access. +* **Reduced Overhead and Enhanced Security**: Centralizes operations to lower overhead while leveraging {aws} IAM features like CloudTrail audit logging and multi-factor authentication. -Fundamentally, an EKS access entry associates a set of Kubernetes permissions with an IAM identity, such as an IAM role. For example, a develoer may assume an IAM role and use that to authenticate to an EKS Cluster. -You can attach Kubernetes permissions to access entries in two ways: +== How to attach permissions +You can attach Kubernetes permissions to access entries in two ways: -* Use an access policy. Access policies are pre-defined Kubernetes permissions templates maintained by {aws}. For more information, see <>. -* Reference a Kubernetes group. If you associate an IAM Identity with a Kubernetes group, you can create Kubernetes resources that grant the group permissions. For more information, see https://kubernetes.io/docs/reference/access-authn-authz/rbac/[Using RBAC Authorization] in the Kubernetes documentation. +* Use an access policy. Access policies are pre-defined Kubernetes permissions templates maintained by {aws}. For more information, see <>. +* Reference a Kubernetes group. If you associate an IAM Identity with a Kubernetes group, you can create Kubernetes resources that grant the group permissions. For more information, see https://kubernetes.io/docs/reference/access-authn-authz/rbac/[Using RBAC Authorization] in the Kubernetes documentation. -*Advantages* +== Considerations -Amazon EKS cluster access management enables you to control authentication and authorization for your Kubernetes clusters directly through Amazon EKS APIs. This feature simplifies access management by eliminating the need to switch between {aws} and Kubernetes APIs when managing user permissions. Using access entries and access policies, you can define granular permissions for {aws} IAM principals, including the ability to modify or revoke cluster-admin permissions from the cluster creator. +When enabling EKS access entries on existing clusters, keep the following in mind: -The feature integrates with infrastructure as code (IaC) tools like {aws} CloudFormation, Terraform, and {aws} CDK, allowing you to define access configurations during cluster creation. If misconfigurations occur, you can restore cluster access through the Amazon EKS API without requiring direct Kubernetes API access. This centralized approach reduces operational overhead and improves security by leveraging existing {aws} IAM capabilities such as CloudTrail audit logging and multi-factor authentication. +* **Legacy Cluster Behavior**: For clusters created before the introduction of access entries (those with initial platform versions earlier than specified in link:eks/latest/userguide/platform-versions.html[Platform version requirements,type="documentation"]), EKS automatically creates an access entry reflecting pre-existing permissions. This entry includes the IAM identity that originally created the cluster and the administrative permissions granted to that identity during cluster creation. +* **Handling Legacy `aws-auth` ConfigMap**: If your cluster relies on the legacy `aws-auth` ConfigMap for access management, only the access entry for the original cluster creator is automatically created upon enabling access entries. Additional roles or permissions added to the ConfigMap (e.g., custom IAM roles for developers or services) are not automatically migrated. To address this, manually create corresponding access entries. -== Get Started +== Get started -. Determine the IAM Identity and Access policy you want to use. +. Determine the IAM Identity and Access policy you want to use. ** <> . Enable EKS Access Entries on your cluster. Confirm you have a supported platform version. ** <> . Create an access entry that associates an IAM Identity with Kubernetes permission. ** <> -. Authenticate to the cluster using the IAM identity. +. Authenticate to the cluster using the IAM identity. ** <> ** <> -== Legacy cluster access configuration - -When you enable EKS access entries on clusters created before this feature was introduced (clusters with initial platform versions earlier than those specified in Platform Version Requirements), EKS automatically creates an access entry that reflects pre-existing permissions. -This access entry shows: - -* The IAM identity that originally created the cluster -* The administrative permissions granted to that identity during cluster creation - -NOTE: Previously, this administrative access was granted automatically and couldn't be modified. With EKS access entries enabled, you can now view and delete this legacy access configuration. - - -[.topic] -[[setting-up-access-entries,setting-up-access-entries.title]] -== Change authentication mode to use access entries - -To begin using access entries, you must change the authentication mode of the cluster to either the `API_AND_CONFIG_MAP` or `API` modes. This adds the API for access entries. - -[[access-entries-setup-console,access-entries-setup-console.title]] -=== {aws} Console - -. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. -. Choose the name of the cluster that you want to create an access entry in. -. Choose the *Access* tab. -. The *Authentication mode* shows the current authentication mode of the cluster. If the mode says [.noloc]`EKS API`, you can already add access entries and you can skip the remaining steps. -. Choose *Manage access*. -. For *Cluster authentication mode*, select a mode with the [.noloc]`EKS API`. Note that you can't change the authentication mode back to a mode that removes the [.noloc]`EKS API` and access entries. -. Choose *Save changes*. Amazon EKS begins to update the cluster, the status of the cluster changes to [.noloc]`Updating`, and the change is recorded in the *Update history* tab. -. Wait for the status of the cluster to return to [.noloc]`Active`. When the cluster is [.noloc]`Active`, you can follow the steps in <> to add access to the cluster for IAM principals. - -[[access-setup-cli,access-setup-cli.title]] -=== {aws} CLI - -. Install the {aws} CLI, as described in link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] in the _{aws} Command Line Interface User Guide_. -. Run the following command. Replace [.replaceable]`my-cluster` with the name of your cluster. If you want to disable the `ConfigMap` method permanently, replace `API_AND_CONFIG_MAP` with `API`. -+ -Amazon EKS begins to update the cluster, the status of the cluster changes to [.noloc]`UPDATING`, and the change is recorded in the [command]*aws eks list-updates*. -+ -[source,bash] ----- -aws eks update-cluster-config --name my-cluster --access-config authenticationMode=API_AND_CONFIG_MAP ----- -. Wait for the status of the cluster to return to [.noloc]`Active`. When the cluster is [.noloc]`Active`, you can follow the steps in <> to add access to the cluster for IAM principals. - - -=== Required platform version - -To use _access entries_, the cluster must have a platform version that is the same or later than the version listed in the following table, or a [.noloc]`Kubernetes` version that is later than the versions listed in the table. If your Kubernetes version is not listed, all platform versions support access entries. - -[cols="1,1", options="header"] -|=== -|Kubernetes version -|Platform version - - -|`1.30` -|`eks.2` - -|`1.29` -|`eks.1` - -|`1.28` -|`eks.6` - -|`1.27` -|`eks.10` - -|`1.26` -|`eks.11` - -|`1.25` -|`eks.12` - -|`1.24` -|`eks.15` - -|`1.23` -|`eks.17` -|=== - -For more information, see <>. - [.topic] -[[creating-access-entries,creating-access-entries.title]] -== Create access entries - - -Before creating access entries, consider the following: - -* A properly set authentication mode. See <>. -* An _access entry_ includes the Amazon Resource Name (ARN) of one, and only one, existing IAM principal. An IAM principal can't be included in more than one access entry. Additional considerations for the ARN that you specify: -+ -** IAM best practices recommend accessing your cluster using IAM _roles_ that have short-term credentials, rather than IAM _users_ that have long-term credentials. For more information, see link:IAM/latest/UserGuide/best-practices.html#bp-users-federation-idp[Require human users to use federation with an identity provider to access {aws} using temporary credentials,type="documentation"] in the _IAM User Guide_. -** If the ARN is for an IAM role, it _can_ include a path. ARNs in `aws-auth` `ConfigMap` entries, _can't_ include a path. For example, your ARN can be `{arn-aws}iam::<111122223333>:role/` or `{arn-aws}iam::<111122223333>:role/`. -** If the type of the access entry is anything other than `STANDARD` (see next consideration about types), the ARN must be in the same {aws} account that your cluster is in. If the type is `STANDARD`, the ARN can be in the same, or different, {aws} account than the account that your cluster is in. -** You can't change the IAM principal after the access entry is created. -** If you ever delete the IAM principal with this ARN, the access entry isn't automatically deleted. We recommend that you delete the access entry with an ARN for an IAM principal that you delete. If you don't delete the access entry and ever recreate the IAM principal, even if it has the same ARN, the access entry won't work. This is because even though the ARN is the same for the recreated IAM principal, the `roleID` or `userID` (you can see this with the `aws sts get-caller-identity` {aws} CLI command) is different for the recreated IAM principal than it was for the original IAM principal. Even though you don't see the IAM principal's `roleID` or `userID` for an access entry, Amazon EKS stores it with the access entry. -* Each access entry has a _type_. You can specify `EC2_LINUX` (for an IAM role used with Linux or Bottlerocket self-managed nodes), `EC2_Windows` (for an IAM role used with Windows self-managed nodes), `FARGATE_LINUX` (for an IAM role used with {aws} Fargate (Fargate)), `HYBRID_LINUX` (for an IAM role used with hybrid nodes) or `STANDARD` as a type. If you don't specify a type, Amazon EKS automatically sets the type to `STANDARD`. It's unnecessary to create an access entry for an IAM role that's used for a managed node group or a Fargate profile. EKS will create access entries (if enabled), or update the auth config map (if access entries are unavailable). -+ -You can't change the type after the access entry is created. -* If the type of the access entry is `STANDARD`, you can specify a _username_ for the access entry. If you don't specify a value for username, Amazon EKS sets one of the following values for you, depending on the type of the access entry and whether the IAM principal that you specified is an IAM role or IAM user. Unless you have a specific reason for specifying your own username, we recommend that don't specify one and let Amazon EKS auto-generate it for you. If you specify your own username: -+ -** It can't start with `system:`, `eks:`, `aws:`, `amazon:`, or `iam:`. -** If the username is for an IAM role, we recommend that you add `{{SessionName}}` to the end of your username. If you add `{{SessionName}}` to your username, the username must include a colon _before_ {{SessionName}}. When this role is assumed, the name of the session specified when assuming the role is automatically passed to the cluster and will appear in CloudTrail logs. For example, you can't have a username of `john{{SessionName}}`. The username would have to be `:john{{SessionName}}` or `jo:hn{{SessionName}}`. The colon only has to be before `{{SessionName}}`. The username generated by Amazon EKS in the following table includes an ARN. Since an ARN includes colons, it meets this requirement. The colon isn't required if you don't include `{{SessionName}}` in your username. Note that the special character "@" is replaced with "-" in the session name. -+ -[cols="1,1,1", options="header"] -|=== -|IAM principal type -|Type -|Username value that Amazon EKS automatically sets - - -|User -|`STANDARD` -|The ARN of the user. Example: `{arn-aws}iam::<111122223333>:user/` - -|Role -|`STANDARD` -|The STS ARN of the role when it's assumed. Amazon EKS appends `{{SessionName}}` to the role. - -Example: `{arn-aws}sts::<111122223333>:assumed-role//{{SessionName}}` - -If the ARN of the role that you specified contained a path, Amazon EKS removes it in the generated username. - -|Role -|`EC2_LINUX` or `EC2_Windows` -|`system:node:{{EC2PrivateDNSName}}` - -|Role -|`FARGATE_LINUX` -|`system:node:{{SessionName}}` - -|Role -|`HYBRID_LINUX` -|`system:node:{{SessionName}}` -|=== -+ -You can change the username after the access entry is created. -* If an access entry's type is `STANDARD`, and you want to use [.noloc]`Kubernetes` RBAC authorization, you can add one or more _group names_ to the access entry. After you create an access entry you can add and remove group names. For the IAM principal to have access to [.noloc]`Kubernetes` objects on your cluster, you must create and manage [.noloc]`Kubernetes` role-based authorization (RBAC) objects. Create [.noloc]`Kubernetes` `RoleBinding` or `ClusterRoleBinding` objects on your cluster that specify the group name as a `subject` for `kind: Group`. [.noloc]`Kubernetes` authorizes the IAM principal access to any cluster objects that you've specified in a [.noloc]`Kubernetes` `Role` or `ClusterRole` object that you've also specified in your binding's `roleRef`. If you specify group names, we recommend that you're familiar with the [.noloc]`Kubernetes` role-based authorization (RBAC) objects. For more information, see https://kubernetes.io/docs/reference/access-authn-authz/rbac/[Using RBAC Authorization] in the [.noloc]`Kubernetes` documentation. -+ -IMPORTANT: Amazon EKS doesn't confirm that any [.noloc]`Kubernetes` RBAC objects that exist on your cluster include any of the group names that you specify. For example, if you create an access entry for group that currently doesn't exist, EKS will create the group instead of returning an error. -+ -Instead of, or in addition to, [.noloc]`Kubernetes` authorizing the IAM principal access to [.noloc]`Kubernetes` objects on your cluster, you can associate Amazon EKS _access policies_ to an access entry. Amazon EKS authorizes IAM principals to access [.noloc]`Kubernetes` objects on your cluster with the permissions in the access policy. You can scope an access policy's permissions to [.noloc]`Kubernetes` namespaces that you specify. Use of access policies don't require you to manage [.noloc]`Kubernetes` RBAC objects. For more information, see <>. -* If you create an access entry with type `EC2_LINUX` or `EC2_Windows`, the IAM principal creating the access entry must have the `iam:PassRole` permission. For more information, see link:IAM/latest/UserGuide/id_roles_use_passrole.html[Granting a user permissions to pass a role to an {aws} service,type="documentation"] in the _IAM User Guide_. -* Similar to standard link:IAM/latest/UserGuide/troubleshoot_general.html#troubleshoot_general_eventual-consistency[IAM behavior,type="documentation"], access entry creation and updates are eventually consistent, and may take several seconds to be effective after the initial API call returns successfully. You must design your applications to account for these potential delays. We recommend that you don't include access entry creates or updates in the critical, high- availability code paths of your application. Instead, make changes in a separate initialization or setup routine that you run less frequently. Also, be sure to verify that the changes have been propagated before production workflows depend on them. -* Access entries do not support link:IAM/latest/UserGuide/using-service-linked-roles.html[service linked roles,type="documentation"]. You cannot create access entries where the principal ARN is a service linked role. You can identify service linked roles by their ARN, which is in the format `{arn-aws}iam::*:role/aws-service-role/*`. - -You can create an access entry using the {aws-management-console} or the {aws} CLI. - - -[[access-create-console,access-create-console.title]] -=== {aws-management-console} -. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. -. Choose the name of the cluster that you want to create an access entry in. -. Choose the *Access* tab. -. Choose *Create access entry*. -. For *IAM principal*, select an existing IAM role or user. IAM best practices recommend accessing your cluster using IAM _roles_ that have short-term credentials, rather than IAM _users_ that have long-term credentials. For more information, see link:IAM/latest/UserGuide/best-practices.html#bp-users-federation-idp[Require human users to use federation with an identity provider to access {aws} using temporary credentials,type="documentation"] in the _IAM User Guide_. -. For *Type*, if the access entry is for the node role used for self-managed Amazon EC2 nodes, select *EC2 Linux* or *EC2 Windows*. Otherwise, accept the default (*Standard*). -. If the *Type* you chose is *Standard* and you want to specify a *Username*, enter the username. -. If the *Type* you chose is *Standard* and you want to use [.noloc]`Kubernetes` RBAC authorization for the IAM principal, specify one or more names for *Groups*. If you don't specify any group names and want to use Amazon EKS authorization, you can associate an access policy in a later step, or after the access entry is created. -. (Optional) For *Tags*, assign labels to the access entry. For example, to make it easier to find all resources with the same tag. -. Choose *Next*. -. On the *Add access policy* page, if the type you chose was *Standard* and you want Amazon EKS to authorize the IAM principal to have permissions to the [.noloc]`Kubernetes` objects on your cluster, complete the following steps. Otherwise, choose *Next*. -+ -.. For *Policy name*, choose an access policy. You can't view the permissions of the access policies, but they include similar permissions to those in the [.noloc]`Kubernetes` user-facing `ClusterRole` objects. For more information, see https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles[User-facing roles] in the [.noloc]`Kubernetes` documentation. -.. Choose one of the following options: -+ -*** *Cluster* – Choose this option if you want Amazon EKS to authorize the IAM principal to have the permissions in the access policy for all [.noloc]`Kubernetes` objects on your cluster. -*** *[.noloc]`Kubernetes` namespace* – Choose this option if you want Amazon EKS to authorize the IAM principal to have the permissions in the access policy for all [.noloc]`Kubernetes` objects in a specific [.noloc]`Kubernetes` namespace on your cluster. For *Namespace*, enter the name of the [.noloc]`Kubernetes` namespace on your cluster. If you want to add additional namespaces, choose *Add new namespace* and enter the namespace name. -.. If you want to add additional policies, choose *Add policy*. You can scope each policy differently, but you can add each policy only once. -.. Choose *Next*. -. Review the configuration for your access entry. If anything looks incorrect, choose *Previous* to go back through the steps and correct the error. If the configuration is correct, choose *Create*. - -[[access-create-cli,access-create-cli.title]] -=== {aws} CLI - -. Install the {aws} CLI, as described in link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] in the {aws} Command Line Interface User Guide. -. To create an access entry -You can use any of the following examples to create access entries: -+ -** Create an access entry for a self-managed Amazon EC2 Linux node group. Replace [.replaceable]`my-cluster` with the name of your cluster, [.replaceable]`111122223333` with your {aws} account ID, and [.replaceable]`EKS-my-cluster-self-managed-ng-1` with the name of your link:eks/latest/userguide/create-node-role.html[node IAM role,type="documentation"]. If your node group is a Windows node group, then replace [.replaceable]`EC2_LINUX` with `EC2_Windows`. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks create-access-entry --cluster-name my-cluster --principal-arn {arn-aws}iam::111122223333:role/EKS-my-cluster-self-managed-ng-1 --type EC2_LINUX ----- -+ -You can't use the `--kubernetes-groups` option when you specify a type other than `STANDARD`. You can't associate an access policy to this access entry, because its type is a value other than `STANDARD`. -** Create an access entry that allows an IAM role that's not used for an Amazon EC2 self-managed node group, that you want [.noloc]`Kubernetes` to authorize access to your cluster with. Replace [.replaceable]`my-cluster` with the name of your cluster, [.replaceable]`111122223333` with your {aws} account ID, and [.replaceable]`my-role` with the name of your IAM role. Replace [.replaceable]`Viewers` with the name of a group that you've specified in a [.noloc]`Kubernetes` `RoleBinding` or `ClusterRoleBinding` object on your cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks create-access-entry --cluster-name my-cluster --principal-arn {arn-aws}iam::111122223333:role/my-role --type STANDARD --user Viewers --kubernetes-groups Viewers ----- -** Create an access entry that allows an IAM user to authenticate to your cluster. This example is provided because this is possible, though IAM best practices recommend accessing your cluster using IAM _roles_ that have short-term credentials, rather than IAM _users_ that have long-term credentials. For more information, see link:IAM/latest/UserGuide/best-practices.html#bp-users-federation-idp[Require human users to use federation with an identity provider to access {aws} using temporary credentials,type="documentation"] in the _IAM User Guide_. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks create-access-entry --cluster-name my-cluster --principal-arn {arn-aws}iam::111122223333:user/my-user --type STANDARD --username my-user ----- -+ -If you want this user to have more access to your cluster than the permissions in the [.noloc]`Kubernetes` API discovery roles, then you need to associate an access policy to the access entry, since the `--kubernetes-groups` option isn't used. For more information, see <> and https://kubernetes.io/docs/reference/access-authn-authz/rbac/#discovery-roles[API discovery roles] in the [.noloc]`Kubernetes` documentation. - - -[.topic] -[[updating-access-entries,updating-access-entries.title]] -== Update access entries - -You can update an access entry using the {aws-management-console} or the {aws} CLI. - - -[[access-update-console,access-update-console.title]] -=== {aws-management-console} -. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. -. Choose the name of the cluster that you want to create an access entry in. -. Choose the *Access* tab. -. Choose the access entry that you want to update. -. Choose *Edit*. -. For *Username*, you can change the existing value. -. For *Groups*, you can remove existing group names or add new group names. If the following groups names exist, don't remove them: *system:nodes* or *system:bootstrappers*. Removing these groups can cause your cluster to function improperly. If you don't specify any group names and want to use Amazon EKS authorization, associate an xref:access-policies[access policy,linkend=access-policies] in a later step. -. For *Tags*, you can assign labels to the access entry. For example, to make it easier to find all resources with the same tag. You can also remove existing tags. -. Choose *Save changes*. -. If you want to associate an access policy to the entry, see <>. - -[[access-update-cli,access-update-cli.title]] -=== {aws} CLI -. Install the {aws} CLI, as described in link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] in the {aws} Command Line Interface User Guide. -. To update an access entry -Replace [.replaceable]`my-cluster` with the name of your cluster, [.replaceable]`111122223333` with your {aws} account ID, and [.replaceable]`EKS-my-cluster-my-namespace-Viewers` with the name of an IAM role. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks update-access-entry --cluster-name my-cluster --principal-arn {arn-aws}iam::111122223333:role/EKS-my-cluster-my-namespace-Viewers --kubernetes-groups Viewers ----- -+ -You can't use the `--kubernetes-groups` option if the type of the access entry is a value other than `STANDARD`. You also can't associate an access policy to an access entry with a type other than `STANDARD`. +include::access-policies.adoc[leveloffset=+1] -[.topic] -[[deleting-access-entries,deleting-access-entries.title]] -== Delete access entries +include::migrating-access-entries.adoc[leveloffset=+1] -If you discover that you deleted an access entry in error, you can always recreate it. If the access entry that you're deleting is associated to any access policies, the associations are automatically deleted. You don't have to disassociate access policies from an access entry before deleting the access entry. +include::access-policy-reference.adoc[leveloffset=+1] -You can delete an access entry using the {aws-management-console} or the {aws} CLI. +include::setting-up-access-entries.adoc[leveloffset=+1] +include::creating-access-entries.adoc[leveloffset=+1] -[[access-delete-console,access-delete-console.title]] -=== {aws-management-console} -. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. -. Choose the name of the cluster that you want to delete an access entry from. -. Choose the *Access* tab. -. In the *Access entries* list, choose the access entry that you want to delete. -. Choose Delete. -. In the confirmation dialog box, choose *Delete*. +include::updating-access-entries.adoc[leveloffset=+1] -[[access-delete-cli,access-delete-cli.title]] -=== {aws} CLI -. Install the {aws} CLI, as described in link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] in the {aws} Command Line Interface User Guide. -. To delete an access entry -Replace [.replaceable]`my-cluster` with the name of your cluster, [.replaceable]`111122223333` with your {aws} account ID, and [.replaceable]`my-role` with the name of the IAM role that you no longer want to have access to your cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks delete-access-entry --cluster-name my-cluster --principal-arn {arn-aws}iam::111122223333:role/my-role ----- +include::deleting-access-entries.adoc[leveloffset=+1] +include::set-custom-username.adoc[leveloffset=+1] +include::create-standard-entry-policy.adoc[leveloffset=+1] +include::create-k8s-group-entry.adoc[leveloffset=+1] diff --git a/latest/ug/manage-access/k8s-access/access-policies.adoc b/latest/ug/manage-access/k8s-access/access-policies.adoc index 3fad2dfd6..70d254867 100644 --- a/latest/ug/manage-access/k8s-access/access-policies.adoc +++ b/latest/ug/manage-access/k8s-access/access-policies.adoc @@ -1,52 +1,50 @@ -//!!NODE_ROOT
+include::../../attributes.txt[] [.topic] -[[access-policies,access-policies.title]] +[#access-policies] = Associate access policies with access entries -:info_doctype: section - -include::../../attributes.txt[] +:info_titleabbrev: Associate access policies [abstract] -- Learn how to associate and disassociate Amazon EKS access policies to and from access entries to grant Kubernetes permissions to IAM principals. -- -You can assign one or more access policies to _access entries_ of _type_ `STANDARD`. Amazon EKS automatically grants the other types of access entries the permissions required to function properly in your cluster. Amazon EKS access policies include [.noloc]`Kubernetes` permissions, not IAM permissions. Before associating an access policy to an access entry, make sure that you're familiar with the [.noloc]`Kubernetes` permissions included in each access policy. For more information, see <>. If none of the access policies meet your requirements, then don't associate an access policy to an access entry. Instead, specify one or more _group names_ for the access entry and create and manage [.noloc]`Kubernetes` role-based access control objects. For more information, see <>. +You can assign one or more access policies to _access entries_ of _type_ `STANDARD`. Amazon EKS automatically grants the other types of access entries the permissions required to function properly in your cluster. Amazon EKS access policies include Kubernetes permissions, not IAM permissions. Before associating an access policy to an access entry, make sure that you're familiar with the Kubernetes permissions included in each access policy. For more information, see <>. If none of the access policies meet your requirements, then don't associate an access policy to an access entry. Instead, specify one or more _group names_ for the access entry and create and manage Kubernetes role-based access control objects. For more information, see <>. * An existing access entry. To create one, see <>. -* An {aws} Identity and Access Management role or user with the following permissions: `ListAccessEntries`, `DescribeAccessEntry`, `UpdateAccessEntry`, `ListAccessPolicies`, `AssociateAccessPolicy`, and `DisassociateAccessPolicy`. For more information, see link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html#amazonelastickubernetesservice-actions-as-permissions[Actions defined by Amazon Elastic Kubernetes Service,type="documentation"] in the _Service Authorization Reference_. +* An {aws} Identity and Access Management role or user with the following permissions: `ListAccessEntries`, `DescribeAccessEntry`, `UpdateAccessEntry`, `ListAccessPolicies`, `AssociateAccessPolicy`, and `DisassociateAccessPolicy`. For more information, see link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html#amazonelastickubernetesservice-actions-as-permissions[Actions defined by Amazon Elastic Kubernetes Service,type="documentation"] in the _Service Authorization Reference_. Before associating access policies with access entries, consider the following requirements: * You can associate multiple access policies to each access entry, but you can only associate each policy to an access entry once. If you associate multiple access policies, the access entry's IAM principal has all permissions included in all associated access policies. -* You can scope an access policy to all resources on a cluster or by specifying the name of one or more [.noloc]`Kubernetes` namespaces. You can use wildcard characters for a namespace name. For example, if you want to scope an access policy to all namespaces that start with `dev-`, you can specify `dev-*` as a namespace name. Make sure that the namespaces exist on your cluster and that your spelling matches the actual namespace name on the cluster. Amazon EKS doesn't confirm the spelling or existence of the namespaces on your cluster. -* You can change the _access scope_ for an access policy after you associate it to an access entry. If you've scoped the access policy to [.noloc]`Kubernetes` namespaces, you can add and remove namespaces for the association, as necessary. -* If you associate an access policy to an access entry that also has _group names_ specified, then the IAM principal has all the permissions in all associated access policies. It also has all the permissions in any [.noloc]`Kubernetes` `Role` or `ClusterRole` object that is specified in any [.noloc]`Kubernetes` `Role` and `RoleBinding` objects that specify the group names. -* If you run the `kubectl auth can-i --list` command, you won't see any [.noloc]`Kubernetes` permissions assigned by access policies associated with an access entry for the IAM principal you're using when you run the command. The command only shows [.noloc]`Kubernetes` permissions if you've granted them in [.noloc]`Kubernetes` `Role` or `ClusterRole` objects that you've bound to the group names or username that you specified for an access entry. -* If you impersonate a [.noloc]`Kubernetes` user or group when interacting with [.noloc]`Kubernetes` objects on your cluster, such as using the `kubectl` command with `--as [.replaceable]``username``` or `--as-group [.replaceable]``group-name```, you're forcing the use of [.noloc]`Kubernetes` RBAC authorization. As a result, the IAM principal has no permissions assigned by any access policies associated to the access entry. The only [.noloc]`Kubernetes` permissions that the user or group that the IAM principal is impersonating has are the [.noloc]`Kubernetes` permissions that you've granted them in [.noloc]`Kubernetes` `Role` or `ClusterRole` objects that you've bound to the group names or user name. For your IAM principal to have the permissions in associated access policies, don't impersonate a [.noloc]`Kubernetes` user or group. The IAM principal will still also have any permissions that you've granted them in the [.noloc]`Kubernetes` `Role` or `ClusterRole` objects that you've bound to the group names or user name that you specified for the access entry. For more information, see https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation[User impersonation] in the [.noloc]`Kubernetes` documentation. +* You can scope an access policy to all resources on a cluster or by specifying the name of one or more Kubernetes namespaces. You can use wildcard characters for a namespace name. For example, if you want to scope an access policy to all namespaces that start with `dev-`, you can specify `dev-*` as a namespace name. Make sure that the namespaces exist on your cluster and that your spelling matches the actual namespace name on the cluster. Amazon EKS doesn't confirm the spelling or existence of the namespaces on your cluster. +* You can change the _access scope_ for an access policy after you associate it to an access entry. If you've scoped the access policy to Kubernetes namespaces, you can add and remove namespaces for the association, as necessary. +* If you associate an access policy to an access entry that also has _group names_ specified, then the IAM principal has all the permissions in all associated access policies. It also has all the permissions in any Kubernetes `Role` or `ClusterRole` object that is specified in any Kubernetes `Role` and `RoleBinding` objects that specify the group names. +* If you run the `kubectl auth can-i --list` command, you won't see any Kubernetes permissions assigned by access policies associated with an access entry for the IAM principal you're using when you run the command. The command only shows Kubernetes permissions if you've granted them in Kubernetes `Role` or `ClusterRole` objects that you've bound to the group names or username that you specified for an access entry. +* If you impersonate a Kubernetes user or group when interacting with Kubernetes objects on your cluster, such as using the `kubectl` command with `--as [.replaceable]``username``` or `--as-group [.replaceable]``group-name```, you're forcing the use of Kubernetes RBAC authorization. As a result, the IAM principal has no permissions assigned by any access policies associated to the access entry. The only Kubernetes permissions that the user or group that the IAM principal is impersonating has are the Kubernetes permissions that you've granted them in Kubernetes `Role` or `ClusterRole` objects that you've bound to the group names or user name. For your IAM principal to have the permissions in associated access policies, don't impersonate a Kubernetes user or group. The IAM principal will still also have any permissions that you've granted them in the Kubernetes `Role` or `ClusterRole` objects that you've bound to the group names or user name that you specified for the access entry. For more information, see https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation[User impersonation] in the Kubernetes documentation. You can associate an access policy to an access entry using the {aws-management-console} or the {aws} CLI. -[[access-associate-console,access-associate-console.title]] +[#access-associate-console] == {aws-management-console} . Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. . Choose the name of the cluster that has an access entry that you want to associate an access policy to. . Choose the *Access* tab. -. If the type of the access entry is *Standard*, you can associate or disassociate Amazon EKS *access policies*. If the type of your access entry is anything other than *Standard*, then this option isn't available. +. If the type of the access entry is *Standard*, you can associate or disassociate Amazon EKS *access policies*. If the type of your access entry is anything other than *Standard*, then this option isn't available. . Choose *Associate access policy*. . For *Policy name*, select the policy with the permissions you want the IAM principal to have. To view the permissions included in each policy, see <>. -. For *Access scope*, choose an access scope. If you choose *Cluster*, the permissions in the access policy are granted to the IAM principal for resources in all [.noloc]`Kubernetes` namespaces. If you choose *[.noloc]`Kubernetes` namespace*, you can then choose *Add new namespace*. In the *Namespace* field that appears, you can enter the name of a [.noloc]`Kubernetes` namespace on your cluster. If you want the IAM principal to have the permissions across multiple namespaces, then you can enter multiple namespaces. +. For *Access scope*, choose an access scope. If you choose *Cluster*, the permissions in the access policy are granted to the IAM principal for resources in all Kubernetes namespaces. If you choose *Kubernetes namespace*, you can then choose *Add new namespace*. In the *Namespace* field that appears, you can enter the name of a Kubernetes namespace on your cluster. If you want the IAM principal to have the permissions across multiple namespaces, then you can enter multiple namespaces. . Choose *Add access policy*. -[[access-associate-cli,access-associate-cli.title]] +[#access-associate-cli] == {aws} CLI -. Version `2.12.3` or later or version `1.27.160` or later of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device or {aws} CloudShell. To check your current version, use `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or [.noloc]`Homebrew` for [.noloc]`macOS` are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the _{aws} Command Line Interface User Guide_. The {aws} CLI version that is installed in {aws} CloudShell might also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the _{aws} CloudShell User Guide_. +. Version `2.12.3` or later or version `1.27.160` or later of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device or {aws} CloudShell. To check your current version, use `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or Homebrew for macOS are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the _{aws} Command Line Interface User Guide_. The {aws} CLI version that is installed in {aws} CloudShell might also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the _{aws} CloudShell User Guide_. + . View the available access policies. + @@ -93,7 +91,7 @@ An example output is as follows. ] } ---- -. Associate an access policy to an access entry. The following example associates the `AmazonEKSViewPolicy` access policy to an access entry. Whenever the [.replaceable]`my-role` IAM role attempts to access [.noloc]`Kubernetes` objects on the cluster, Amazon EKS will authorize the role to use the permissions in the policy to access [.noloc]`Kubernetes` objects in the [.replaceable]`my-namespace1` and [.replaceable]`my-namespace2` [.noloc]`Kubernetes` namespaces only. Replace [.replaceable]`my-cluster` with the name of your cluster, [.replaceable]`111122223333` with your {aws} account ID, and [.replaceable]`my-role` with the name of the IAM role that you want Amazon EKS to authorize access to [.noloc]`Kubernetes` cluster objects for. +. Associate an access policy to an access entry. The following example associates the `AmazonEKSViewPolicy` access policy to an access entry. Whenever the [.replaceable]`my-role` IAM role attempts to access Kubernetes objects on the cluster, Amazon EKS will authorize the role to use the permissions in the policy to access Kubernetes objects in the [.replaceable]`my-namespace1` and [.replaceable]`my-namespace2` Kubernetes namespaces only. Replace [.replaceable]`my-cluster` with the name of your cluster, [.replaceable]`111122223333` with your {aws} account ID, and [.replaceable]`my-role` with the name of the IAM role that you want Amazon EKS to authorize access to Kubernetes cluster objects for. + [source,bash,subs="verbatim,attributes"] ---- @@ -144,7 +142,7 @@ An example output is as follows. } ---- + -In the previous example, the IAM principal for this access entry has view permissions across all namespaces on the cluster, and administrator permissions to two [.noloc]`Kubernetes` namespaces. +In the previous example, the IAM principal for this access entry has view permissions across all namespaces on the cluster, and administrator permissions to two Kubernetes namespaces. . Disassociate an access policy from an access entry. In this example, the `AmazonEKSAdminPolicy` policy is disassociated from an access entry. The IAM principal retains the permissions in the `AmazonEKSViewPolicy` access policy for objects in the [.replaceable]`my-namespace1` and [.replaceable]`my-namespace2` namespaces however, because that access policy is not disassociated from the access entry. + [source,bash,subs="verbatim,attributes"] diff --git a/latest/ug/manage-access/k8s-access/access-policy-reference.adoc b/latest/ug/manage-access/k8s-access/access-policy-reference.adoc index 833184b69..5231c0a2c 100644 --- a/latest/ug/manage-access/k8s-access/access-policy-reference.adoc +++ b/latest/ug/manage-access/k8s-access/access-policy-reference.adoc @@ -1,26 +1,61 @@ +include::../../attributes.txt[] -//!!NODE_ROOT
[.topic] -[[access-policy-permissions,access-policy-permissions.title]] +[#access-policy-permissions] = Review access policy permissions -:info_doctype: section - -include::../../attributes.txt[] - -Access policies include `rules` that contain [.noloc]`Kubernetes` `verbs` (permissions) and `resources`. Access policies don't include IAM permissions or resources. Similar to [.noloc]`Kubernetes` `Role` and `ClusterRole` objects, access policies only include `allow` `rules`. You can't modify the contents of an access policy. You can't create your own access policies. If the permissions in the access policies don't meet your needs, then create [.noloc]`Kubernetes` RBAC objects and specify _group names_ for your access entries. For more information, see <>. The permissions contained in access policies are similar to the permissions in the [.noloc]`Kubernetes` user-facing cluster roles. For more information, see https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles[User-facing roles] in the [.noloc]`Kubernetes` documentation. - -Choose any access policy to see its contents. Each row of each table in each access policy is a separate rule. - - -[[access-policy-permissions-amazoneksadminpolicy,access-policy-permissions-amazoneksadminpolicy.title]] +:info_titleabbrev: Review access policies + +Access policies include `rules` that contain Kubernetes `verbs` (permissions) and `resources`. Access policies don't include IAM permissions or resources. Similar to Kubernetes `Role` and `ClusterRole` objects, access policies only include `allow` `rules`. You can't modify the contents of an access policy. You can't create your own access policies. If the permissions in the access policies don't meet your needs, then create Kubernetes RBAC objects and specify _group names_ for your access entries. For more information, see <>. The permissions contained in access policies are similar to the permissions in the Kubernetes user-facing cluster roles. For more information, see https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles[User-facing roles] in the Kubernetes documentation. + +[#access-policies-cli-command] +== List all policies +Use any one of the access policies listed on this page, or retrieve a list of all available access policies using the {aws} CLI: + +[source,bash,subs="verbatim,attributes"] +---- +aws eks list-access-policies +---- + +The expected output should look like this (abbreviated for brevity): + +[source] +---- +{ + "accessPolicies": [ + { + "name": "AmazonAIOpsAssistantPolicy", + "arn": "arn:aws:eks::aws:cluster-access-policy/AmazonAIOpsAssistantPolicy" + }, + { + "name": "AmazonARCRegionSwitchScalingPolicy", + "arn": "arn:aws:eks::aws:cluster-access-policy/AmazonARCRegionSwitchScalingPolicy" + }, + { + "name": "AmazonEKSAdminPolicy", + "arn": "arn:aws:eks::aws:cluster-access-policy/AmazonEKSAdminPolicy" + }, + { + "name": "AmazonEKSAdminViewPolicy", + "arn": "arn:aws:eks::aws:cluster-access-policy/AmazonEKSAdminViewPolicy" + }, + { + "name": "AmazonEKSAutoNodePolicy", + "arn": "arn:aws:eks::aws:cluster-access-policy/AmazonEKSAutoNodePolicy" + } + // Additional policies omitted + ] +} +---- + +[#access-policy-permissions-amazoneksadminpolicy] == AmazonEKSAdminPolicy -This access policy includes permissions that grant an IAM principal most permissions to resources. When associated to an access entry, its access scope is typically one or more [.noloc]`Kubernetes` namespaces. If you want an IAM principal to have administrator access to all resources on your cluster, associate the <> access policy to your access entry instead. +This access policy includes permissions that grant an IAM principal most permissions to resources. When associated to an access entry, its access scope is typically one or more Kubernetes namespaces. If you want an IAM principal to have administrator access to all resources on your cluster, associate the <> access policy to your access entry instead. *ARN* – `{arn-aws}eks::aws:cluster-access-policy/AmazonEKSAdminPolicy` -[cols="1,1,1", options="header"] +[%header,cols="3"] |=== |Kubernetes API groups |Kubernetes resources @@ -114,18 +149,19 @@ This access policy includes permissions that grant an IAM principal most permiss | |`namespaces` |`get`,``list``, `watch` + |=== -[[access-policy-permissions-amazoneksclusteradminpolicy,access-policy-permissions-amazoneksclusteradminpolicy.title]] +[#access-policy-permissions-amazoneksclusteradminpolicy] == AmazonEKSClusterAdminPolicy -This access policy includes permissions that grant an IAM principal administrator access to a cluster. When associated to an access entry, its access scope is typically the cluster, rather than a [.noloc]`Kubernetes` namespace. If you want an IAM principal to have a more limited administrative scope, consider associating the <> access policy to your access entry instead. +This access policy includes permissions that grant an IAM principal administrator access to a cluster. When associated to an access entry, its access scope is typically the cluster, rather than a Kubernetes namespace. If you want an IAM principal to have a more limited administrative scope, consider associating the <> access policy to your access entry instead. *ARN* – `{arn-aws}eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy` -[cols="1,1,1,1", options="header"] +[%header,cols="4"] |=== |Kubernetes API groups |Kubernetes nonResourceURLs @@ -142,17 +178,18 @@ This access policy includes permissions that grant an IAM principal administrato |`{asterisk}` | |`{asterisk}` + |=== -[[access-policy-permissions-amazoneksadminviewpolicy,access-policy-permissions-amazoneksadminviewpolicy.title]] +[#access-policy-permissions-amazoneksadminviewpolicy] == AmazonEKSAdminViewPolicy This access policy includes permissions that grant an IAM principal access to list/view all resources in a cluster. Note this includes https://kubernetes.io/docs/concepts/configuration/secret/[Kubernetes Secrets.] *ARN* – `{arn-aws}eks::aws:cluster-access-policy/AmazonEKSAdminViewPolicy` -[cols="1,1,1", options="header"] +[%header,cols="3"] |=== |Kubernetes API groups |Kubernetes resources @@ -162,18 +199,19 @@ This access policy includes permissions that grant an IAM principal access to li |`{asterisk}` |`{asterisk}` |`get`, `list`, `watch` + |=== -[[access-policy-permissions-amazonekseditpolicy,access-policy-permissions-amazonekseditpolicy.title]] +[#access-policy-permissions-amazonekseditpolicy] == AmazonEKSEditPolicy -This access policy includes permissions that allow an IAM principal to edit most [.noloc]`Kubernetes` resources. +This access policy includes permissions that allow an IAM principal to edit most Kubernetes resources. *ARN* – `{arn-aws}eks::aws:cluster-access-policy/AmazonEKSEditPolicy` -[cols="1,1,1", options="header"] +[%header,cols="3"] |=== |Kubernetes API groups |Kubernetes resources @@ -262,14 +300,14 @@ This access policy includes permissions that allow an IAM principal to edit most |=== -[[access-policy-permissions-amazoneksviewpolicy.json,access-policy-permissions-amazoneksviewpolicy.json.title]] +[#access-policy-permissions-amazoneksviewpolicy.json] == AmazonEKSViewPolicy -This access policy includes permissions that allow an IAM principal to view most [.noloc]`Kubernetes` resources. +This access policy includes permissions that allow an IAM principal to view most Kubernetes resources. *ARN* – `{arn-aws}eks::aws:cluster-access-policy/AmazonEKSViewPolicy` -[cols="1,1,1", options="header"] +[%header,cols="3"] |=== |Kubernetes API groups |Kubernetes resources @@ -317,6 +355,24 @@ This access policy includes permissions that allow an IAM principal to view most |`get`, `list`, `watch` |=== +== AmazonEKSSecretReaderPolicy + +This access policy includes permissions that allow an IAM principal to read https://kubernetes.io/docs/concepts/configuration/secret/[Kubernetes Secrets.] + +*ARN* – `{arn-aws}eks::aws:cluster-access-policy/AmazonEKSSecretReaderPolicy` + +[%header,cols="3"] +|=== +|Kubernetes API groups +|Kubernetes resources +|Kubernetes verbs (permissions) + + +| +|`secrets` +|`get`, `list`, `watch` +|=== + == AmazonEKSAutoNodePolicy @@ -338,6 +394,11 @@ If you manually specifiy a Node IAM role in a NodeClass, you need to create an A == AmazonEKSBlockStoragePolicy +[NOTE] +==== +This policy is designated for {aws} service-linked roles only and cannot be used with customer-managed roles. +==== + *ARN* – `{arn-aws}eks::aws:cluster-access-policy/AmazonEKSBlockStoragePolicy` This policy includes permissions that allow Amazon EKS to manage leader election and coordination resources for storage operations: @@ -378,6 +439,11 @@ Amazon EKS automatically creates an access entry with this access policy for the == AmazonEKSComputePolicy +[NOTE] +==== +This policy is designated for {aws} service-linked roles only and cannot be used with customer-managed roles. +==== + *ARN* – `{arn-aws}eks::aws:cluster-access-policy/AmazonEKSComputePolicy` This policy includes permissions that allow Amazon EKS to manage leader election resources for compute operations: @@ -390,6 +456,11 @@ Amazon EKS automatically creates an access entry with this access policy for the == AmazonEKSBlockStorageClusterPolicy +[NOTE] +==== +This policy is designated for {aws} service-linked roles only and cannot be used with customer-managed roles. +==== + *ARN* – `{arn-aws}eks::aws:cluster-access-policy/AmazonEKSBlockStorageClusterPolicy` This policy grants permissions necessary for the block storage capability of Amazon EKS Auto Mode. It enables efficient management of block storage resources within Amazon EKS clusters. The policy includes the following permissions: @@ -429,6 +500,11 @@ Amazon EKS automatically creates an access entry with this access policy for the == AmazonEKSComputeClusterPolicy +[NOTE] +==== +This policy is designated for {aws} service-linked roles only and cannot be used with customer-managed roles. +==== + *ARN* – `{arn-aws}eks::aws:cluster-access-policy/AmazonEKSComputeClusterPolicy` This policy grants permissions necessary for the compute management capability of Amazon EKS Auto Mode. It enables efficient orchestration and scaling of compute resources within Amazon EKS clusters. The policy includes the following permissions: @@ -533,15 +609,19 @@ The policy allows the networking components to interact with node-related resour Amazon EKS automatically creates an access entry with this access policy for the cluster IAM role when Auto Mode is enabled, ensuring that the necessary permissions are in place for the networking capability to function properly. -[[access-policy-permissions-amazonekshybridpolicy,access-policy-permissions-amazonekshybridpolicy.title]] +[#access-policy-permissions-amazonekshybridpolicy] == AmazonEKSHybridPolicy +[NOTE] +==== +This policy is designated for {aws} service-linked roles only and cannot be used with customer-managed roles. +==== -This access policy includes permissions that grant EKS access to the nodes of a cluster. When associated to an access entry, its access scope is typically the cluster, rather than a [noloc]``Kubernetes`` namespace. This policy is used by Amazon EKS hybrid nodes. +This access policy includes permissions that grant EKS access to the nodes of a cluster. When associated to an access entry, its access scope is typically the cluster, rather than a Kubernetes namespace. This policy is used by Amazon EKS hybrid nodes. -*ARN* – `arn:aws:eks::aws:cluster-access-policy/AmazonEKSHybridPolicy` +*ARN* – `{arn-aws}eks::aws:cluster-access-policy/AmazonEKSHybridPolicy` -[cols="1,1,1,1", options="header"] +[%header,cols="4"] |=== | Kubernetes API groups | Kubernetes nonResourceURLs @@ -556,17 +636,46 @@ This access policy includes permissions that grant EKS access to the nodes of a |=== -[[access-policy-updates,access-policy-updates.title]] +[[access-policy-permissions-AmazonEKSClusterInsightsPolicy,access-policy-permissions-AmazonEKSClusterInsightsPolicy.title]] +== AmazonEKSClusterInsightsPolicy + +[NOTE] +==== +This policy is designated for {aws} service-linked roles only and cannot be used with customer-managed roles. +==== + +*ARN* – `{arn-aws}eks::aws:cluster-access-policy/AmazonEKSClusterInsightsPolicy` + +This policy grants read-only permissions for Amazon EKS Cluster Insights functionality. The policy includes the following permissions: + +Node Access: +- List and view cluster nodes +- Read node status information + +DaemonSet Access: +- Read access to kube-proxy configuration + +This policy is automatically managed by the EKS service for Cluster Insights. For more information, see <>. + +[#access-policy-updates] == Access policy updates View details about updates to access policies, since they were introduced. For automatic alerts about changes to this page, subscribe to the RSS feed in <>. -[cols="1,1,1", options="header"] +[%header,cols="3"] |=== |Change |Description |Date +|Add `AmazonEKSSecretReaderPolicy` +|Add a new policy for read-only access to secrets +|November 6, 2025 + +|Add policy for EKS Cluster Insights +|Publish `AmazonEKSClusterInsightsPolicy` +|December 2, 2024 + |Add policies for Amazon EKS Hybrid |Publish `AmazonEKSHybridPolicy` |December 2, 2024 diff --git a/latest/ug/manage-access/k8s-access/auth-configmap.adoc b/latest/ug/manage-access/k8s-access/auth-configmap.adoc index 4ceeb3b3d..257749a44 100644 --- a/latest/ug/manage-access/k8s-access/auth-configmap.adoc +++ b/latest/ug/manage-access/k8s-access/auth-configmap.adoc @@ -1,12 +1,9 @@ -//!!NODE_ROOT
- -[.topic] -[[auth-configmap,auth-configmap.title]] -= Grant [.noloc]`IAM` users access to [.noloc]`Kubernetes` with a [.noloc]`ConfigMap` -:info_doctype: section - include::../../attributes.txt[] +[.topic] +[#auth-configmap] += Grant IAM users access to Kubernetes with a ConfigMap +:info_titleabbrev: aws-auth ConfigMap [abstract] -- @@ -16,21 +13,21 @@ Learn how to manage IAM principal access to your Amazon EKS cluster using the aw [IMPORTANT] ==== -The `aws-auth ConfigMap` is deprecated. For the recommended method to manage access to [.noloc]`Kubernetes` APIs, see <>. +The `aws-auth ConfigMap` is deprecated. For the recommended method to manage access to Kubernetes APIs, see <>. ==== -Access to your cluster using link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principals,type="documentation"] is enabled by the https://github.com/kubernetes-sigs/aws-iam-authenticator#readme[{aws} IAM Authenticator for Kubernetes], which runs on the Amazon EKS control plane. The authenticator gets its configuration information from the `aws-auth` `ConfigMap`. For all `aws-auth` `ConfigMap` settings, see https://github.com/kubernetes-sigs/aws-iam-authenticator#full-configuration-format[Full Configuration Format] on [.noloc]`GitHub`. +Access to your cluster using link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principals,type="documentation"] is enabled by the https://github.com/kubernetes-sigs/aws-iam-authenticator#readme[{aws} IAM Authenticator for Kubernetes], which runs on the Amazon EKS control plane. The authenticator gets its configuration information from the `aws-auth` `ConfigMap`. For all `aws-auth` `ConfigMap` settings, see https://github.com/kubernetes-sigs/aws-iam-authenticator#full-configuration-format[Full Configuration Format] on GitHub. -[[aws-auth-users,aws-auth-users.title]] +[#aws-auth-users] == Add IAM principals to your Amazon EKS cluster -When you create an Amazon EKS cluster, the link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] that creates the cluster is automatically granted `system:masters` permissions in the cluster's role-based access control (RBAC) configuration in the Amazon EKS control plane. This principal doesn't appear in any visible configuration, so make sure to keep track of which principal originally created the cluster. To grant additional IAM principals the ability to interact with your cluster, edit the `aws-auth ConfigMap` within [.noloc]`Kubernetes` and create a [.noloc]`Kubernetes` `rolebinding` or `clusterrolebinding` with the name of a `group` that you specify in the `aws-auth ConfigMap`. +When you create an Amazon EKS cluster, the link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] that creates the cluster is automatically granted `system:masters` permissions in the cluster's role-based access control (RBAC) configuration in the Amazon EKS control plane. This principal doesn't appear in any visible configuration, so make sure to keep track of which principal originally created the cluster. To grant additional IAM principals the ability to interact with your cluster, edit the `aws-auth ConfigMap` within Kubernetes and create a Kubernetes `rolebinding` or `clusterrolebinding` with the name of a `group` that you specify in the `aws-auth ConfigMap`. [NOTE] ==== -For more information about [.noloc]`Kubernetes` role-based access control (RBAC) configuration, see https://kubernetes.io/docs/reference/access-authn-authz/rbac/[Using RBAC Authorization] in the [.noloc]`Kubernetes` documentation. +For more information about Kubernetes role-based access control (RBAC) configuration, see https://kubernetes.io/docs/reference/access-authn-authz/rbac/[Using RBAC Authorization] in the Kubernetes documentation. ==== . Determine which credentials `kubectl` is using to access your cluster. On your computer, you can see which credentials `kubectl` uses with the following command. Replace [.replaceable]`~/.kube/config` with the path to your `kubeconfig` file if you don't use the default path. @@ -54,7 +51,7 @@ current-context: admin@my-cluster.region-code.eksctl.io [...] ---- + -In the previous example output, the credentials for a user named [.replaceable]`admin` are configured for a cluster named [.replaceable]`my-cluster`. If this is the user that created the cluster, then it already has access to your cluster. If it's not the user that created the cluster, then you need to complete the remaining steps to enable cluster access for other IAM principals. link:IAM/latest/UserGuide/id_users.html[IAM best practices,type="documentation"] recommend that you grant permissions to roles instead of users. You can see which other principals currently have access to your cluster with the following command: +In the previous example output, the credentials for a user named [.replaceable]`admin` are configured for a cluster named [.replaceable]`my-cluster`. If this is the user that created the cluster, then it already has access to your cluster. If it's not the user that created the cluster, then you need to complete the remaining steps to enable cluster access for other IAM principals. link:IAM/latest/UserGuide/id_users.html[IAM best practices,type="documentation"] recommend that you grant permissions to roles instead of users. You can see which other principals currently have access to your cluster with the following command: + [source,bash,subs="verbatim,attributes"] ---- @@ -88,9 +85,9 @@ Events: .... + The previous example is a default `aws-auth` `ConfigMap`. Only the node instance role has access to the cluster. -. Make sure that you have existing [.noloc]`Kubernetes` `roles` and `rolebindings` or `clusterroles` and `clusterrolebindings` that you can map IAM principals to. For more information about these resources, see https://kubernetes.io/docs/reference/access-authn-authz/rbac/[Using RBAC Authorization] in the [.noloc]`Kubernetes` documentation. +. Make sure that you have existing Kubernetes `roles` and `rolebindings` or `clusterroles` and `clusterrolebindings` that you can map IAM principals to. For more information about these resources, see https://kubernetes.io/docs/reference/access-authn-authz/rbac/[Using RBAC Authorization] in the Kubernetes documentation. + -.. View your existing [.noloc]`Kubernetes` `roles` or `clusterroles`. `Roles` are scoped to a `namespace`, but `clusterroles` are scoped to the cluster. +.. View your existing Kubernetes `roles` or `clusterroles`. `Roles` are scoped to a `namespace`, but `clusterroles` are scoped to the cluster. + [source,bash,subs="verbatim,attributes"] ---- @@ -116,7 +113,7 @@ Replace [.replaceable]`cluster-role-name` with a `clusterrole` name returned in ---- kubectl describe clusterrole cluster-role-name ---- -.. View your existing [.noloc]`Kubernetes` `rolebindings` or `clusterrolebindings`. `Rolebindings` are scoped to a `namespace`, but `clusterrolebindings` are scoped to the cluster. +.. View your existing Kubernetes `rolebindings` or `clusterrolebindings`. `Rolebindings` are scoped to a `namespace`, but `clusterrolebindings` are scoped to the cluster. + [source,bash,subs="verbatim,attributes"] ---- @@ -184,10 +181,10 @@ roleRef: + IMPORTANT: We recommend using `eksctl`, or another tool, to edit the `ConfigMap`. For information about other tools you can use, see https://aws.github.io/aws-eks-best-practices/security/docs/iam/#use-tools-to-make-changes-to-the-aws-auth-configmap[Use tools to make changes to the aws-authConfigMap] in the Amazon EKS best practices guides. An improperly formatted `aws-auth` `ConfigMap` can cause you to lose access to your cluster. + -** View steps to xref:configmap-eksctl[edit configmap with eksctl]. -** View steps to xref:configmap-manual[edit configmap manually]. +** View steps to <>. +** View steps to <>. -[[configmap-eksctl,configmap-eksctl.title]] +[#configmap-eksctl] === Edit Configmap with Eksctl . You need version `{eksctl-min-version}` or later of the `eksctl` command line tool installed on your device or {aws} CloudShell. To install or update `eksctl`, see https://eksctl.io/installation[Installation] in the `eksctl` documentation. @@ -206,7 +203,7 @@ An example output is as follows. ARN USERNAME GROUPS ACCOUNT {arn-aws}iam::111122223333:role/eksctl-my-cluster-my-nodegroup-NodeInstanceRole-1XLS7754U3ZPA system:node:{{EC2PrivateDNSName}} system:bootstrappers,system:nodes ---- -. Add a mapping for a role. Replace [.replaceable]`my-role` with your role name. Replace [.replaceable]`eks-console-dashboard-full-access-group` with the name of the group specified in your [.noloc]`Kubernetes` `RoleBinding` or `ClusterRoleBinding` object. Replace [.replaceable]`111122223333` with your account ID. You can replace [.replaceable]`admin` with any name you choose. +. Add a mapping for a role. Replace [.replaceable]`my-role` with your role name. Replace [.replaceable]`eks-console-dashboard-full-access-group` with the name of the group specified in your Kubernetes `RoleBinding` or `ClusterRoleBinding` object. Replace [.replaceable]`111122223333` with your account ID. You can replace [.replaceable]`admin` with any name you choose. + [source,bash,subs="verbatim,attributes"] ---- @@ -224,7 +221,7 @@ An example output is as follows. [...] 2022-05-09 14:51:20 [ℹ] adding identity "{arn-aws}iam::111122223333:role/my-role" to auth ConfigMap ---- -. Add a mapping for a user. link:IAM/latest/UserGuide/id_users.html[IAM best practices,type="documentation"] recommend that you grant permissions to roles instead of users. Replace [.replaceable]`my-user` with your user name. Replace [.replaceable]`eks-console-dashboard-restricted-access-group` with the name of the group specified in your [.noloc]`Kubernetes` `RoleBinding` or `ClusterRoleBinding` object. Replace [.replaceable]`111122223333` with your account ID. You can replace [.replaceable]`my-user` with any name you choose. +. Add a mapping for a user. link:IAM/latest/UserGuide/id_users.html[IAM best practices,type="documentation"] recommend that you grant permissions to roles instead of users. Replace [.replaceable]`my-user` with your user name. Replace [.replaceable]`eks-console-dashboard-restricted-access-group` with the name of the group specified in your Kubernetes `RoleBinding` or `ClusterRoleBinding` object. Replace [.replaceable]`111122223333` with your account ID. You can replace [.replaceable]`my-user` with any name you choose. + [source,bash,subs="verbatim,attributes"] ---- @@ -257,7 +254,7 @@ ARN {arn-aws}iam::111122223333:user/my-user my-user eks-console-dashboard-restricted-access-group ---- -[[configmap-manual,configmap-manual.title]] +[#configmap-manual] === Edit Configmap manually . Open the `ConfigMap` for editing. + @@ -269,23 +266,23 @@ kubectl edit -n kube-system configmap/aws-auth NOTE: If you receive an error stating "``Error from server (NotFound): configmaps "aws-auth" not found``", then use the procedure in <> to apply the stock `ConfigMap`. . Add your IAM principals to the `ConfigMap`. An IAM group isn't an IAM principal, so it can't be added to the `ConfigMap`. + -** *To add an IAM role (for example, for link:IAM/latest/UserGuide/id_roles_providers.html[federated users,type="documentation"]):* Add the role details to the `mapRoles` section of the `ConfigMap`, under `data`. Add this section if it does not already exist in the file. Each entry supports the following parameters: +** *To add an IAM role (for example, for link:IAM/latest/UserGuide/id_roles_providers.html[federated users,type="documentation"]):* Add the role details to the `mapRoles` section of the `ConfigMap`, under `data`. Add this section if it does not already exist in the file. Each entry supports the following parameters: + *** *rolearn*: The ARN of the IAM role to add. This value can't include a path. For example, you can't specify an ARN such as `{arn-aws}iam::[.replaceable]``111122223333``:role/my-team/developers/[.replaceable]``role-name```. The ARN needs to be `{arn-aws}iam::[.replaceable]``111122223333``:role/[.replaceable]``role-name``` instead. -*** *username*: The user name within [.noloc]`Kubernetes` to map to the IAM role. -*** *groups*: The group or list of [.noloc]`Kubernetes` groups to map the role to. The group can be a default group, or a group specified in a `clusterrolebinding` or `rolebinding`. For more information, see https://kubernetes.io/docs/reference/access-authn-authz/rbac/#default-roles-and-role-bindings[Default roles and role bindings] in the [.noloc]`Kubernetes` documentation. +*** *username*: The user name within Kubernetes to map to the IAM role. +*** *groups*: The group or list of Kubernetes groups to map the role to. The group can be a default group, or a group specified in a `clusterrolebinding` or `rolebinding`. For more information, see https://kubernetes.io/docs/reference/access-authn-authz/rbac/#default-roles-and-role-bindings[Default roles and role bindings] in the Kubernetes documentation. ** *To add an IAM user:* link:IAM/latest/UserGuide/id_users.html[IAM best practices,type="documentation"] recommend that you grant permissions to roles instead of users. Add the user details to the `mapUsers` section of the `ConfigMap`, under `data`. Add this section if it does not already exist in the file. Each entry supports the following parameters: + *** *userarn*: The ARN of the IAM user to add. -*** *username*: The user name within [.noloc]`Kubernetes` to map to the IAM user. -*** *groups*: The group, or list of [.noloc]`Kubernetes` groups to map the user to. The group can be a default group, or a group specified in a `clusterrolebinding` or `rolebinding`. For more information, see https://kubernetes.io/docs/reference/access-authn-authz/rbac/#default-roles-and-role-bindings[Default roles and role bindings] in the [.noloc]`Kubernetes` documentation. +*** *username*: The user name within Kubernetes to map to the IAM user. +*** *groups*: The group, or list of Kubernetes groups to map the user to. The group can be a default group, or a group specified in a `clusterrolebinding` or `rolebinding`. For more information, see https://kubernetes.io/docs/reference/access-authn-authz/rbac/#default-roles-and-role-bindings[Default roles and role bindings] in the Kubernetes documentation. + . For example, the following YAML block contains: -** A `mapRoles` section that maps the IAM node instance to [.noloc]`Kubernetes` groups so that nodes can register themselves with the cluster and the `my-console-viewer-role` IAM role that is mapped to a [.noloc]`Kubernetes` group that can view all [.noloc]`Kubernetes` resources for all clusters. For a list of the IAM and [.noloc]`Kubernetes` group permissions required for the `my-console-viewer-role` IAM role, see <>. -** A `mapUsers` section that maps the `admin` IAM user from the default {aws} account to the `system:masters` [.noloc]`Kubernetes` group and the `my-user` user from a different {aws} account that is mapped to a [.noloc]`Kubernetes` group that can view [.noloc]`Kubernetes` resources for a specific namespace. For a list of the IAM and [.noloc]`Kubernetes` group permissions required for the `my-user` IAM user, see <>. +** A `mapRoles` section that maps the IAM node instance to Kubernetes groups so that nodes can register themselves with the cluster and the `my-console-viewer-role` IAM role that is mapped to a Kubernetes group that can view all Kubernetes resources for all clusters. For a list of the IAM and Kubernetes group permissions required for the `my-console-viewer-role` IAM role, see <>. +** A `mapUsers` section that maps the `admin` IAM user from the default {aws} account to the `system:masters` Kubernetes group and the `my-user` user from a different {aws} account that is mapped to a Kubernetes group that can view Kubernetes resources for a specific namespace. For a list of the IAM and Kubernetes group permissions required for the `my-user` IAM user, see <>. + -Add or remove lines as necessary and replace all [.replaceable]`example values` with your own values. +Add or remove lines as necessary and replace all example values with your own values. + [source,yaml,subs="verbatim,attributes"] ---- @@ -318,7 +315,7 @@ data: . Save the file and exit your text editor. -[[aws-auth-configmap,aws-auth-configmap.title]] +[#aws-auth-configmap] == Apply the `aws-auth`   `ConfigMap` to your cluster The `aws-auth` `ConfigMap` is automatically created and applied to your cluster when you create a managed node group or when you create a node group using `eksctl`. It is initially created to allow nodes to join your cluster, but you also use this `ConfigMap` to add role-based access control (RBAC) access to IAM principals. If you've launched self-managed nodes and haven't applied the `aws-auth` `ConfigMap` to your cluster, you can do so with the following procedure. @@ -369,4 +366,4 @@ NOTE: If you receive any authorization or resource type errors, see < +include::../../attributes.txt[] [.topic] -[[authenticate-oidc-identity-provider,authenticate-oidc-identity-provider.title]] -= Grant users access to [.noloc]`Kubernetes` with an external [.noloc]`OIDC` provider -:info_doctype: section - -include::../../attributes.txt[] +[#authenticate-oidc-identity-provider] += Grant users access to Kubernetes with an external OIDC provider +:info_titleabbrev: Link OIDC provider [abstract] -- Learn how to authenticate users for your Amazon EKS cluster using OpenID Connect (OIDC) identity providers to manage access and permissions with roles, bindings, and RBAC authorization. -- -Amazon EKS supports using [.noloc]`OpenID Connect` ([.noloc]`OIDC`) identity providers as a method to authenticate users to your cluster. [.noloc]`OIDC` identity providers can be used with, or as an alternative to {aws} Identity and Access Management (IAM). For more information about using IAM, see <>. After configuring authentication to your cluster, you can create [.noloc]`Kubernetes` `roles` and `clusterroles` to assign permissions to the roles, and then bind the roles to the identities using [.noloc]`Kubernetes` `rolebindings` and `clusterrolebindings`. For more information, see https://kubernetes.io/docs/reference/access-authn-authz/rbac/[Using RBAC Authorization] in the [.noloc]`Kubernetes` documentation. +Amazon EKS supports using OpenID Connect (OIDC) identity providers as a method to authenticate users to your cluster. OIDC identity providers can be used with, or as an alternative to {aws} Identity and Access Management (IAM). For more information about using IAM, see <>. After configuring authentication to your cluster, you can create Kubernetes `roles` and `clusterroles` to assign permissions to the roles, and then bind the roles to the identities using Kubernetes `rolebindings` and `clusterrolebindings`. For more information, see https://kubernetes.io/docs/reference/access-authn-authz/rbac/[Using RBAC Authorization] in the Kubernetes documentation. -* You can associate one [.noloc]`OIDC` identity provider to your cluster. -* [.noloc]`Kubernetes` doesn't provide an [.noloc]`OIDC` identity provider. You can use an existing public [.noloc]`OIDC` identity provider, or you can run your own identity provider. For a list of certified providers, see https://openid.net/certification/[OpenID Certification] on the OpenID site. -* The issuer URL of the [.noloc]`OIDC` identity provider must be publicly accessible, so that Amazon EKS can discover the signing keys. Amazon EKS doesn't support [.noloc]`OIDC` identity providers with self-signed certificates. +* You can associate one OIDC identity provider to your cluster. +* Kubernetes doesn't provide an OIDC identity provider. You can use an existing public OIDC identity provider, or you can run your own identity provider. For a list of certified providers, see https://openid.net/certification/[OpenID Certification] on the OpenID site. +* The issuer URL of the OIDC identity provider must be publicly accessible, so that Amazon EKS can discover the signing keys. Amazon EKS doesn't support OIDC identity providers with self-signed certificates. * You can't disable IAM authentication to your cluster, because it's still required for joining nodes to a cluster. -* An Amazon EKS cluster must still be created by an {aws} link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"], rather than an [.noloc]`OIDC` identity provider user. This is because the cluster creator interacts with the Amazon EKS APIs, rather than the [.noloc]`Kubernetes` APIs. -* [.noloc]`OIDC` identity provider-authenticated users are listed in the cluster's audit log if CloudWatch logs are turned on for the control plane. For more information, see <>. -* You can't sign in to the {aws-management-console} with an account from an [.noloc]`OIDC` provider. You can only <> by signing into the {aws-management-console} with an {aws} Identity and Access Management account. +* An Amazon EKS cluster must still be created by an {aws} link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"], rather than an OIDC identity provider user. This is because the cluster creator interacts with the Amazon EKS APIs, rather than the Kubernetes APIs. +* OIDC identity provider-authenticated users are listed in the cluster's audit log if CloudWatch logs are turned on for the control plane. For more information, see <>. +* You can't sign in to the {aws-management-console} with an account from an OIDC provider. You can only <> by signing into the {aws-management-console} with an {aws} Identity and Access Management account. -[[associate-oidc-identity-provider,associate-oidc-identity-provider.title]] -== Associate an [.noloc]`OIDC` identity provider +[#associate-oidc-identity-provider] +== Associate an OIDC identity provider -Before you can associate an [.noloc]`OIDC` identity provider with your cluster, you need the following information from your provider: +Before you can associate an OIDC identity provider with your cluster, you need the following information from your provider: @@ -42,10 +40,10 @@ The ID for the client application that makes authentication requests to the OIDC You can associate an identity provider using `eksctl` or the {aws-management-console}. -[[identity-associate-eksctl,identity-associate-eksctl.title]] +[#identity-associate-eksctl] === Associate an identity provider using eksctl -. Create a file named [.replaceable]`associate-identity-provider.yaml` with the following contents. Replace the [.replaceable]`example values` with your own. The values in the `identityProviders` section are obtained from your [.noloc]`OIDC` identity provider. Values are only required for the `name`, `type`, `issuerUrl`, and `clientId` settings under `identityProviders`. +. Create a file named `associate-identity-provider.yaml` with the following contents. Replace the example values with your own. The values in the `identityProviders` section are obtained from your OIDC identity provider. Values are only required for the `name`, `type`, `issuerUrl`, and `clientId` settings under `identityProviders`. + [source,yaml,subs="verbatim,attributes"] ---- @@ -79,115 +77,41 @@ IMPORTANT: Don't specify `system:`, or any portion of that string, for `groupsPr ---- eksctl associate identityprovider -f associate-identity-provider.yaml ---- -. To use `kubectl` to work with your cluster and [.noloc]`OIDC` identity provider, see https://kubernetes.io/docs/reference/access-authn-authz/authentication/#using-kubectl[Using kubectl] in the [.noloc]`Kubernetes` documentation. +. To use `kubectl` to work with your cluster and OIDC identity provider, see https://kubernetes.io/docs/reference/access-authn-authz/authentication/#using-kubectl[Using kubectl] in the Kubernetes documentation. -[[identity-associate-console,identity-associate-console.title]] +[#identity-associate-console] === Associate an identity provider using the {aws} Console . Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. . Select your cluster, and then select the *Access* tab. -. In the *[.noloc]`OIDC` Identity Providers* section, select** Associate Identity Provider**. -. On the *Associate [.noloc]`OIDC` Identity Provider* page, enter or select the following options, and then select *Associate*. +. In the *OIDC Identity Providers* section, select* Associate Identity Provider*. +. On the *Associate OIDC Identity Provider* page, enter or select the following options, and then select *Associate*. + ** For *Name*, enter a unique name for the provider. ** For *Issuer URL*, enter the URL for your provider. This URL must be accessible over the internet. -** For *Client ID*, enter the [.noloc]`OIDC` identity provider's client ID (also known as *audience*). +** For *Client ID*, enter the OIDC identity provider's client ID (also known as *audience*). ** For *Username claim*, enter the claim to use as the username. ** For *Groups claim*, enter the claim to use as the user's group. ** (Optional) Select *Advanced options*, enter or select the following information. + *** *Username prefix* – Enter a prefix to prepend to username claims. The prefix is prepended to username claims to prevent clashes with existing names. If you do not provide a value, and the username is a value other than `email`, the prefix defaults to the value for *Issuer URL*. You can use the value`` -`` to disable all prefixing. Don't specify `system:` or any portion of that string. *** *Groups prefix* – Enter a prefix to prepend to groups claims. The prefix is prepended to group claims to prevent clashes with existing names (such as`` system: groups``). For example, the value `oidc:` creates group names like `oidc:engineering` and `oidc:infra`. Don't specify `system:` or any portion of that string.. -*** *Required claims* – Select *Add claim* and enter one or more key value pairs that describe required claims in the client ID token. The pairs describe required claims in the ID Token. If set, each claim is verified to be present in the ID token with a matching value. -.. To use `kubectl` to work with your cluster and [.noloc]`OIDC` identity provider, see https://kubernetes.io/docs/reference/access-authn-authz/authentication/#using-kubectl[Using kubectl] in the [.noloc]`Kubernetes` documentation. +*** *Required claims* – Select *Add claim* and enter one or more key value pairs that describe required claims in the client ID token. The pairs describe required claims in the ID Token. If set, each claim is verified to be present in the ID token with a matching value. +.. To use `kubectl` to work with your cluster and OIDC identity provider, see https://kubernetes.io/docs/reference/access-authn-authz/authentication/#using-kubectl[Using kubectl] in the Kubernetes documentation. -[[oidc-identity-provider-iam-policy,oidc-identity-provider-iam-policy.title]] +[#oidc-identity-provider-iam-policy] == Example IAM policy -If you want to prevent an [.noloc]`OIDC` identity provider from being associated with a cluster, create and associate the following IAM policy to the IAM accounts of your Amazon EKS administrators. For more information, see link:IAM/latest/UserGuide/access_policies_create.html[Creating IAM policies,type="documentation"] and link:IAM/latest/UserGuide/access_policies_manage-attach-detach.html#add-policies-console[Adding IAM identity permissions,type="documentation"] in the _IAM User Guide_ and link:service-authorization/latest/reference/list_amazonelasticcontainerserviceforkubernetes.html[Actions, resources, and condition keys for Amazon Elastic Kubernetes Service,type="documentation"] in the Service Authorization Reference. +If you want to prevent an OIDC identity provider from being associated with a cluster, create and associate the following IAM policy to the IAM accounts of your Amazon EKS administrators. For more information, see link:IAM/latest/UserGuide/access_policies_create.html[Creating IAM policies,type="documentation"] and link:IAM/latest/UserGuide/access_policies_manage-attach-detach.html#add-policies-console[Adding IAM identity permissions,type="documentation"] in the _IAM User Guide_ and link:service-authorization/latest/reference/list_amazonelasticcontainerserviceforkubernetes.html[Actions, resources, and condition keys for Amazon Elastic Kubernetes Service,type="documentation"] in the Service Authorization Reference. [source,json,subs="verbatim,attributes"] ---- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "denyOIDC", - "Effect": "Deny", - "Action": [ - "eks:AssociateIdentityProviderConfig" - ], - "Resource": "{arn-aws}eks:us-west-2.amazonaws.com:111122223333:cluster/*" - - }, - { - "Sid": "eksAdmin", - "Effect": "Allow", - "Action": [ - "eks:*" - ], - "Resource": "*" - } - ] -} +include::iam_snippet:80073812-1626-4a47-bd2f-69242ecea039[] ---- -The following example policy allows [.noloc]`OIDC` identity provider association if the `clientID` is `kubernetes` and the `issuerUrl` is `https://cognito-idp.us-west-2amazonaws.com/*`. +The following example policy allows OIDC identity provider association if the `clientID` is `kubernetes` and the `issuerUrl` is `https://cognito-idp.us-west-2amazonaws.com/*`. [source,json,subs="verbatim,attributes"] ---- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "AllowCognitoOnly", - "Effect": "Deny", - "Action": "eks:AssociateIdentityProviderConfig", - "Resource": "{arn-aws}eks:us-west-2:111122223333:cluster/my-instance", - "Condition": { - "StringNotLikeIfExists": { - "eks:issuerUrl": "https://cognito-idp.us-west-2.amazonaws.com/*" - } - } - }, - { - "Sid": "DenyOtherClients", - "Effect": "Deny", - "Action": "eks:AssociateIdentityProviderConfig", - "Resource": "{arn-aws}eks:us-west-2:111122223333:cluster/my-instance", - "Condition": { - "StringNotEquals": { - "eks:clientId": "kubernetes" - } - } - }, - { - "Sid": "AllowOthers", - "Effect": "Allow", - "Action": "eks:*", - "Resource": "*" - } - ] -} +include::iam_snippet:232b8739-9c0c-47c4-883e-63b3178048bc[] ---- - - -[[partner-validated-identity-providers,partner-validated-identity-providers.title]] -== Partner validated [.noloc]`OIDC` identity providers - -Amazon EKS maintains relationships with a network of partners that offer support for compatible [.noloc]`OIDC` identity providers. Refer to the following partners' documentation for details on how to integrate the identity provider with Amazon EKS. - -[cols="1,1,1", options="header"] -|=== -|Partner -|Product -|Documentation - - -|PingIdentity -|https://docs.pingidentity.com/r/en-us/pingoneforenterprise/p14e_landing[PingOne for Enterprise] -|https://docs.pingidentity.com/r/en-us/solution-guides/htg_config_oidc_authn_aws_eks_custers[Installation instructions] -|=== - -Amazon EKS aims to give you a wide selection of options to cover all use cases. If you develop a commercially supported [.noloc]`OIDC` compatible identity provider that is not listed here, then contact our partner team at link:mailto:aws-container-partners@amazon.com[aws-container-partners@amazon. -com] for more information. \ No newline at end of file diff --git a/latest/ug/manage-access/k8s-access/create-k8s-group-entry.adoc b/latest/ug/manage-access/k8s-access/create-k8s-group-entry.adoc new file mode 100644 index 000000000..f526b91da --- /dev/null +++ b/latest/ug/manage-access/k8s-access/create-k8s-group-entry.adoc @@ -0,0 +1,64 @@ +include::../../attributes.txt[] + +[.topic] +[#create-k8s-group-access-entry] += Create an access entry using Kubernetes groups with the {aws} CLI +:info_titleabbrev: Tutorial: Create with Kubernetes groups + +Create Amazon EKS access entries that use Kubernetes groups for authorization and require manual RBAC configuration. + +NOTE: For most use cases, we recommend using EKS Access Policies instead of the Kubernetes groups approach described on this page. EKS Access Policies provide a simpler, more {aws}-integrated way to manage access without requiring manual RBAC configuration. Use the Kubernetes groups approach only when you need more granular control than what EKS Access Policies offer. + +== Overview + +Access entries define how IAM identities (users and roles) access your Kubernetes clusters. The Kubernetes groups approach grants IAM users or roles permission to access your EKS cluster through standard Kubernetes RBAC groups. This method requires creating and managing Kubernetes RBAC resources (Roles, RoleBindings, ClusterRoles, and ClusterRoleBindings) and is recommended when you need highly customized permission sets, complex authorization requirements, or want to maintain consistent access control patterns across hybrid Kubernetes environments. + +This topic does not cover creating access entries for IAM identities used for Amazon EC2 instances to join EKS clusters. + +== Prerequisites + +* The _authentication mode_ of your cluster must be configured to enable _access entries_. For more information, see <>. +* Install and configure the {aws} CLI, as described in link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] in the {aws} Command Line Interface User Guide. +* Familiarity with Kubernetes RBAC is recommended. For more information, see https://kubernetes.io/docs/reference/access-authn-authz/rbac/[Using RBAC Authorization] in the Kubernetes documentation. + +[#k8s-group-s1] +== Step 1: Define access entry + +. Find the ARN of the IAM identity, such as a user or role, that you want to grant permissions to. +** Each IAM identity can have only one EKS access entry. +. Determine which Kubernetes groups you want to associate with this IAM identity. +** You will need to create or use existing Kubernetes `Role`/`ClusterRole` and `RoleBinding`/`ClusterRoleBinding` resources that reference these groups. +. Determine if the auto-generated username is appropriate for the access entry, or if you need to manually specify a username. +** {aws} auto-generates this value based on the IAM identity. You can set a custom username. This is visible in Kubernetes logs. +** For more information, see <>. + +[#k8s-group-s2] +== Step 2: Create access entry with Kubernetes groups + +After planning the access entry, use the {aws} CLI to create it with the appropriate Kubernetes groups. + +[source,bash,subs="verbatim,attributes"] +---- +aws eks create-access-entry --cluster-name --principal-arn --type STANDARD --kubernetes-groups +---- + +Replace: + +* `` with your EKS cluster name +* `` with the ARN of the IAM user or role +* `` with a comma-separated list of Kubernetes groups (e.g., "system:developers,system:readers") + +link:cli/latest/reference/eks/create-access-entry.html[View the CLI reference for all configuration options,type="documentation"]. + +== Step 3: Configure Kubernetes RBAC + +For the IAM principal to have access to Kubernetes objects on your cluster, you must create and manage Kubernetes role-based access control (RBAC) objects: + +. Create Kubernetes `Role` or `ClusterRole` objects that define the permissions. +. Create Kubernetes `RoleBinding` or `ClusterRoleBinding` objects on your cluster that specify the group name as a `subject` for `kind: Group`. + +For detailed information about configuring groups and permissions in Kubernetes, see https://kubernetes.io/docs/reference/access-authn-authz/rbac/[Using RBAC Authorization] in the Kubernetes documentation. + +== Next steps + +* xref:create-kubeconfig[Create a kubeconfig so you can use kubectl with an IAM identity] diff --git a/latest/ug/manage-access/k8s-access/create-standard-entry-policy.adoc b/latest/ug/manage-access/k8s-access/create-standard-entry-policy.adoc new file mode 100644 index 000000000..068422622 --- /dev/null +++ b/latest/ug/manage-access/k8s-access/create-standard-entry-policy.adoc @@ -0,0 +1,78 @@ +include::../../attributes.txt[] + +[.topic] +[#create-standard-access-entry-policy] += Create an access entry for an IAM role or user using an access policy and the {aws} CLI +:info_titleabbrev: Tutorial: Create with access policy + +Create Amazon EKS access entries that use {aws}-managed EKS access policies to grant IAM identities standardized permissions for accessing and managing Kubernetes clusters. + +== Overview + +Access entries in Amazon EKS define how IAM identities (users and roles) can access and interact with your Kubernetes clusters. By creating access entries with EKS access policies, you can: + +* Grant specific IAM users or roles permission to access your EKS cluster +* Control permissions using {aws}-managed EKS access policies that provide standardized, predefined permission sets +* Scope permissions to specific namespaces or cluster-wide +* Simplify access management without modifying the `aws-auth` ConfigMap or creating Kubernetes RBAC resources +* Use {aws}-integrated approach to Kubernetes access control that covers common use cases while maintaining security best practices + +This approach is recommended for most use cases because it provides {aws}-managed, standardized permissions without requiring manual Kubernetes RBAC configuration. EKS access policies eliminate the need to manually configure Kubernetes RBAC resources and offer predefined permission sets that cover common use cases. + +== Prerequisites + +* The _authentication mode_ of your cluster must be configured to enable _access entries_. For more information, see <>. +* Install and configure the {aws} CLI, as described in link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] in the {aws} Command Line Interface User Guide. + + +[#ap1-s1] +== Step 1: Define access entry + +. Find the ARN of IAM identity, such as a user or role, that you want to grant permissions to. +** Each IAM identity can have only one EKS access entry. +. Determine if you want the Amazon EKS access policy permissions to apply to only a specific Kubernetes namespace, or across the entire cluster. +** If you want to limit the permissions to a specific namespace, make note of the namespace name. +. Select the EKS access policy you want for the IAM identity. This policy gives in-cluster permissions. Note the ARN of the policy. +** For a list of policies, see xref:access-policy-permissions[available access policies]. +. Determine if the auto-generated username is appropriate for the access entry, or if you need to manually specify a username. +** {aws} auto-generates this value based on the IAM identity. You can set a custom username. This is visible in Kubernetes logs. +** For more information, see <>. + +[#ap1-s2] +== Step 2: Create access entry + +After planning the access entry, use the {aws} CLI to create it. + +The following example covers most use cases. link:cli/latest/reference/eks/create-access-entry.html[View the CLI reference for all configuration options,type="documentation"]. + +You will attach the access policy in the next step. + +[source,bash,subs="verbatim,attributes"] +---- +aws eks create-access-entry --cluster-name --principal-arn --type STANDARD +---- + +== Step 3: Associate access policy + +The command differs based on whether you want the policy to be limited to a specified Kubernetes namespace. + +You need the ARN of the access policy. Review the xref:access-policy-permissions[available access policies]. + +=== Create policy without namespace scope + +[source,bash,subs="verbatim,attributes"] +---- +aws eks associate-access-policy --cluster-name --principal-arn --policy-arn +---- + +=== Create with namespace scope + +[source,bash,subs="verbatim,attributes"] +---- +aws eks associate-access-policy --cluster-name --principal-arn \ + --access-scope type=namespace,namespaces=my-namespace1,my-namespace2 --policy-arn +---- + +== Next steps + +* xref:create-kubeconfig[Create a kubeconfig so you can use kubectl with an IAM identity] diff --git a/latest/ug/manage-access/k8s-access/creating-access-entries.adoc b/latest/ug/manage-access/k8s-access/creating-access-entries.adoc new file mode 100644 index 000000000..3f2e6f9ca --- /dev/null +++ b/latest/ug/manage-access/k8s-access/creating-access-entries.adoc @@ -0,0 +1,128 @@ +include::../../attributes.txt[] + +[.topic] +[#creating-access-entries] += Create access entries +:info_titleabbrev: Create access entries + +Before creating access entries, consider the following: + +* A properly set authentication mode. See <>. +* An _access entry_ includes the Amazon Resource Name (ARN) of one, and only one, existing IAM principal. An IAM principal can't be included in more than one access entry. Additional considerations for the ARN that you specify: ++ +** IAM best practices recommend accessing your cluster using IAM _roles_ that have short-term credentials, rather than IAM _users_ that have long-term credentials. For more information, see link:IAM/latest/UserGuide/best-practices.html#bp-users-federation-idp[Require human users to use federation with an identity provider to access {aws} using temporary credentials,type="documentation"] in the _IAM User Guide_. +** If the ARN is for an IAM role, it _can_ include a path. ARNs in `aws-auth` `ConfigMap` entries, _can't_ include a path. For example, your ARN can be `{arn-aws}iam::<111122223333>:role/` or `{arn-aws}iam::<111122223333>:role/`. +** If the type of the access entry is anything other than `STANDARD` (see next consideration about types), the ARN must be in the same {aws} account that your cluster is in. If the type is `STANDARD`, the ARN can be in the same, or different, {aws} account than the account that your cluster is in. +** You can't change the IAM principal after the access entry is created. +** If you ever delete the IAM principal with this ARN, the access entry isn't automatically deleted. We recommend that you delete the access entry with an ARN for an IAM principal that you delete. If you don't delete the access entry and ever recreate the IAM principal, even if it has the same ARN, the access entry won't work. This is because even though the ARN is the same for the recreated IAM principal, the `roleID` or `userID` (you can see this with the `aws sts get-caller-identity` {aws} CLI command) is different for the recreated IAM principal than it was for the original IAM principal. Even though you don't see the IAM principal's `roleID` or `userID` for an access entry, Amazon EKS stores it with the access entry. +* Each access entry has a _type_. The type of the access entry depends on the type of resource it is associated with, and does not define the permissions. If you don't specify a type, Amazon EKS automatically sets the type to `STANDARD` +** `EC2_LINUX` - For an IAM role used with Linux or Bottlerocket self-managed nodes +** `EC2_WINDOWS` - For an IAM role used with Windows self-managed nodes +** `FARGATE_LINUX` - For an IAM role used with {aws} Fargate (Fargate) +** `HYBRID_LINUX` - For an IAM role used with hybrid nodes +** `STANDARD` - Default type if none specified +** `EC2` - For EKS Auto Mode custom node classes. For more information, see <>. +** You can't change the type after the access entry is created. +* It's unnecessary to create an access entry for an IAM role that's used for a managed node group or a Fargate profile. EKS will create access entries (if enabled), or update the auth config map (if access entries are unavailable) +* If the type of the access entry is `STANDARD`, you can specify a _username_ for the access entry. If you don't specify a value for username, Amazon EKS sets one of the following values for you, depending on the type of the access entry and whether the IAM principal that you specified is an IAM role or IAM user. Unless you have a specific reason for specifying your own username, we recommend that don't specify one and let Amazon EKS auto-generate it for you. If you specify your own username: ++ +** It can't start with `system:`, `eks:`, `aws:`, `amazon:`, or `iam:`. +** If the username is for an IAM role, we recommend that you add `{{SessionName}}` or `{{SessionNameRaw}}` to the end of your username. If you add either `{{SessionName}}` or `{{SessionNameRaw}}` to your username, the username must include a colon _before_ {{SessionName}}. When this role is assumed, the name of the {aws} STS session name that is specified when assuming the role is automatically passed to the cluster and will appear in CloudTrail logs. For example, you can't have a username of `john{{SessionName}}`. The username would have to be `:john{{SessionName}}` or `jo:hn{{SessionName}}`. The colon only has to be before `{{SessionName}}`. The username generated by Amazon EKS in the following table includes an ARN. Since an ARN includes colons, it meets this requirement. The colon isn't required if you don't include `{{SessionName}}` in your username. Note that in `{{SessionName}}` the special character "@" is replaced with "-" in the session name. `{{SessionNameRaw}}` keeps all special characters in the session name. ++ +[%header,cols="3"] +|=== +|IAM principal type +|Type +|Username value that Amazon EKS automatically sets + + +|User +|`STANDARD` +|The ARN of the user. Example: `{arn-aws}iam::<111122223333>:user/` + +|Role +|`STANDARD` +|The STS ARN of the role when it's assumed. Amazon EKS appends `{{SessionName}}` to the role. + +Example: `{arn-aws}sts::<111122223333>:assumed-role//{{SessionName}}` + +If the ARN of the role that you specified contained a path, Amazon EKS removes it in the generated username. + +|Role +|`EC2_LINUX` or `EC2_Windows` +|`system:node:{{EC2PrivateDNSName}}` + +|Role +|`FARGATE_LINUX` +|`system:node:{{SessionName}}` + +|Role +|`HYBRID_LINUX` +|`system:node:{{SessionName}}` +|=== ++ +You can change the username after the access entry is created. +* If an access entry's type is `STANDARD`, and you want to use Kubernetes RBAC authorization, you can add one or more _group names_ to the access entry. After you create an access entry you can add and remove group names. For the IAM principal to have access to Kubernetes objects on your cluster, you must create and manage Kubernetes role-based authorization (RBAC) objects. Create Kubernetes `RoleBinding` or `ClusterRoleBinding` objects on your cluster that specify the group name as a `subject` for `kind: Group`. Kubernetes authorizes the IAM principal access to any cluster objects that you've specified in a Kubernetes `Role` or `ClusterRole` object that you've also specified in your binding's `roleRef`. If you specify group names, we recommend that you're familiar with the Kubernetes role-based authorization (RBAC) objects. For more information, see https://kubernetes.io/docs/reference/access-authn-authz/rbac/[Using RBAC Authorization] in the Kubernetes documentation. ++ +IMPORTANT: Amazon EKS doesn't confirm that any Kubernetes RBAC objects that exist on your cluster include any of the group names that you specify. For example, if you create an access entry for group that currently doesn't exist, EKS will create the group instead of returning an error. ++ +Instead of, or in addition to, Kubernetes authorizing the IAM principal access to Kubernetes objects on your cluster, you can associate Amazon EKS _access policies_ to an access entry. Amazon EKS authorizes IAM principals to access Kubernetes objects on your cluster with the permissions in the access policy. You can scope an access policy's permissions to Kubernetes namespaces that you specify. Use of access policies don't require you to manage Kubernetes RBAC objects. For more information, see <>. +* If you create an access entry with type `EC2_LINUX` or `EC2_Windows`, the IAM principal creating the access entry must have the `iam:PassRole` permission. For more information, see link:IAM/latest/UserGuide/id_roles_use_passrole.html[Granting a user permissions to pass a role to an {aws} service,type="documentation"] in the _IAM User Guide_. +* Similar to standard link:IAM/latest/UserGuide/troubleshoot_general.html#troubleshoot_general_eventual-consistency[IAM behavior,type="documentation"], access entry creation and updates are eventually consistent, and may take several seconds to be effective after the initial API call returns successfully. You must design your applications to account for these potential delays. We recommend that you don't include access entry creates or updates in the critical, high- availability code paths of your application. Instead, make changes in a separate initialization or setup routine that you run less frequently. Also, be sure to verify that the changes have been propagated before production workflows depend on them. +* Access entries do not support link:IAM/latest/UserGuide/using-service-linked-roles.html[service linked roles,type="documentation"]. You cannot create access entries where the principal ARN is a service linked role. You can identify service linked roles by their ARN, which is in the format `{arn-aws}iam::*:role/aws-service-role/*`. + +You can create an access entry using the {aws-management-console} or the {aws} CLI. + + +[#access-create-console] +== {aws-management-console} +. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. +. Choose the name of the cluster that you want to create an access entry in. +. Choose the *Access* tab. +. Choose *Create access entry*. +. For *IAM principal*, select an existing IAM role or user. IAM best practices recommend accessing your cluster using IAM _roles_ that have short-term credentials, rather than IAM _users_ that have long-term credentials. For more information, see link:IAM/latest/UserGuide/best-practices.html#bp-users-federation-idp[Require human users to use federation with an identity provider to access {aws} using temporary credentials,type="documentation"] in the _IAM User Guide_. +. For *Type*, if the access entry is for the node role used for self-managed Amazon EC2 nodes, select *EC2 Linux* or *EC2 Windows*. Otherwise, accept the default (*Standard*). +. If the *Type* you chose is *Standard* and you want to specify a *Username*, enter the username. +. If the *Type* you chose is *Standard* and you want to use Kubernetes RBAC authorization for the IAM principal, specify one or more names for *Groups*. If you don't specify any group names and want to use Amazon EKS authorization, you can associate an access policy in a later step, or after the access entry is created. +. (Optional) For *Tags*, assign labels to the access entry. For example, to make it easier to find all resources with the same tag. +. Choose *Next*. +. On the *Add access policy* page, if the type you chose was *Standard* and you want Amazon EKS to authorize the IAM principal to have permissions to the Kubernetes objects on your cluster, complete the following steps. Otherwise, choose *Next*. ++ +.. For *Policy name*, choose an access policy. You can't view the permissions of the access policies, but they include similar permissions to those in the Kubernetes user-facing `ClusterRole` objects. For more information, see https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles[User-facing roles] in the Kubernetes documentation. +.. Choose one of the following options: ++ +*** *Cluster* – Choose this option if you want Amazon EKS to authorize the IAM principal to have the permissions in the access policy for all Kubernetes objects on your cluster. +*** *Kubernetes namespace* – Choose this option if you want Amazon EKS to authorize the IAM principal to have the permissions in the access policy for all Kubernetes objects in a specific Kubernetes namespace on your cluster. For *Namespace*, enter the name of the Kubernetes namespace on your cluster. If you want to add additional namespaces, choose *Add new namespace* and enter the namespace name. +.. If you want to add additional policies, choose *Add policy*. You can scope each policy differently, but you can add each policy only once. +.. Choose *Next*. +. Review the configuration for your access entry. If anything looks incorrect, choose *Previous* to go back through the steps and correct the error. If the configuration is correct, choose *Create*. + +[#access-create-cli] +== {aws} CLI + +. Install the {aws} CLI, as described in link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] in the {aws} Command Line Interface User Guide. +. To create an access entry +You can use any of the following examples to create access entries: ++ +** Create an access entry for a self-managed Amazon EC2 Linux node group. Replace [.replaceable]`my-cluster` with the name of your cluster, [.replaceable]`111122223333` with your {aws} account ID, and [.replaceable]`EKS-my-cluster-self-managed-ng-1` with the name of your <>. If your node group is a Windows node group, then replace [.replaceable]`EC2_LINUX` with `EC2_Windows`. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks create-access-entry --cluster-name my-cluster --principal-arn {arn-aws}iam::111122223333:role/EKS-my-cluster-self-managed-ng-1 --type EC2_LINUX +---- ++ +You can't use the `--kubernetes-groups` option when you specify a type other than `STANDARD`. You can't associate an access policy to this access entry, because its type is a value other than `STANDARD`. +** Create an access entry that allows an IAM role that's not used for an Amazon EC2 self-managed node group, that you want Kubernetes to authorize access to your cluster with. Replace [.replaceable]`my-cluster` with the name of your cluster, [.replaceable]`111122223333` with your {aws} account ID, and [.replaceable]`my-role` with the name of your IAM role. Replace [.replaceable]`Viewers` with the name of a group that you've specified in a Kubernetes `RoleBinding` or `ClusterRoleBinding` object on your cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks create-access-entry --cluster-name my-cluster --principal-arn {arn-aws}iam::111122223333:role/my-role --type STANDARD --user Viewers --kubernetes-groups Viewers +---- +** Create an access entry that allows an IAM user to authenticate to your cluster. This example is provided because this is possible, though IAM best practices recommend accessing your cluster using IAM _roles_ that have short-term credentials, rather than IAM _users_ that have long-term credentials. For more information, see link:IAM/latest/UserGuide/best-practices.html#bp-users-federation-idp[Require human users to use federation with an identity provider to access {aws} using temporary credentials,type="documentation"] in the _IAM User Guide_. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks create-access-entry --cluster-name my-cluster --principal-arn {arn-aws}iam::111122223333:user/my-user --type STANDARD --username my-user +---- ++ +If you want this user to have more access to your cluster than the permissions in the Kubernetes API discovery roles, then you need to associate an access policy to the access entry, since the `--kubernetes-groups` option isn't used. For more information, see <> and https://kubernetes.io/docs/reference/access-authn-authz/rbac/#discovery-roles[API discovery roles] in the Kubernetes documentation. \ No newline at end of file diff --git a/latest/ug/manage-access/k8s-access/deleting-access-entries.adoc b/latest/ug/manage-access/k8s-access/deleting-access-entries.adoc new file mode 100644 index 000000000..4cb223b8b --- /dev/null +++ b/latest/ug/manage-access/k8s-access/deleting-access-entries.adoc @@ -0,0 +1,30 @@ +include::../../attributes.txt[] + +[.topic] +[#deleting-access-entries] += Delete access entries + +If you discover that you deleted an access entry in error, you can always recreate it. If the access entry that you're deleting is associated to any access policies, the associations are automatically deleted. You don't have to disassociate access policies from an access entry before deleting the access entry. + +You can delete an access entry using the {aws-management-console} or the {aws} CLI. + + +[#access-delete-console] +== {aws-management-console} +. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. +. Choose the name of the cluster that you want to delete an access entry from. +. Choose the *Access* tab. +. In the *Access entries* list, choose the access entry that you want to delete. +. Choose Delete. +. In the confirmation dialog box, choose *Delete*. + +[#access-delete-cli] +== {aws} CLI +. Install the {aws} CLI, as described in link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] in the {aws} Command Line Interface User Guide. +. To delete an access entry +Replace [.replaceable]`my-cluster` with the name of your cluster, [.replaceable]`111122223333` with your {aws} account ID, and [.replaceable]`my-role` with the name of the IAM role that you no longer want to have access to your cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks delete-access-entry --cluster-name my-cluster --principal-arn {arn-aws}iam::111122223333:role/my-role +---- \ No newline at end of file diff --git a/latest/ug/manage-access/k8s-access/disassociate-oidc-identity-provider.adoc b/latest/ug/manage-access/k8s-access/disassociate-oidc-identity-provider.adoc index c5ccae3ad..dc407580d 100644 --- a/latest/ug/manage-access/k8s-access/disassociate-oidc-identity-provider.adoc +++ b/latest/ug/manage-access/k8s-access/disassociate-oidc-identity-provider.adoc @@ -1,13 +1,11 @@ -//!!NODE_ROOT
+include::../../attributes.txt[] [.topic] -[[disassociate-oidc-identity-provider,disassociate-oidc-identity-provider.title]] -= Disassociate an [.noloc]`OIDC` identity provider from your cluster -:info_doctype: section - -include::../../attributes.txt[] +[#disassociate-oidc-identity-provider] += Disassociate an OIDC identity provider from your cluster +:info_titleabbrev: Unlink OIDC provider -If you disassociate an [.noloc]`OIDC` identity provider from your cluster, users included in the provider can no longer access the cluster. However, you can still access the cluster with link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principals,type="documentation"]. +If you disassociate an OIDC identity provider from your cluster, users included in the provider can no longer access the cluster. However, you can still access the cluster with link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principals,type="documentation"]. . Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. -. In the *[.noloc]`OIDC` Identity Providers* section, select *Disassociate*, enter the identity provider name, and then select `Disassociate`. \ No newline at end of file +. In the *OIDC Identity Providers* section, select *Disassociate*, enter the identity provider name, and then select `Disassociate`. \ No newline at end of file diff --git a/latest/ug/manage-access/k8s-access/grant-k8s-access.adoc b/latest/ug/manage-access/k8s-access/grant-k8s-access.adoc index 427812745..78d058eca 100644 --- a/latest/ug/manage-access/k8s-access/grant-k8s-access.adoc +++ b/latest/ug/manage-access/k8s-access/grant-k8s-access.adoc @@ -1,62 +1,56 @@ -//!!NODE_ROOT
- +include::../../attributes.txt[] [.topic] -[[grant-k8s-access,grant-k8s-access.title]] -= Grant [.noloc]`IAM` users and roles access to Kubernetes [.noloc]`APIs` -:info_doctype: section -:info_title: Grant IAM users and roles access to Kubernetes APIs -:info_titleabbrev: Grant access to Kubernetes APIs -:info_abstract: Learn how to grant access to Kubernetes APIs on Amazon EKS clusters using IAM roles, users, or OpenID Connect providers, and manage permissions with access entries or the aws-auth ConfigMap. - -include::../../attributes.txt[] +[#grant-k8s-access] += Grant IAM users and roles access to Kubernetes APIs +:info_titleabbrev: Kubernetes API access [abstract] -- Learn how to grant access to Kubernetes APIs on Amazon EKS clusters using IAM roles, users, or OpenID Connect providers, and manage permissions with access entries or the aws-auth ConfigMap. -- -Your cluster has an [.noloc]`Kubernetes` API endpoint. Kubectl uses this API. You can authenticate to this API using two types of identities: +Your cluster has an Kubernetes API endpoint. Kubectl uses this API. You can authenticate to this API using two types of identities: * *An {aws} Identity and Access Management (IAM) _principal_ (role or user)* - – This type requires authentication to IAM. Users can sign in to {aws} as an link:IAM/latest/UserGuide/introduction.html[IAM,type="documentation"] user or with a link:identity/federation/[federated identity,type="marketing"] by using credentials provided through an identity source. Users can only sign in with a federated identity if your administrator previously set up identity federation using IAM roles. When users access {aws} by using federation, they're indirectly link:IAM/latest/UserGuide/when-to-use-iam.html#security-iam-authentication-iamrole[assuming a role,type="documentation"]. When users use this type of identity, you: + – This type requires authentication to IAM. Users can sign in to {aws} as an link:IAM/latest/UserGuide/introduction.html[IAM,type="documentation"] user or with a link:identity/federation/[federated identity,type="marketing"] by using credentials provided through an identity source. Users can only sign in with a federated identity if your administrator previously set up identity federation using IAM roles. When users access {aws} by using federation, they're indirectly link:IAM/latest/UserGuide/when-to-use-iam.html#security-iam-authentication-iamrole[assuming a role,type="documentation"]. When users use this type of identity, you: + -** Can assign them [.noloc]`Kubernetes` permissions so that they can work with [.noloc]`Kubernetes` objects on your cluster. For more information about how to assign permissions to your IAM principals so that they're able to access [.noloc]`Kubernetes` objects on your cluster, see <>. -** Can assign them IAM permissions so that they can work with your Amazon EKS cluster and its resources using the Amazon EKS API, {aws} CLI, {aws} CloudFormation, {aws-management-console}, or `eksctl`. For more information, see link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html#amazonelastickubernetesservice-actions-as-permissions[Actions defined by Amazon Elastic Kubernetes Service,type="documentation"] in the Service Authorization Reference. +** Can assign them Kubernetes permissions so that they can work with Kubernetes objects on your cluster. For more information about how to assign permissions to your IAM principals so that they're able to access Kubernetes objects on your cluster, see <>. +** Can assign them IAM permissions so that they can work with your Amazon EKS cluster and its resources using the Amazon EKS API, {aws} CLI, {aws} CloudFormation, {aws-management-console}, or `eksctl`. For more information, see link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html#amazonelastickubernetesservice-actions-as-permissions[Actions defined by Amazon Elastic Kubernetes Service,type="documentation"] in the Service Authorization Reference. ** Nodes join your cluster by assuming an IAM role. The ability to access your cluster using IAM principals is provided by the https://github.com/kubernetes-sigs/aws-iam-authenticator#readme[{aws} IAM Authenticator for Kubernetes], which runs on the Amazon EKS control plane. -* *A user in your own [.noloc]`OpenID Connect` ([.noloc]`OIDC`) provider* - – This type requires authentication to your https://openid.net/connect/[OIDC] provider. For more information about setting up your own [.noloc]`OIDC` provider with your Amazon EKS cluster, see <>. When users use this type of identity, you: +* *A user in your own OpenID Connect (OIDC) provider* + – This type requires authentication to your https://openid.net/connect/[OIDC] provider. For more information about setting up your own OIDC provider with your Amazon EKS cluster, see <>. When users use this type of identity, you: + -** Can assign them [.noloc]`Kubernetes` permissions so that they can work with [.noloc]`Kubernetes` objects on your cluster. +** Can assign them Kubernetes permissions so that they can work with Kubernetes objects on your cluster. ** Can't assign them IAM permissions so that they can work with your Amazon EKS cluster and its resources using the Amazon EKS API, {aws} CLI, {aws} CloudFormation, {aws-management-console}, or `eksctl`. You can use both types of identities with your cluster. The IAM authentication method cannot be disabled. The OIDC authentication method is optional. -[[authentication-modes,authentication-modes.title]] +[#authentication-modes] == Associate IAM Identities with Kubernetes Permissions -The https://github.com/kubernetes-sigs/aws-iam-authenticator#readme[{aws} IAM Authenticator for Kubernetes] is installed on your cluster's control plane. It enables link:IAM/latest/UserGuide/introduction.html[{aws} Identity and Access Management,type="documentation"] (IAM) principals (roles and users) that you allow to access [.noloc]`Kubernetes` resources on your cluster. You can allow IAM principals to access [.noloc]`Kubernetes` objects on your cluster using one of the following methods: +The https://github.com/kubernetes-sigs/aws-iam-authenticator#readme[{aws} IAM Authenticator for Kubernetes] is installed on your cluster's control plane. It enables link:IAM/latest/UserGuide/introduction.html[{aws} Identity and Access Management,type="documentation"] (IAM) principals (roles and users) that you allow to access Kubernetes resources on your cluster. You can allow IAM principals to access Kubernetes objects on your cluster using one of the following methods: * *Creating access entries* - – If your cluster is at or later than the platform version listed in the link:eks/latest/userguide/access-entries.html[Prerequisites,type="documentation"] section for your cluster's [.noloc]`Kubernetes` version, we recommend that you use this option. + – If your cluster is at or later than the platform version listed in the <> section for your cluster's Kubernetes version, we recommend that you use this option. + -Use _access entries_ to manage the [.noloc]`Kubernetes` permissions of IAM principals from outside the cluster. You can add and manage access to the cluster by using the EKS API, {aws} Command Line Interface, {aws} SDKs, {aws} CloudFormation, and {aws-management-console}. This means you can manage users with the same tools that you created the cluster with. +Use _access entries_ to manage the Kubernetes permissions of IAM principals from outside the cluster. You can add and manage access to the cluster by using the EKS API, {aws} Command Line Interface, {aws} SDKs, {aws} CloudFormation, and {aws-management-console}. This means you can manage users with the same tools that you created the cluster with. + -To get started, follow link:eks/latest/userguide/setting-up-access-entries.html[Change authentication mode to use access entries,type="documentation"], then link:eks/latest/userguide/migrating-access-entries.html[Migrating existing aws-auth ConfigMap entries to access entries,type="documentation"]. +To get started, follow <>, then <>. * *Adding entries to the `aws-auth` `ConfigMap`* - – If your cluster's platform version is earlier than the version listed in the link:eks/latest/userguide/access-entries.html[Prerequisites,type="documentation"] section, then you must use this option. If your cluster's platform version is at or later than the platform version listed in the link:eks/latest/userguide/access-entries.html[Prerequisites,type="documentation"] section for your cluster's [.noloc]`Kubernetes` version, and you've added entries to the `ConfigMap`, then we recommend that you migrate those entries to access entries. You can't migrate entries that Amazon EKS added to the `ConfigMap` however, such as entries for IAM roles used with managed node groups or Fargate profiles. For more information, see <>. + – If your cluster's platform version is earlier than the version listed in the <> section, then you must use this option. If your cluster's platform version is at or later than the platform version listed in the <> section for your cluster's Kubernetes version, and you've added entries to the `ConfigMap`, then we recommend that you migrate those entries to access entries. You can't migrate entries that Amazon EKS added to the `ConfigMap` however, such as entries for IAM roles used with managed node groups or Fargate profiles. For more information, see <>. + ** If you have to use the `aws-auth` `ConfigMap` option, you can add entries to the `ConfigMap` using the `eksctl create iamidentitymapping` command. For more information, see https://eksctl.io/usage/iam-identity-mappings/[Manage IAM users and roles] in the `eksctl` documentation. -[[set-cam,set-cam.title]] +[#set-cam] == Set Cluster Authentication Mode -Each cluster has an _authentication mode_. The authentication mode determines which methods you can use to allow IAM principals to access [.noloc]`Kubernetes` objects on your cluster. There are three authentication modes. +Each cluster has an _authentication mode_. The authentication mode determines which methods you can use to allow IAM principals to access Kubernetes objects on your cluster. There are three authentication modes. [IMPORTANT] ==== @@ -80,10 +74,9 @@ With this authentication mode, you can use both methods to add IAM principals to *Access entries only*:: With this authentication mode, you can use the EKS API, {aws} Command Line Interface, {aws} SDKs, {aws} CloudFormation, and {aws-management-console} to manage access to the cluster for IAM principals. + -Each access entry has a _type_ and you can use the combination of an _access scope_ to limit the principal to a specific namespace and an _access policy_ to set preconfigured reusable permissions policies. Alternatively, you can use the [.noloc]`STANDARD` type and [.noloc]`Kubernetes` [.noloc]`RBAC` groups to assign custom permissions. - +Each access entry has a _type_ and you can use the combination of an _access scope_ to limit the principal to a specific namespace and an _access policy_ to set preconfigured reusable permissions policies. Alternatively, you can use the STANDARD type and Kubernetes RBAC groups to assign custom permissions. -[cols="1,1", options="header"] +[%header,cols="2"] |=== |Authentication mode |Methods @@ -97,6 +90,7 @@ Each access entry has a _type_ and you can use the combination of an _access sco |EKS API only (`API`) |access entries in the EKS API, {aws} Command Line Interface, {aws} SDKs, {aws} CloudFormation, and {aws-management-console} + |=== [NOTE] diff --git a/latest/ug/clusters/management/images b/latest/ug/manage-access/k8s-access/images similarity index 100% rename from latest/ug/clusters/management/images rename to latest/ug/manage-access/k8s-access/images diff --git a/latest/ug/manage-access/k8s-access/migrating-access-entries.adoc b/latest/ug/manage-access/k8s-access/migrating-access-entries.adoc index 4c91b06f9..63583e6d7 100644 --- a/latest/ug/manage-access/k8s-access/migrating-access-entries.adoc +++ b/latest/ug/manage-access/k8s-access/migrating-access-entries.adoc @@ -1,22 +1,18 @@ - -//!!NODE_ROOT
+include::../../attributes.txt[] [.topic] -[[migrating-access-entries,migrating-access-entries.title]] +[#migrating-access-entries] = Migrating existing `aws-auth ConfigMap` entries to access entries -:info_doctype: section - -include::../../attributes.txt[] +:info_titleabbrev: Migrate to access entries //GDC: problems with xrefs -If you've added entries to the `aws-auth` `ConfigMap` on your cluster, we recommend that you create access entries for the existing entries in your `aws-auth` `ConfigMap`. After creating the access entries, you can remove the entries from your `ConfigMap`. You can't associate link:eks/latest/userguide/access-policies.html[access policies,type="documentation"] to entries in the `aws-auth` `ConfigMap`. If you want to associate access polices to your IAM principals, create access entries. +If you've added entries to the `aws-auth` `ConfigMap` on your cluster, we recommend that you create access entries for the existing entries in your `aws-auth` `ConfigMap`. After creating the access entries, you can remove the entries from your `ConfigMap`. You can't associate <> to entries in the `aws-auth` `ConfigMap`. If you want to associate access polices to your IAM principals, create access entries. [IMPORTANT] ==== - -Don't remove existing `aws-auth` `ConfigMap` entries that were created by Amazon EKS when you added a link:eks/latest/userguide/managed-node-groups.html[managed node group,type="documentation"] or a link:eks/latest/userguide/fargate-profile.html["Fargate profile",type="documentation"] to your cluster. If you remove entries that Amazon EKS created in the `ConfigMap`, your cluster won't function properly. You can however, remove any entries for link:eks/latest/userguide/worker.html["self-managed",type="documentation"] node groups after you've created access entries for them. - +* When a cluster is in `API_AND_CONFIGMAP` authentication mode and there's a mapping for the same IAM role in both the `aws-auth` `ConfigMap` and in access entries, the role will use the access entry's mapping for authentication. Access entries take precedence over `ConfigMap` entries for the same IAM principal. +* Before removing existing `aws-auth` `ConfigMap` entries that were created by Amazon EKS for <> or a <> to your cluster, double check if the correct access entries for those specific resources exist in your Amazon EKS cluster. If you remove entries that Amazon EKS created in the `ConfigMap` without having the equivalent access entries, your cluster won't function properly. ==== @@ -25,8 +21,8 @@ Don't remove existing `aws-auth` `ConfigMap` entries that were created by Amazon * Familiarity with access entries and access policies. For more information, see <> and <>. * An existing cluster with a platform version that is at or later than the versions listed in the Prerequisites of the <> topic. * Version `{eksctl-min-version}` or later of the `eksctl` command line tool installed on your device or {aws} CloudShell. To install or update `eksctl`, see https://eksctl.io/installation[Installation] in the `eksctl` documentation. -* [.noloc]`Kubernetes` permissions to modify the `aws-auth` `ConfigMap` in the `kube-system` namespace. -* An {aws} Identity and Access Management role or user with the following permissions: `CreateAccessEntry` and `ListAccessEntries`. For more information, see link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html#amazonelastickubernetesservice-actions-as-permissions[Actions defined by Amazon Elastic Kubernetes Service,type="documentation"] in the Service Authorization Reference. +* Kubernetes permissions to modify the `aws-auth` `ConfigMap` in the `kube-system` namespace. +* An {aws} Identity and Access Management role or user with the following permissions: `CreateAccessEntry` and `ListAccessEntries`. For more information, see link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html#amazonelastickubernetesservice-actions-as-permissions[Actions defined by Amazon Elastic Kubernetes Service,type="documentation"] in the Service Authorization Reference. == `eksctl` [[migrating_access_entries_eksctl]] @@ -56,4 +52,4 @@ ARN [source,bash,subs="verbatim,attributes"] ---- eksctl delete iamidentitymapping --arn {arn-aws}iam::111122223333:role/EKS-my-cluster-my-namespace-Viewers --cluster my-cluster ----- \ No newline at end of file +---- diff --git a/latest/ug/manage-access/k8s-access/set-custom-username.adoc b/latest/ug/manage-access/k8s-access/set-custom-username.adoc new file mode 100644 index 000000000..e52c8c1c2 --- /dev/null +++ b/latest/ug/manage-access/k8s-access/set-custom-username.adoc @@ -0,0 +1,56 @@ +include::../../attributes.txt[] + +[.topic] +[#set-custom-username] += Set a custom username for EKS access entries +:info_titleabbrev: Set custom username + +When creating access entries for Amazon EKS, you can either use the automatically generated username or specify a custom username. This page explains both options and guides you through setting a custom username. + +== Overview + +The username in an access entry is used to identify the IAM principal in Kubernetes logs and audit trails. By default, Amazon EKS generates a username based on the IAM identity's ARN, but you can specify a custom username if needed. + +== Default username generation + +If you don't specify a value for username, Amazon EKS automatically generates a username based on the IAM Identity: + +* *For IAM Users*: +** EKS sets the Kubernetes username to the ARN of the IAM User +** Example: ++ +[source] +---- +{arn-aws}iam::<111122223333>:user/ +---- + +* *For IAM Roles*: +** EKS sets the Kubernetes username based on the ARN of the IAM Role +** The STS ARN of the role when it's assumed. Amazon EKS appends `{{SessionName}}` to the role. If the ARN of the role that you specified contained a path, Amazon EKS removes it in the generated username. +** Example: ++ +[source] +---- +{arn-aws}sts::<111122223333>:assumed-role//{{SessionName}} +---- + +Unless you have a specific reason for specifying your own username, we recommend that you don't specify one and let Amazon EKS auto-generate it for you. + +== Setting a custom username + +When creating an access entry, you can specify a custom username using the `--username` parameter: + +[source,bash,subs="verbatim,attributes"] +---- +aws eks create-access-entry --cluster-name --principal-arn --type STANDARD --username +---- + +=== Requirements for custom usernames + +If you specify a custom username: + +* The username can't start with `system:`, `eks:`, `aws:`, `amazon:`, or `iam:`. +* If the username is for an IAM role, we recommend that you add `{{SessionName}}` or `{{SessionNameRaw}}` to the end of your username. +** If you add either `{{SessionName}}` or `{{SessionNameRaw}}` to your username, the username must include a colon _before_ {{SessionName}}. + + diff --git a/latest/ug/manage-access/k8s-access/setting-up-access-entries.adoc b/latest/ug/manage-access/k8s-access/setting-up-access-entries.adoc new file mode 100644 index 000000000..4b8a226eb --- /dev/null +++ b/latest/ug/manage-access/k8s-access/setting-up-access-entries.adoc @@ -0,0 +1,50 @@ +include::../../attributes.txt[] + +[.topic] +[#setting-up-access-entries] += Change authentication mode to use access entries +:info_titleabbrev: Authentication mode + +To begin using access entries, you must change the authentication mode of the cluster to either the `API_AND_CONFIG_MAP` or `API` modes. This adds the API for access entries. + +[#access-entries-setup-console] +== {aws} Console + +. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. +. Choose the name of the cluster that you want to create an access entry in. +. Choose the *Access* tab. +. The *Authentication mode* shows the current authentication mode of the cluster. If the mode says EKS API, you can already add access entries and you can skip the remaining steps. +. Choose *Manage access*. +. For *Cluster authentication mode*, select a mode with the EKS API. Note that you can't change the authentication mode back to a mode that removes the EKS API and access entries. +. Choose *Save changes*. Amazon EKS begins to update the cluster, the status of the cluster changes to Updating, and the change is recorded in the *Update history* tab. +. Wait for the status of the cluster to return to Active. When the cluster is Active, you can follow the steps in <> to add access to the cluster for IAM principals. + +[#access-setup-cli] +== {aws} CLI + +. Install the {aws} CLI, as described in link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] in the _{aws} Command Line Interface User Guide_. +. Run the following command. Replace [.replaceable]`my-cluster` with the name of your cluster. If you want to disable the `ConfigMap` method permanently, replace `API_AND_CONFIG_MAP` with `API`. ++ +Amazon EKS begins to update the cluster, the status of the cluster changes to UPDATING, and the change is recorded in the [command]*aws eks list-updates*. ++ +[source,bash] +---- +aws eks update-cluster-config --name my-cluster --access-config authenticationMode=API_AND_CONFIG_MAP +---- +. Wait for the status of the cluster to return to Active. When the cluster is Active, you can follow the steps in <> to add access to the cluster for IAM principals. + + +== Required platform version + +To use _access entries_, the cluster must have a platform version that is the same or later than the version listed in the following table, or a Kubernetes version that is later than the versions listed in the table. If your Kubernetes version is not listed, all platform versions support access entries. + +[%header,cols="2"] +|=== +| Kubernetes version | Platform version +| Not Listed | All Supported +| `1.30` | `eks.2` +| `1.29` | `eks.1` +| `1.28` | `eks.6` +|=== + +For more information, see link:eks/latest/userguide/platform-versions.html[platform-versions,type="documentation"]. \ No newline at end of file diff --git a/latest/ug/manage-access/k8s-access/updating-access-entries.adoc b/latest/ug/manage-access/k8s-access/updating-access-entries.adoc new file mode 100644 index 000000000..a2224153a --- /dev/null +++ b/latest/ug/manage-access/k8s-access/updating-access-entries.adoc @@ -0,0 +1,34 @@ +include::../../attributes.txt[] + +[.topic] +[#updating-access-entries] += Update access entries + +You can update an access entry using the {aws-management-console} or the {aws} CLI. + + +[#access-update-console] +== {aws-management-console} +. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. +. Choose the name of the cluster that you want to create an access entry in. +. Choose the *Access* tab. +. Choose the access entry that you want to update. +. Choose *Edit*. +. For *Username*, you can change the existing value. +. For *Groups*, you can remove existing group names or add new group names. If the following groups names exist, don't remove them: *system:nodes* or *system:bootstrappers*. Removing these groups can cause your cluster to function improperly. If you don't specify any group names and want to use Amazon EKS authorization, associate an <> in a later step. +. For *Tags*, you can assign labels to the access entry. For example, to make it easier to find all resources with the same tag. You can also remove existing tags. +. Choose *Save changes*. +. If you want to associate an access policy to the entry, see <>. + +[#access-update-cli] +== {aws} CLI +. Install the {aws} CLI, as described in link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] in the {aws} Command Line Interface User Guide. +. To update an access entry +Replace [.replaceable]`my-cluster` with the name of your cluster, [.replaceable]`111122223333` with your {aws} account ID, and [.replaceable]`EKS-my-cluster-my-namespace-Viewers` with the name of an IAM role. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks update-access-entry --cluster-name my-cluster --principal-arn {arn-aws}iam::111122223333:role/EKS-my-cluster-my-namespace-Viewers --kubernetes-groups Viewers +---- ++ +You can't use the `--kubernetes-groups` option if the type of the access entry is a value other than `STANDARD`. You also can't associate an access policy to an access entry with a type other than `STANDARD`. \ No newline at end of file diff --git a/latest/ug/manage-access/view-kubernetes-resources.adoc b/latest/ug/manage-access/view-kubernetes-resources.adoc index 7a66764b4..25419af30 100644 --- a/latest/ug/manage-access/view-kubernetes-resources.adoc +++ b/latest/ug/manage-access/view-kubernetes-resources.adoc @@ -1,81 +1,51 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[view-kubernetes-resources,view-kubernetes-resources.title]] -= View [.noloc]`Kubernetes` resources in the {aws-management-console} -:info_doctype: section -:info_title: View Kubernetes resources in the {aws-management-console} -:info_titleabbrev: Access cluster resources with console -:info_abstract: Learn how to view Kubernetes resources in the {aws-management-console}. +[#view-kubernetes-resources] += View Kubernetes resources in the {aws-management-console} +:info_titleabbrev: Access cluster resources [abstract] -- -Learn how to view [.noloc]`Kubernetes` resources in the {aws-management-console}. +Learn how to view Kubernetes resources in the {aws-management-console}. -- -You can view the [.noloc]`Kubernetes` resources deployed to your cluster with the {aws-management-console}. You can't view [.noloc]`Kubernetes` resources with the {aws} CLI or https://eksctl.io/[eksctl]. To view [.noloc]`Kubernetes` resources using a command-line tool, use <>. +You can view the Kubernetes resources deployed to your cluster with the {aws-management-console}. You can't view Kubernetes resources with the {aws} CLI or https://eksctl.io/[eksctl]. To view Kubernetes resources using a command-line tool, use <>. [NOTE] ==== -To view the *Resources* tab and *Nodes* section on the *Compute* tab in the {aws-management-console}, the link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] that you're using must have specific IAM and [.noloc]`Kubernetes` permissions. For more information, see <>. +To view the *Resources* tab and *Nodes* section on the *Compute* tab in the {aws-management-console}, the link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] that you're using must have specific IAM and Kubernetes permissions. For more information, see <>. ==== . Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. -. In the *Clusters* list, select the cluster that contains the [.noloc]`Kubernetes` resources that you want to view. +. In the *Clusters* list, select the cluster that contains the Kubernetes resources that you want to view. . Select the *Resources* tab. -. Select a *Resource type* group that you want to view resources for, such as *Workloads*. You see a list of resource types in that group. -. Select a resource type, such as *Deployments*, in the *Workloads* group. You see a description of the resource type, a link to the [.noloc]`Kubernetes` documentation for more information about the resource type, and a list of resources of that type that are deployed on your cluster. If the list is empty, then there are no resources of that type deployed to your cluster. +. Select a *Resource type* group that you want to view resources for, such as *Workloads*. You see a list of resource types in that group. +. Select a resource type, such as *Deployments*, in the *Workloads* group. You see a description of the resource type, a link to the Kubernetes documentation for more information about the resource type, and a list of resources of that type that are deployed on your cluster. If the list is empty, then there are no resources of that type deployed to your cluster. . Select a resource to view more information about it. Try the following examples: + -** Select the *Workloads* group, select the *Deployments* resource type, and then select the *coredns* resource. When you select a resource, you are in *Structured view*, by default. For some resource types, you see a *Pods* section in *Structured view*. This section lists the [.noloc]`Pods` managed by the workload. You can select any [.noloc]`Pod` listed to view information about the [.noloc]`Pod`. Not all resource types display information in *Structured View*. If you select *Raw view* in the top right corner of the page for the resource, you see the complete JSON response from the [.noloc]`Kubernetes` API for the resource. -** Select the *Cluster* group and then select the *Nodes* resource type. You see a list of all nodes in your cluster. The nodes can be any <>. This is the same list that you see in the *Nodes* section when you select the *Compute* tab for your cluster. Select a node resource from the list. In *Structured view*, you also see a *Pods* section. This section shows you all [.noloc]`Pods` running on the node. +** Select the *Workloads* group, select the *Deployments* resource type, and then select the *coredns* resource. When you select a resource, you are in *Structured view*, by default. For some resource types, you see a *Pods* section in *Structured view*. This section lists the Pods managed by the workload. You can select any Pod listed to view information about the Pod. Not all resource types display information in *Structured View*. If you select *Raw view* in the top right corner of the page for the resource, you see the complete JSON response from the Kubernetes API for the resource. +** Select the *Cluster* group and then select the *Nodes* resource type. You see a list of all nodes in your cluster. The nodes can be any <>. This is the same list that you see in the *Nodes* section when you select the *Compute* tab for your cluster. Select a node resource from the list. In *Structured view*, you also see a *Pods* section. This section shows you all Pods running on the node. -[[view-kubernetes-resources-permissions,view-kubernetes-resources-permissions.title]] +[#view-kubernetes-resources-permissions] == Required permissions -To view the *Resources* tab and *Nodes* section on the *Compute* tab in the {aws-management-console}, the link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] that you're using must have specific minimum IAM and [.noloc]`Kubernetes` permissions. Complete the following steps to assign the required permissions to your IAM principals. +To view the *Resources* tab and *Nodes* section on the *Compute* tab in the {aws-management-console}, the link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] that you're using must have specific minimum IAM and Kubernetes permissions. Complete the following steps to assign the required permissions to your IAM principals. -. Make sure that the `eks:AccessKubernetesApi`, and other necessary IAM permissions to view [.noloc]`Kubernetes` resources, are assigned to the IAM principal that you're using. For more information about how to edit permissions for an IAM principal, see link:IAM/latest/UserGuide/access_controlling.html#access_controlling-principals[Controlling access for principals,type="documentation"] in the IAM User Guide. For more information about how to edit permissions for a role, see link:IAM/latest/UserGuide/roles-managingrole-editing-console.html#roles-modify_permissions-policy[Modifying a role permissions policy (console),type="documentation"] in the IAM User Guide. +. Make sure that the `eks:AccessKubernetesApi`, and other necessary IAM permissions to view Kubernetes resources, are assigned to the IAM principal that you're using. For more information about how to edit permissions for an IAM principal, see link:IAM/latest/UserGuide/access_controlling.html#access_controlling-principals[Controlling access for principals,type="documentation"] in the IAM User Guide. For more information about how to edit permissions for a role, see link:IAM/latest/UserGuide/roles-managingrole-editing-console.html#roles-modify_permissions-policy[Modifying a role permissions policy (console),type="documentation"] in the IAM User Guide. + -The following example policy includes the necessary permissions for a principal to view [.noloc]`Kubernetes` resources for all clusters in your account. Replace [.replaceable]`111122223333` with your {aws} account ID. +The following example policy includes the necessary permissions for a principal to view Kubernetes resources for all clusters in your account. Replace [.replaceable]`111122223333` with your {aws} account ID. + [source,json,subs="verbatim,attributes"] ---- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "eks:ListFargateProfiles", - "eks:DescribeNodegroup", - "eks:ListNodegroups", - "eks:ListUpdates", - "eks:AccessKubernetesApi", - "eks:ListAddons", - "eks:DescribeCluster", - "eks:DescribeAddonVersions", - "eks:ListClusters", - "eks:ListIdentityProviderConfigs", - "iam:ListRoles" - ], - "Resource": "*" - }, - { - "Effect": "Allow", - "Action": "ssm:GetParameter", - "Resource": "{arn-aws}ssm:*:111122223333:parameter/*" - } - ] -} ----- -+ -To view nodes in <>, the <> should be able to impersonate the principal in the cluster. This allows the <> to map the principal to a [.noloc]`Kubernetes` user. -. Create a [.noloc]`Kubernetes` `rolebinding` or `clusterrolebinding` that is bound to a [.noloc]`Kubernetes` `role` or `clusterrole` that has the necessary permissions to view the [.noloc]`Kubernetes` resources. To learn more about [.noloc]`Kubernetes` roles and role bindings, see https://kubernetes.io/docs/reference/access-authn-authz/rbac/[Using RBAC Authorization] in the [.noloc]`Kubernetes` documentation. You can apply one of the following manifests to your cluster that create a `role` and `rolebinding` or a `clusterrole` and `clusterrolebinding` with the necessary [.noloc]`Kubernetes` permissions: -+ -View [.noloc]`Kubernetes` resources in all namespaces::: +include::iam_snippet:6ed8ab20-4391-4d6d-831b-70710f90ec60[] +---- ++ +To view nodes in <>, the <> should be able to impersonate the principal in the cluster. This allows the <> to map the principal to a Kubernetes user. +. Create a Kubernetes `rolebinding` or `clusterrolebinding` that is bound to a Kubernetes `role` or `clusterrole` that has the necessary permissions to view the Kubernetes resources. To learn more about Kubernetes roles and role bindings, see https://kubernetes.io/docs/reference/access-authn-authz/rbac/[Using RBAC Authorization] in the Kubernetes documentation. You can apply one of the following manifests to your cluster that create a `role` and `rolebinding` or a `clusterrole` and `clusterrolebinding` with the necessary Kubernetes permissions: ++ +View Kubernetes resources in all namespaces::: ** The group name in the file is `eks-console-dashboard-full-access-group`. Apply the manifest to your cluster with the following command: + [source,bash,subs="verbatim,attributes"] @@ -84,7 +54,7 @@ kubectl apply -f https://s3.us-west-2.amazonaws.com/amazon-eks/docs/eks-console- ---- -View [.noloc]`Kubernetes` resources in a specific namespace::: +View Kubernetes resources in a specific namespace::: ** The namespace in this file is `default`. The group name in the file is `eks-console-dashboard-restricted-access-group`. Apply the manifest to your cluster with the following command: + [source,bash,subs="verbatim,attributes"] @@ -92,7 +62,7 @@ View [.noloc]`Kubernetes` resources in a specific namespace::: kubectl apply -f https://s3.us-west-2.amazonaws.com/amazon-eks/docs/eks-console-restricted-access.yaml ---- + -If you need to change the [.noloc]`Kubernetes` group name, namespace, permissions, or any other configuration in the file, then download the file and edit it before applying it to your cluster: +If you need to change the Kubernetes group name, namespace, permissions, or any other configuration in the file, then download the file and edit it before applying it to your cluster: + .. Download the file with one of the following commands: + @@ -117,7 +87,7 @@ kubectl apply -f eks-console-full-access.yaml ---- kubectl apply -f eks-console-restricted-access.yaml ---- -. Map the link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] to the [.noloc]`Kubernetes` user or group in the `aws-auth` `ConfigMap`. You can use a tool such as `eksctl` to update the `ConfigMap` or you can update it manually by editing it. +. Map the link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] to the Kubernetes user or group in the `aws-auth` `ConfigMap`. You can use a tool such as `eksctl` to update the `ConfigMap` or you can update it manually by editing it. + IMPORTANT: We recommend using `eksctl`, or another tool, to edit the `ConfigMap`. For information about other tools you can use, see https://aws.github.io/aws-eks-best-practices/security/docs/iam/#use-tools-to-make-changes-to-the-aws-auth-configmap[Use tools to make changes to the aws-authConfigMap] in the Amazon EKS best practices guides. An improperly formatted `aws-auth` `ConfigMap` can cause you to lose access to your cluster. @@ -161,7 +131,7 @@ An example output is as follows. [...] 2022-05-09 14:51:20 [ℹ] adding identity "{arn-aws}iam::111122223333:role/my-console-viewer-role" to auth ConfigMap ---- -. Add a mapping for a user. link:IAM/latest/UserGuide/id_users.html[IAM best practices,type="documentation"] recommend that you grant permissions to roles instead of users. This example assume that you attached the IAM permissions in the first step to a user named [.replaceable]`my-user`. Replace [.replaceable]`111122223333` with your account ID. +. Add a mapping for a user. link:IAM/latest/UserGuide/id_users.html[IAM best practices,type="documentation"] recommend that you grant permissions to roles instead of users. This example assume that you attached the IAM permissions in the first step to a user named [.replaceable]`my-user`. Replace [.replaceable]`111122223333` with your account ID. + [source,bash,subs="verbatim,attributes"] ---- @@ -209,7 +179,7 @@ For more information about adding users or roles to the `aws-auth` `ConfigMap`, ---- kubectl edit -n kube-system configmap/aws-auth ---- -. Add the mappings to the `aws-auth` `ConfigMap`, but don't replace any of the existing mappings. The following example adds mappings between link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principals,type="documentation"] with permissions added in the first step and the [.noloc]`Kubernetes` groups created in the previous step: +. Add the mappings to the `aws-auth` `ConfigMap`, but don't replace any of the existing mappings. The following example adds mappings between link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principals,type="documentation"] with permissions added in the first step and the Kubernetes groups created in the previous step: + ** The [.replaceable]`my-console-viewer-role` role and the `eks-console-dashboard-full-access-group`. ** The [.replaceable]`my-user` user and the `eks-console-dashboard-restricted-access-group`. @@ -233,4 +203,4 @@ mapUsers: | ---- + IMPORTANT: The role ARN can't include a path such as `role/my-team/developers/my-console-viewer-role`. The format of the ARN must be `{arn-aws}iam::[.replaceable]``111122223333``:role/[.replaceable]``my-console-viewer-role```. In this example, `my-team/developers/` needs to be removed. -. Save the file and exit your text editor. +. Save the file and exit your text editor. \ No newline at end of file diff --git a/latest/ug/ml/capacity-blocks-mng.adoc b/latest/ug/ml/capacity-blocks-mng.adoc index c8d664dcb..5726cb2d8 100644 --- a/latest/ug/ml/capacity-blocks-mng.adoc +++ b/latest/ug/ml/capacity-blocks-mng.adoc @@ -1,9 +1,9 @@ -//!!NODE_ROOT
include::../attributes.txt[] + [.topic] -[[capacity-blocks-mng,capacity-blocks-mng.title]] +[#capacity-blocks-mng] = Create a managed node group with Capacity Blocks for ML -:info_titleabbrev: Reserve GPUs for MNG +:info_titleabbrev: Reserve GPUs for managed node groups [abstract] -- @@ -12,21 +12,21 @@ Capacity Blocks for machine learning (ML) allow you to reserve highly sought-aft Capacity Blocks for machine learning (ML) allow you to reserve GPU instances on a future date to support your short duration ML workloads. For more information, see link:AWSEC2/latest/UserGuide/ec2-capacity-blocks.html[Capacity Blocks for ML,type="documentation"] in the _Amazon EC2 User Guide for Linux Instances_. -[[capacity-blocks-mng-considerations,capacity-blocks-mng-considerations.title]] +[#capacity-blocks-mng-considerations] == Considerations [IMPORTANT] ==== -* Capacity Blocks are only available for certain Amazon EC2 instance types and {aws} Regions. For compatibility information, see link:AWSEC2/latest/UserGuide/capacity-blocks-using.html#capacity-blocks-prerequisites[Work with Capacity Blocks Prerequisites,type="documentation"] in the _Amazon EC2 User Guide for Linux Instances_. +* Capacity Blocks are only available for certain Amazon EC2 instance types and {aws} Regions. For compatibility information, see link:AWSEC2/latest/UserGuide/capacity-blocks-using.html#capacity-blocks-prerequisites[Work with Capacity Blocks Prerequisites,type="documentation"] in the _Amazon EC2 User Guide for Linux Instances_. * For more information, see link:autoscaling/ec2/userguide/launch-template-capacity-blocks.html[Use Capacity Blocks for machine learning workloads,type="documentation"] in the _Amazon EC2 Auto Scaling User Guide_. * Managed node groups with Capacity Blocks can only be created with custom launch templates. * When upgrading managed node groups with Capacity Blocks, make sure that the desired size of the node group is set to `0`. ==== -[[capacity-blocks-mng-procedure,capacity-blocks-mng-procedure.title]] +[#capacity-blocks-mng-procedure] == Create a managed node group with Amazon EC2 Capacity Blocks You can use Capacity Blocks with Amazon EKS managed node groups for provisioning and scaling GPU-accelerated worker nodes. The {aws} CloudFormation template examples that follow don't cover every aspect needed in a production clusters. Typically, you'd also want a bootstrapping script to join the node to the cluster and specify an Amazon EKS accelerated AMI. For more information, see <>. @@ -82,8 +82,8 @@ aws eks create-nodegroup \ + ** Create a scheduled scaling policy for the ASG that aligns to the Capacity Block reservation start time. For more information, see link:autoscaling/ec2/userguide/ec2-auto-scaling-scheduled-scaling.html[Scheduled scaling for Amazon EC2 Auto Scaling,type="documentation"] in the _Amazon EC2 Auto Scaling User Guide_. ** Use the Amazon EKS console or `eks update-nodegroup-config` to update the scaling config and set the desired size of the node group. -** Use the [.noloc]`Kubernetes` Cluster Autoscaler. For more information, see https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md[Cluster Autoscaler on {aws}]. -. The node group is now ready for workloads and [.noloc]`Pods` to be scheduled. -. In order for your [.noloc]`Pods` to be gracefully drained before reservation ends, Amazon EKS uses a scheduled scaling policy to scale down the node group size to `0` . This scheduled scaling will be set with name titled `Amazon EKS Node Group Capacity Scaledown Before Reservation End` . We recommend not editing or deleting this action. +** Use the Kubernetes Cluster Autoscaler. For more information, see https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md[Cluster Autoscaler on {aws}]. +. The node group is now ready for workloads and Pods to be scheduled. +. In order for your Pods to be gracefully drained before reservation ends, Amazon EKS uses a scheduled scaling policy to scale down the node group size to `0` . This scheduled scaling will be set with name titled `Amazon EKS Node Group Capacity Scaledown Before Reservation End` . We recommend not editing or deleting this action. + -Amazon EC2 starts shutting down the instances 30 minutes before reservation end time. As a result, Amazon EKS will setup a scheduled scale down on the node group 40 minutes prior to their reservation end in order to safely and gracefully evict [.noloc]`Pods`. +Amazon EC2 starts shutting down the instances 30 minutes before reservation end time. As a result, Amazon EKS will setup a scheduled scale down on the node group 40 minutes prior to their reservation end in order to safely and gracefully evict Pods. \ No newline at end of file diff --git a/latest/ug/ml/capacity-blocks.adoc b/latest/ug/ml/capacity-blocks.adoc index 8f9ab3eb7..c55c4e2b2 100644 --- a/latest/ug/ml/capacity-blocks.adoc +++ b/latest/ug/ml/capacity-blocks.adoc @@ -1,9 +1,9 @@ -//!!NODE_ROOT
include::../attributes.txt[] + [.topic] -[[capacity-blocks,capacity-blocks.title]] +[#capacity-blocks] = Create self-managed nodes with Capacity Blocks for ML -:info_titleabbrev: Reserve GPUs for SMN +:info_titleabbrev: Reserve GPUs for self-managed nodes [abstract] -- @@ -12,22 +12,21 @@ Capacity Blocks for machine learning (ML) allow you to reserve highly sought-aft Capacity Blocks for machine learning (ML) allow you to reserve GPU instances on a future date to support your short duration ML workloads. For more information, see link:AWSEC2/latest/UserGuide/ec2-capacity-blocks.html[Capacity Blocks for ML,type="documentation"] in the _Amazon EC2 User Guide for Linux Instances_. -[[capacity-blocks-considerations,capacity-blocks-considerations.title]] +[#capacity-blocks-considerations] == Considerations [IMPORTANT] ==== -* Capacity Blocks are only available for certain Amazon EC2 instance types and {aws} Regions. For compatibility information, see link:AWSEC2/latest/UserGuide/capacity-blocks-using.html#capacity-blocks-prerequisites[Work with Capacity Blocks Prerequisites,type="documentation"] in the _Amazon EC2 User Guide for Linux Instances_. -* Capacity Blocks currently cannot be used with [.noloc]`Karpenter`. -* If you create a self-managed node group prior to the capacity reservation becoming active, then set the desired capacity to `0`. +* Capacity Blocks are only available for certain Amazon EC2 instance types and {aws} Regions. For compatibility information, see link:AWSEC2/latest/UserGuide/capacity-blocks-using.html#capacity-blocks-prerequisites[Work with Capacity Blocks Prerequisites,type="documentation"] in the _Amazon EC2 User Guide for Linux Instances_. +* If you create a self-managed node group prior to the capacity reservation becoming active, then set the desired capacity to `0`. * To allow sufficient time to gracefully drain the node(s), we suggest that you schedule scaling to scale to zero more than 30 minutes before the Capacity Block reservation end time. -* In order for your [.noloc]`Pods` to be gracefully drained, we recommend that you set up {aws} Node Termination Handler as explained in the example steps. +* In order for your Pods to be gracefully drained, we recommend that you set up {aws} Node Termination Handler as explained in the example steps. ==== -[[capacity-blocks-procedure,capacity-blocks-procedure.title]] +[#capacity-blocks-procedure] == Use Capacity Blocks with self-managed nodes You can use Capacity Blocks with Amazon EKS for provisioning and scaling your self-managed nodes. The following steps give a general example overview. The {aws} CloudFormation template examples don't cover every aspect needed in a production workload. Typically you'd also want a bootstrapping script to join the node to the cluster, specify an Amazon EKS accelerated AMI, and an appropriate instance profile for joining the cluster. For more information, see <>. @@ -91,13 +90,13 @@ NodeGroup: PropagateAtLaunch: true Value: owned ---- -. Once the node group is created successfully, make sure to record the `NodeInstanceRole` for the node group that was created. You need this in order to make sure that when node group is scaled, the new nodes join the cluster and [.noloc]`Kubernetes` is able to recognize the nodes. For more information, see the {aws-management-console} instructions in <>. +. Once the node group is created successfully, make sure to record the `NodeInstanceRole` for the node group that was created. You need this in order to make sure that when node group is scaled, the new nodes join the cluster and Kubernetes is able to recognize the nodes. For more information, see the {aws-management-console} instructions in <>. . We recommend that you create a scheduled scaling policy for the Auto Scaling group that aligns to the Capacity Block reservation times. For more information, see link:autoscaling/ec2/userguide/ec2-auto-scaling-scheduled-scaling.html[Scheduled scaling for Amazon EC2 Auto Scaling,type="documentation"] in the _Amazon EC2 Auto Scaling User Guide_. + You can use all of the instances you reserved until 30 minutes before the end time of the Capacity Block. Instances that are still running at that time will start terminating. To allow sufficient time to gracefully drain the node(s), we suggest that you schedule scaling to scale to zero more than 30 minutes before the Capacity Block reservation end time. + If you want to instead scale up manually whenever the capacity reservation becomes `Active`, then you need to update the Auto Scaling group's desired capacity at the start time of the Capacity Block reservation. Then you would need to also scale down manually more than 30 minutes before the Capacity Block reservation end time. -. The node group is now ready for workloads and [.noloc]`Pods` to be scheduled. -. In order for your [.noloc]`Pods` to be gracefully drained, we recommend that you set up {aws} Node Termination Handler. This handler will be able to watch for "ASG Scale-in" lifecycle events from Amazon EC2 Auto Scaling using EventBridge and allow the [.noloc]`Kubernetes` control plane to take required action before the instance becomes unavailable. Otherwise, your [.noloc]`Pods` and [.noloc]`Kubernetes` objects will get stuck in a pending state. For more information, see https://github.com/aws/aws-node-termination-handler[{aws} Node Termination Handler] on GitHub. +. The node group is now ready for workloads and Pods to be scheduled. +. In order for your Pods to be gracefully drained, we recommend that you set up {aws} Node Termination Handler. This handler will be able to watch for "ASG Scale-in" lifecycle events from Amazon EC2 Auto Scaling using EventBridge and allow the Kubernetes control plane to take required action before the instance becomes unavailable. Otherwise, your Pods and Kubernetes objects will get stuck in a pending state. For more information, see https://github.com/aws/aws-node-termination-handler[{aws} Node Termination Handler] on GitHub. + -If you don't setup a Node Termination Handler, we recommend that you start draining your [.noloc]`Pods` manually before hitting the 30 minute window so that they have enough time to be gracefully drained. +If you don't setup a Node Termination Handler, we recommend that you start draining your Pods manually before hitting the 30 minute window so that they have enough time to be gracefully drained. \ No newline at end of file diff --git a/latest/ug/ml/images b/latest/ug/ml/images new file mode 120000 index 000000000..5e6757319 --- /dev/null +++ b/latest/ug/ml/images @@ -0,0 +1 @@ +../images \ No newline at end of file diff --git a/latest/ug/ml/inferentia-support.adoc b/latest/ug/ml/inferentia-support.adoc index be85ead18..f15c75ad6 100644 --- a/latest/ug/ml/inferentia-support.adoc +++ b/latest/ug/ml/inferentia-support.adoc @@ -1,13 +1,9 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[inferentia-support,inferentia-support.title]] -= Use {aws} [.noloc]`Inferentia` instances with Amazon EKS for Machine Learning -:info_doctype: section -:info_title: Use {aws} Inferentia instances with your EKS cluster for Machine Learning -:info_titleabbrev: Prepare Inferentia clusters -:info_abstract: Learn how to create an Amazon EKS cluster with nodes running Amazon EC2 Inf1 instances for machine learning inference using {aws} Inferentia chips and deploy a TensorFlow Serving application. +[#inferentia-support] += Use {aws} Inferentia instances with Amazon EKS for Machine Learning +:info_titleabbrev: Set up inference clusters with Inferentia [abstract] -- @@ -19,11 +15,11 @@ This topic describes how to create an Amazon EKS cluster with nodes running link [NOTE] ==== -Neuron device logical IDs must be contiguous. If a [.noloc]`Pod` requesting multiple Neuron devices is scheduled on an `inf1.6xlarge` or `inf1.24xlarge` instance type (which have more than one Neuron device), that [.noloc]`Pod` will fail to start if the [.noloc]`Kubernetes` scheduler selects non-contiguous device IDs. For more information, see https://github.com/aws/aws-neuron-sdk/issues/110[Device logical IDs must be contiguous] on [.noloc]`GitHub`. +Neuron device logical IDs must be contiguous. If a Pod requesting multiple Neuron devices is scheduled on an `inf1.6xlarge` or `inf1.24xlarge` instance type (which have more than one Neuron device), that Pod will fail to start if the Kubernetes scheduler selects non-contiguous device IDs. For more information, see https://github.com/aws/aws-neuron-sdk/issues/110[Device logical IDs must be contiguous] on GitHub. ==== -[[inferentia-prerequisites,inferentia-prerequisites.title]] +[#inferentia-prerequisites] == Prerequisites * Have `eksctl` installed on your computer. If you don't have it installed, see https://eksctl.io/installation[Installation] in the `eksctl` documentation. @@ -31,7 +27,7 @@ Neuron device logical IDs must be contiguous. If a [.noloc]`Pod` requesting mult * (Optional) Have `python3` installed on your computer. If you don't have it installed, then see https://www.python.org/downloads/[Python downloads] for installation instructions. -[[create-cluster-inferentia,create-cluster-inferentia.title]] +[#create-cluster-inferentia] == Create a cluster . Create a cluster with Inf1 Amazon EC2 instance nodes. You can replace [.replaceable]`inf1.2xlarge` with any link:ec2/instance-types/inf1/[Inf1 instance type,type="marketing"]. The `eksctl` utility detects that you are launching a node group with an `Inf1` instance type and will start your nodes using one of the Amazon EKS optimized accelerated Amazon Linux AMIs. @@ -60,8 +56,8 @@ NOTE: Note the value of the following line of the output. It's used in a later ( [9] adding identity "{arn-aws}iam::111122223333:role/eksctl-inferentia-nodegroup-ng-in-NodeInstanceRole-FI7HIYS3BS09" to auth ConfigMap ---- + -When launching a node group with `Inf1` instances, `eksctl` automatically installs the {aws} Neuron [.noloc]`Kubernetes` device plugin. This plugin advertises Neuron devices as a system resource to the [.noloc]`Kubernetes` scheduler, which can be requested by a container. In addition to the default Amazon EKS node IAM policies, the Amazon S3 read only access policy is added so that the sample application, covered in a later step, can load a trained model from Amazon S3. -. Make sure that all [.noloc]`Pods` have started correctly. +When launching a node group with `Inf1` instances, `eksctl` automatically installs the {aws} Neuron Kubernetes device plugin. This plugin advertises Neuron devices as a system resource to the Kubernetes scheduler, which can be requested by a container. In addition to the default Amazon EKS node IAM policies, the Amazon S3 read only access policy is added so that the sample application, covered in a later step, can load a trained model from Amazon S3. +. Make sure that all Pods have started correctly. + [source,bash,subs="verbatim,attributes"] ---- @@ -79,12 +75,12 @@ neuron-device-plugin-daemonset-hwjsj 1/1 Running 0 5m ---- -[[deploy-tensorflow-serving-application,deploy-tensorflow-serving-application.title]] +[#deploy-tensorflow-serving-application] == (Optional) Deploy a TensorFlow Serving application image A trained model must be compiled to an Inferentia target before it can be deployed on Inferentia instances. To continue, you will need a https://awsdocs-neuron.readthedocs-hosted.com/en/latest/neuron-guide/neuron-frameworks/tensorflow-neuron/index.html[Neuron optimized TensorFlow] model saved in Amazon S3. If you don't already have a SavedModel, please follow the tutorial for link:dlami/latest/devguide/tutorial-inferentia-tf-neuron.html[creating a Neuron compatible ResNet50 model,type="documentation"] and upload the resulting SavedModel to S3. ResNet-50 is a popular machine learning model used for image recognition tasks. For more information about compiling Neuron models, see link:dlami/latest/devguide/tutorial-inferentia.html[The {aws} Inferentia Chip With DLAMI,type="documentation"] in the {aws} Deep Learning AMIs Developer Guide. -The sample deployment manifest manages a pre-built inference serving container for TensorFlow provided by {aws} Deep Learning Containers. Inside the container is the {aws} Neuron Runtime and the TensorFlow Serving application. A complete list of pre-built Deep Learning Containers optimized for Neuron is maintained on [.noloc]`GitHub` under https://github.com/aws/deep-learning-containers/blob/master/available_images.md#neuron-inference-containers[Available Images]. At start-up, the DLC will fetch your model from Amazon S3, launch Neuron TensorFlow Serving with the saved model, and wait for prediction requests. +The sample deployment manifest manages a pre-built inference serving container for TensorFlow provided by {aws} Deep Learning Containers. Inside the container is the {aws} Neuron Runtime and the TensorFlow Serving application. A complete list of pre-built Deep Learning Containers optimized for Neuron is maintained on GitHub under https://github.com/aws/deep-learning-containers/blob/master/available_images.md#neuron-inference-containers[Available Images]. At start-up, the DLC will fetch your model from Amazon S3, launch Neuron TensorFlow Serving with the saved model, and wait for prediction requests. The number of Neuron devices allocated to your serving application can be adjusted by changing the `aws.amazon.com/neuron` resource in the deployment yaml. Please note that communication between TensorFlow Serving and the Neuron runtime happens over GRPC, which requires passing the `IPC_LOCK` capability to the container. @@ -191,7 +187,7 @@ spec: app: eks-neuron-test role: master ---- -. Create a [.noloc]`Kubernetes` service for your TensorFlow model Serving application. +. Create a Kubernetes service for your TensorFlow model Serving application. + [source,bash,subs="verbatim,attributes"] ---- @@ -199,7 +195,7 @@ kubectl apply -f rn50_service.yaml ---- -[[make-predictions-against-tensorflow-service,make-predictions-against-tensorflow-service.title]] +[#make-predictions-against-tensorflow-service] == (Optional) Make predictions against your TensorFlow Serving service . To test locally, forward the gRPC port to the `eks-neuron-test` service. + @@ -248,4 +244,4 @@ An example output is as follows. [source,bash,subs="verbatim,attributes"] ---- [[(u'n02123045', u'tabby', 0.68817204), (u'n02127052', u'lynx', 0.12701613), (u'n02123159', u'tiger_cat', 0.08736559), (u'n02124075', u'Egyptian_cat', 0.063844085), (u'n02128757', u'snow_leopard', 0.009240591)]] ----- +---- \ No newline at end of file diff --git a/latest/ug/ml/machine-learning-on-eks.adoc b/latest/ug/ml/machine-learning-on-eks.adoc index 3a1aaf863..1aec2b1c5 100644 --- a/latest/ug/ml/machine-learning-on-eks.adoc +++ b/latest/ug/ml/machine-learning-on-eks.adoc @@ -1,68 +1,81 @@ -//!!NODE_ROOT include::../attributes.txt[] + [.topic] -[[machine-learning-on-eks,machine-learning-on-eks.title]] -= Overview of Machine Learning on Amazon EKS -:doctype: book -:sectnums: -:toc: left -:icons: font -:experimental: -:idprefix: -:idseparator: - -:sourcedir: . -:info_doctype: chapter -:info_title: Machine Learning on Amazon EKS Overview -:info_titleabbrev: Machine Learning on EKS -:keywords: Machine Learning, Amazon EKS, Artificial Intelligence -:info_abstract: Learn to manage containerized applications with Amazon EKS +[#machine-learning-on-eks] += Overview of Artificial Intelligence (AI) and Machine Learning (ML) on Amazon EKS +:info_titleabbrev: AI/ML on EKS [abstract] -- Complete guide for running Machine Learning applications on Amazon EKS. This includes everything from provisioning infrastructure to choosing and deploying Machine Learning workloads on Amazon EKS. -- -[[ml-features,ml-features.title]] +Amazon Elastic Kubernetes Service (EKS) is a managed Kubernetes platform that empowers organizations to deploy, manage, and scale AI and machine learning (ML) workloads with unparalleled flexibility and control. Built on the open source Kubernetes ecosystem, EKS lets you harness your existing Kubernetes expertise, while integrating seamlessly with open source tools and {aws} services. + +Whether you’re training large-scale models, running real-time online inference, or deploying generative AI applications, EKS delivers the performance, scalability, and cost efficiency your AI/ML projects demand. + + +## Why Choose EKS for AI/ML? -Machine Learning (ML) is an area of Artificial Intelligence (AI) where machines process large amounts of data to look for patterns and make connections between the data. This can expose new relationships and help predict outcomes that might not have been apparent otherwise. +EKS is a managed Kubernetes platform that helps you deploy and manage complex AI/ML workloads. +Built on the open source Kubernetes ecosystem, it integrates with {aws} services, providing the control and scalability needed for advanced projects. +For teams new to AI/ML deployments, existing Kubernetes skills transfer directly, allowing efficient orchestration of multiple workloads. -For large-scale ML projects, data centers must be able to store large amounts of data, process data quickly, and integrate data from many sources. The platforms running ML applications must be reliable and secure, but also offer resiliency to recover from data center outages and application failures. {aws} Elastic Kubernetes Service (EKS), running in the {aws} cloud, is particularly suited for ML workloads. +EKS supports everything from operating system customizations to compute scaling, and its open source foundation promotes technological flexibility, preserving choice for future infrastructure decisions. +The platform provides the performance and tuning options AI/ML workloads require, supporting features such as: -The primary goal of this section of the EKS User Guide is to help you put together the hardware and software component to build platforms to run Machine Learning workloads in an EKS cluster. -We start by explaining the features and services available to you in EKS and the {aws} cloud, then provide you with tutorials to help you work with ML platforms, frameworks, and models. +* Full cluster control to fine-tune costs and configurations without hidden abstractions +* Sub-second latency for real-time inference workloads in production +* Advanced customizations like multi-instance GPUs, multi-cloud strategies, and OS-level tuning +* Ability to centralize workloads using EKS as a unified orchestrator across AI/ML pipelines -== Advantages of Machine Learning on EKS and the {aws} cloud +## Key use cases -Amazon Elastic Kubernetes Service (EKS) is a powerful, managed Kubernetes platform that has become a cornerstone for deploying and managing AI/ML workloads in the cloud. With its ability to handle complex, resource-intensive tasks, Amazon EKS provides a scalable and flexible foundation for running AI/ML models, making it an ideal choice for organizations aiming to harness the full potential of machine learning. +Amazon EKS provides a robust platform for a wide range of AI/ML workloads, supporting various technologies and deployment patterns: -Key Advantages of AI/ML Platforms on Amazon EKS include: +* **Real-time (online) inference:** EKS powers immediate predictions on incoming data, such as fraud detection, with sub-second latency using tools like https://docs.aws.amazon.com/dlami/latest/devguide/tutorial-torchserve.html[TorchServe], +https://aws.amazon.com/blogs/containers/quora-3x-faster-machine-learning-25-lower-costs-with-nvidia-triton-on-amazon-eks/[Triton Inference Server], and https://kserve.github.io/website/0.8/get_started/first_isvc/[KServe] on Amazon EC2 https://aws.amazon.com/ec2/instance-types/inf1/[Inf1] +and https://aws.amazon.com/ec2/instance-types/inf2/[Inf2] instances. +These workloads benefit from dynamic scaling with https://karpenter.sh/[Karpenter] and https://keda.sh/[KEDA], while leveraging https://aws.amazon.com/efs/[Amazon EFS] for model sharding across pods. +https://docs.aws.amazon.com/AmazonECR/latest/userguide/pull-through-cache-creating-rule.html[Amazon ECR Pull Through Cache (PTC)] accelerates model updates, +and https://aws.amazon.com/bottlerocket/[Bottlerocket] data volumes with https://docs.aws.amazon.com/ebs/latest/userguide/what-is-ebs.html[Amazon EBS]-optimized volumes ensure fast data access. -* *Scalability and Flexibility* -Amazon EKS enables organizations to scale AI/ML workloads seamlessly. Whether you're training large language models that require vast amounts of compute power or deploying inference pipelines that need to handle unpredictable traffic patterns, EKS scales up and down efficiently, optimizing resource use and cost. +* **General model training:** Organizations leverage EKS to train complex models on large datasets over extended periods using the https://www.kubeflow.org/docs/components/trainer/[Kubeflow Training Operator (KRO)], +https://docs.ray.io/en/latest/serve/index.html[Ray Serve], and https://pytorch.org/docs/stable/distributed.elastic.html[Torch Distributed Elastic] on https://aws.amazon.com/ec2/instance-types/p4/[Amazon EC2 P4d] and https://aws.amazon.com/ec2/instance-types/trn1/[Amazon EC2 Trn1] instances. +These workloads are supported by batch scheduling with tools like https://volcano.sh/en/#home_slider[Volcano], +https://yunikorn.apache.org/[Yunikorn], +and https://kueue.sigs.k8s.io/[Kueue]. +https://aws.amazon.com/efs/[Amazon EFS] enables sharing of model checkpoints, and https://aws.amazon.com/s3/[Amazon S3] handles model import/export with lifecycle policies for version management. -* *High Performance with GPUs and Neuron Instances* -Amazon EKS supports a wide range of compute options, including GPUs and {aws}} Neuron instances, which are essential for accelerating AI/ML workloads. This support allows for high-performance training and low-latency inference, ensuring that models run efficiently in production environments. +* **Retrieval augmented generation (RAG) pipelines:** EKS manages customer support chatbots and similar applications by integrating retrieval and generation processes. These workloads often use tools like https://argoproj.github.io/workflows/[Argo Workflows] and https://www.kubeflow.org/[Kubeflow] for orchestration, vector databases like https://www.pinecone.io/blog/serverless/[Pinecone], https://weaviate.io/[Weaviate], or https://aws.amazon.com/opensearch-service/[Amazon OpenSearch], and expose applications to users via the +<>. https://docs.nvidia.com/nim/index.html[NVIDIA NIM] optimizes GPU utilization, while <> and https://aws.amazon.com/grafana/[Grafana] monitor resource usage. -* *Integration with AI/ML Tools* -Amazon EKS integrates seamlessly with popular AI/ML tools and frameworks like TensorFlow, PyTorch, and Ray, providing a familiar and robust ecosystem for data scientists and engineers. These integrations enable users to leverage existing tools while benefiting from the scalability and management capabilities of Kubernetes. +* **Generative AI model deployment:** Companies deploy real-time content creation services on EKS, such as text or image generation, using https://docs.ray.io/en/latest/serve/index.html[Ray Serve], https://github.com/vllm-project/vllm[vLLM], and https://aws.amazon.com/blogs/containers/quora-3x-faster-machine-learning-25-lower-costs-with-nvidia-triton-on-amazon-eks/[Triton Inference Server] on Amazon https://aws.amazon.com/ec2/instance-types/g5/[EC2 G5] and https://aws.amazon.com/ai/machine-learning/inferentia/[Inferentia] accelerators. These deployments optimize performance and memory utilization for large-scale models. https://jupyter.org/hub[JupyterHub] enables iterative development, https://www.gradio.app/[Gradio] provides simple web interfaces, and the <> allows mounting S3 buckets as file systems for accessing large model files. -* *Automation and Management* -Kubernetes on Amazon EKS automates many of the operational tasks associated with managing AI/ML workloads. Features like automatic scaling, rolling updates, and self-healing ensure that your applications remain highly available and resilient, reducing the overhead of manual intervention. +* **Batch (offline) inference:** Organizations process large datasets efficiently through scheduled jobs with https://docs.aws.amazon.com/batch/latest/userguide/what-is-batch.html[{aws} Batch] or https://volcano.sh/en/docs/schduler_introduction/[Volcano]. These workloads often use https://aws.amazon.com/ec2/instance-types/inf1/[Inf1] and https://aws.amazon.com/ec2/instance-types/inf2/[Inf2] EC2 instances for {aws} https://aws.amazon.com/ai/machine-learning/inferentia/[Inferentia] chips, Amazon EC2 https://aws.amazon.com/ec2/instance-types/g4/[G4dn] instances for NVIDIA T4 GPUs, or https://aws.amazon.com/ec2/instance-types/c5/[c5] and https://aws.amazon.com/ec2/instance-types/c6i[c6i] CPU instances, maximizing resource utilization during off-peak hours for analytics tasks. The https://aws.amazon.com/ai/machine-learning/neuron/[{aws} Neuron SDK] and NVIDIA GPU drivers optimize performance, while MIG/TS enables GPU sharing. Storage solutions include https://aws.amazon.com/s3/[Amazon S3] and Amazon https://aws.amazon.com/efs/[EFS] and https://aws.amazon.com/fsx/lustre/[FSx for Lustre], with CSI drivers for various storage classes. Model management leverages tools like https://www.kubeflow.org/docs/components/pipelines/[Kubeflow Pipelines], https://argoproj.github.io/workflows/[Argo Workflows], and https://docs.ray.io/en/latest/cluster/getting-started.html[Ray Cluster], while monitoring is handled by <>, https://aws.amazon.com/grafana/[Grafana] and custom model monitoring tools. -* *Security and Compliance* -Running AI/ML workloads on Amazon EKS provides robust security features, including fine-grained IAM roles, encryption, and network policies, ensuring that sensitive data and models are protected. EKS also adheres to various compliance standards, making it suitable for enterprises with strict regulatory requirements. +## Case studies -== Why Choose Amazon EKS for AI/ML? -Amazon EKS offers a comprehensive, managed environment that simplifies the deployment of AI/ML models while providing the performance, scalability, and security needed for production workloads. With its ability to integrate with a variety of AI/ML tools and its support for advanced compute resources, EKS empowers organizations to accelerate their AI/ML initiatives and deliver innovative solutions at scale. +Customers choose Amazon EKS for various reasons, such as optimizing GPU usage or running real-time inference workloads with sub-second latency, as demonstrated in the following case studies. For a list of all case studies for Amazon EKS, see https://aws.amazon.com/solutions/case-studies/browse-customer-success-stories/?refid=cr_card&customer-references-cards.sort-by=item.additionalFields.sortDate&customer-references-cards.sort-order=desc&awsf.customer-references-location=*all&awsf.customer-references-industry=*all&awsf.customer-references-use-case=*all&awsf.language=language%23english&awsf.customer-references-segment=*all&awsf.content-type=*all&awsf.customer-references-product=product%23eks&awsm.page-customer-references-cards=1[{aws} Customer Success Stories]. -By choosing Amazon EKS, you gain access to a robust infrastructure that can handle the complexities of modern AI/ML workloads, allowing you to focus on innovation and value creation rather than managing underlying systems. Whether you are deploying simple models or complex AI systems, Amazon EKS provides the tools and capabilities needed to succeed in a competitive and rapidly evolving field. +* https://aws.amazon.com/solutions/case-studies/unitary-eks-case-study/?did=cr_card&trk=cr_card[Unitary] processes 26 million videos daily using AI for content moderation, requiring high-throughput, low-latency inference and have achieved an 80% reduction in container boot times, ensuring fast response to scaling events as traffic fluctuates. +* https://aws.amazon.com/solutions/case-studies/miro-eks-case-study/[Miro], the visual collaboration platform supporting 70 million users worldwide, reported an 80% reduction in compute costs compared to their previous self-managed Kubernetes clusters. +* https://aws.amazon.com/solutions/case-studies/synthesia-case-study/?did=cr_card&trk=cr_card[Synthesia], which offers generative AI video creation as a service for customers to create realistic videos from text prompts, achieved a 30x improvement in ML model training throughput. +* https://aws.amazon.com/solutions/case-studies/harri-eks-case-study/?did=cr_card&trk=cr_card[Harri], providing HR technology for the hospitality industry, achieved 90% faster scaling in response to spikes in demand and reduced its compute costs by 30% by migrating to https://aws.amazon.com/ec2/graviton/[{aws} Graviton processors]. +* https://aws.amazon.com/solutions/case-studies/ada-support-eks-case-study/[Ada Support], an AI-powered customer service automation company, achieved a 15% reduction in compute costs alongside a 30% increase in compute efficiency. +* https://aws.amazon.com/blogs/startups/how-snorkel-ai-achieved-over-40-cost-savings-by-scaling-machine-learning-workloads-using-amazon-eks/[Snorkel AI], which equips enterprises to build and adapt foundation models and large language models, achieved over 40% cost savings by implementing intelligent scaling mechanisms for their GPU resources. == Start using Machine Learning on EKS -To begin planning for and using Machine Learning platforms and workloads on EKS on the {aws} cloud, proceed to the <> section. +To begin planning for and using Machine Learning platforms and workloads on EKS on the {aws} cloud, proceed to the <> section. + +include::ml-realtime-inference.adoc[leveloffset=+1] + +include::ml-cluster-configuration.adoc[leveloffset=+1] + +include::ml-compute-management.adoc[leveloffset=+1] -include::ml-get-started.adoc[leveloffset=+1] +include::ml-recipes.adoc[leveloffset=+1] -include::ml-prepare-for-cluster.adoc[leveloffset=+1] +include::ml-resources.adoc[leveloffset=+1] -include::ml-tutorials.adoc[leveloffset=+1] diff --git a/latest/ug/ml/ml-cluster-configuration.adoc b/latest/ug/ml/ml-cluster-configuration.adoc new file mode 100644 index 000000000..8e1c8ca41 --- /dev/null +++ b/latest/ug/ml/ml-cluster-configuration.adoc @@ -0,0 +1,27 @@ +include::../attributes.txt[] + +[.topic] +[#ml-cluster-configuration] += Amazon EKS cluster configuration for AI/ML workloads +:info_titleabbrev: Cluster configuration + +[abstract] +-- +Learn different ways to setup and configure Amazon EKS clusters for AI/ML workloads. +-- + +This section is designed to help you configure Amazon EKS clusters optimized for AI/ML workloads. You'll find guidance on running GPU-accelerated containers using Linux and Windows optimized AMIs, setting up training clusters with Elastic Fabric Adapter (EFA) for high-performance networking, and creating inference clusters with {aws} Inferentia instances, including prerequisites, step-by-step procedures, and deployment considerations. + + +[.topiclist] +[[Topic List]] + +include::ml-eks-optimized-ami.adoc[leveloffset=+1] + +include::ml-eks-k8s-device-plugin.adoc[leveloffset=+1] + +include::ml-eks-windows-optimized-ami.adoc[leveloffset=+1] + +include::node-efa.adoc[leveloffset=+1] + +include::inferentia-support.adoc[leveloffset=+1] diff --git a/latest/ug/ml/ml-compute-management.adoc b/latest/ug/ml/ml-compute-management.adoc new file mode 100644 index 000000000..df1c8d7b7 --- /dev/null +++ b/latest/ug/ml/ml-compute-management.adoc @@ -0,0 +1,20 @@ +include::../attributes.txt[] + +[.topic] +[#ml-compute-management] += Manage compute resources for AI/ML workloads on Amazon EKS +:info_titleabbrev: Capacity management + +[abstract] +-- +Learn how to manage compute resources for machine learning workloads in Amazon EKS. +-- + +This section is designed to help you manage compute resources for machine learning workloads in Amazon Elastic Kubernetes Service (EKS). You'll find details on reserving GPUs using Capacity Blocks for managed node groups and self-managed nodes, including prerequisites, launch template setup, scaling configurations, workload preparation, and key considerations for handling reservation lifecycles and graceful node termination. + +[.topiclist] +[[Topic List]] + +include::capacity-blocks-mng.adoc[leveloffset=+1] + +include::capacity-blocks.adoc[leveloffset=+1] diff --git a/latest/ug/ml/ml-eks-k8s-device-plugin.adoc b/latest/ug/ml/ml-eks-k8s-device-plugin.adoc new file mode 100644 index 000000000..3999e9a4f --- /dev/null +++ b/latest/ug/ml/ml-eks-k8s-device-plugin.adoc @@ -0,0 +1,267 @@ +include::../attributes.txt[] + +[.topic] +[#ml-eks-k8s-device-plugin] += Install Kubernetes device plugin for GPUs +:info_titleabbrev: Install device plugin for GPUs + +Kubernetes https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/device-plugins/[device plugins] have been the primary mechanism for advertising specialized infrastructure such as GPUs, network interfaces, and network adaptors as consumable resources for Kubernetes workloads. While https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/[Dynamic Resource Allocation] (DRA) is positioned as the future for device management in Kubernetes, most specialized infrastructure providers are early in their support for DRA drivers. Kubernetes device plugins remain a widely available approach for using GPUs in Kubernetes clusters today. + +== Considerations + +* When using the EKS-optimized AL2023 AMIs with NVIDIA GPUs, you must install the https://github.com/NVIDIA/k8s-device-plugin[NVIDIA Kubernetes device plugin]. You can install and manage the NVIDIA Kubernetes device plugin with Helm, your choice of Kubernetes tooling, or the NVIDIA GPU operator. +* When using the EKS-optimized Bottlerocket AMIs with NVIDIA GPUs, you do not need to install the NVIDIA Kubernetes device plugin, as it is already included in the EKS-optimized Bottlerocket AMIs. This includes when you use GPU instances with EKS Auto Mode. +* When using the EKS-optimized AL2023 or Bottlerocket AMIs with {aws} Inferentia or Trainium GPUs, then you must install the Neuron Kubernetes device plugin, and optionally install the https://awsdocs-neuron.readthedocs-hosted.com/en/latest/containers/tutorials/k8s-neuron-scheduler.html[Neuron Kubernetes scheduler extension]. For more information, see the https://awsdocs-neuron.readthedocs-hosted.com/en/latest/containers/kubernetes-getting-started.html[Neuron documentation for running on EKS]. + +[#eks-nvidia-device-plugin] +== Install NVIDIA Kubernetes device plugin + +The following procedure describes how to install the NVIDIA Kubernetes device plugin and run a sample test on NVIDIA GPU instances. + +=== Prerequisites + +* EKS cluster created +* NVIDIA GPU nodes running in the cluster using EKS-optimized AL2023 NVIDIA AMI +* Helm installed in your command-line environment, see <>. + +=== Procedure + +. Add the `nvdp` Helm chart repository. ++ +[source,bash] +---- +helm repo add nvdp https://nvidia.github.io/k8s-device-plugin +---- ++ +. Update your local Helm repository to make sure that you have the most recent charts. ++ +[source,bash] +---- +helm repo update +---- ++ +. Get the latest version of the NVIDIA Kubernetes device plugin ++ +[source,bash] +---- +helm search repo nvdp --devel +---- ++ +[source,bash] +---- +NAME CHART VERSION APP VERSION DESCRIPTION +nvdp/gpu-feature-discovery 0.17.4 0.17.4 ... +nvdp/nvidia-device-plugin 0.17.4 0.17.4 ... +---- ++ +. Install the NVIDIA Kubernetes device plugin on your cluster, replacing `0.17.4` with the latest version from the command above. ++ +[source,bash,subs="verbatim,attributes,quotes"] +---- +helm install nvdp nvdp/nvidia-device-plugin \ + --namespace nvidia \ + --create-namespace \ + --version [.replaceable]`0.17.4` \ + --set gfd.enabled=true +---- ++ +. Verify the NVIDIA Kubernetes device plugin is running in your cluster. The output below shows the output with two nodes in the cluster. ++ +[source,bash] +---- +kubectl get ds -n nvidia nvdp-nvidia-device-plugin +---- ++ +[source, bash] +---- +NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE +nvdp-nvidia-device-plugin 2 2 2 2 2 11m +---- ++ +. Verify that your nodes have allocatable GPUs with the following command. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get nodes "-o=custom-columns=NAME:.metadata.name,GPU:.status.allocatable.nvidia\.com/gpu" +---- ++ +[source,bash] +---- +NAME GPU +ip-192-168-11-225.us-west-2.compute.internal 1 +ip-192-168-24-96.us-west-2.compute.internal 1 +---- ++ +. Create a file named `nvidia-smi.yaml` with the following contents. This manifest launches a https://docs.aws.amazon.com/linux/al2023/ug/minimal-container.html[minimal AL2023 container image] that runs `nvidia-smi` on a node. ++ +[source,yaml,subs="verbatim,attributes,quotes"] +---- +apiVersion: v1 +kind: Pod +metadata: + name: nvidia-smi +spec: + restartPolicy: OnFailure + containers: + - name: gpu-demo + image: public.ecr.aws/amazonlinux/amazonlinux:2023-minimal + command: ['/bin/sh', '-c'] + args: ['nvidia-smi && tail -f /dev/null'] + resources: + limits: + nvidia.com/gpu: 1 + tolerations: + - key: 'nvidia.com/gpu' + operator: 'Equal' + value: 'true' + effect: 'NoSchedule' +---- ++ +. Apply the manifest with the following command. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl apply -f nvidia-smi.yaml +---- +. After the Pod has finished running, view its logs with the following command. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl logs nvidia-smi +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- ++-----------------------------------------------------------------------------------------+ +| NVIDIA-SMI XXX.XXX.XX Driver Version: XXX.XXX.XX CUDA Version: XX.X | +|-----------------------------------------+------------------------+----------------------+ +| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC | +| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. | +| | | MIG M. | +|=========================================+========================+======================| +| 0 NVIDIA L4 On | 00000000:31:00.0 Off | 0 | +| N/A 27C P8 11W / 72W | 0MiB / 23034MiB | 0% Default | +| | | N/A | ++-----------------------------------------+------------------------+----------------------+ + ++-----------------------------------------------------------------------------------------+ +| Processes: | +| GPU GI CI PID Type Process name GPU Memory | +| ID ID Usage | +|=========================================================================================| +| No running processes found | ++-----------------------------------------------------------------------------------------+ +---- + +[#eks-neuron-device-plugin] +== Install Neuron Kubernetes device plugin + +The following procedure describes how to install the Neuron Kubernetes device plugin and run a sample test on an Inferentia instance. + +=== Prerequisites + +* EKS cluster created +* Neuron GPU nodes running in the cluster using EKS-optimized AL2023 Neuron AMI or Bottlerocket AMI +* Helm installed in your command-line environment, see <>. + +=== Procedure + +. Install the Neuron Kubernetes device plugin on your cluster. ++ +[source,bash] +---- +helm upgrade --install neuron-helm-chart oci://public.ecr.aws/neuron/neuron-helm-chart \ + --set "npd.enabled=false" +---- ++ +. Verify the Neuron Kubernetes device plugin is running in your cluster. The output below shows the output with a single Neuron node in the cluster. ++ +[source,bash] +---- +kubectl get ds -n kube-system neuron-device-plugin +---- ++ +[source, bash] +---- +NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE +neuron-device-plugin 1 1 1 1 1 72s +---- ++ +. Verify that your nodes have allocatable NueronCores with the following command. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get nodes "-o=custom-columns=NAME:.metadata.name,NeuronCore:.status.allocatable.aws\.amazon\.com/neuroncore" +---- ++ +[source,bash] +---- +NAME NeuronCore +ip-192-168-47-173.us-west-2.compute.internal 2 +---- ++ +. Verify that your nodes have allocatable NueronDevices with the following command. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get nodes "-o=custom-columns=NAME:.metadata.name,NeuronDevice:.status.allocatable.aws\.amazon\.com/neuron" +---- ++ +[source,bash] +---- +NAME NeuronDevice +ip-192-168-47-173.us-west-2.compute.internal 1 +---- ++ +. Create a file named `neuron-ls.yaml` with the following contents. This manifest launches an https://awsdocs-neuron.readthedocs-hosted.com/en/latest/tools/neuron-sys-tools/neuron-monitor-user-guide.html[Neuron Monitor] container that has the `neuron-ls` tool installed. ++ +[source,yaml] +---- +apiVersion: v1 +kind: Pod +metadata: + name: neuron-ls +spec: + restartPolicy: Never + containers: + - name: neuron-container + image: public.ecr.aws/g4h4h0b5/neuron-monitor:1.0.0 + command: ["/bin/sh"] + args: ["-c", "neuron-ls"] + resources: + limits: + aws.amazon.com/neuron: 1 + tolerations: + - key: "aws.amazon.com/neuron" + operator: "Exists" + effect: "NoSchedule" +---- ++ +. Apply the manifest with the following command. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl apply -f neuron-ls.yaml +---- +. After the Pod has finished running, view its logs with the following command. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl logs neuron-ls +---- ++ +An example output is below. ++ +[source,bash,subs="verbatim,attributes"] +---- +instance-type: inf2.xlarge +instance-id: ... ++--------+--------+--------+---------+ +| NEURON | NEURON | NEURON | PCI | +| DEVICE | CORES | MEMORY | BDF | ++--------+--------+--------+---------+ +| 0 | 2 | 32 GB | 00:1f.0 | ++--------+--------+--------+---------+ +---- \ No newline at end of file diff --git a/latest/ug/ml/ml-eks-optimized-ami.adoc b/latest/ug/ml/ml-eks-optimized-ami.adoc index 7e08f399c..5f0060883 100644 --- a/latest/ug/ml/ml-eks-optimized-ami.adoc +++ b/latest/ug/ml/ml-eks-optimized-ami.adoc @@ -1,85 +1,138 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[ml-eks-optimized-ami,ml-eks-optimized-ami.title]] -= Run GPU-accelerated containers (Linux on EC2) -:info_titleabbrev: Run Linux GPU AMIs +[#ml-eks-optimized-ami] += Use EKS-optimized accelerated AMIs for GPU instances +:info_titleabbrev: Use EKS Linux GPU AMIs -include::../attributes.txt[] +Amazon EKS supports EKS-optimized Amazon Linux and Bottlerocket AMIs for GPU instances. The EKS-optimized accelerated AMIs simplify running AI and ML workloads in EKS clusters by providing pre-built, validated operating system images for the accelerated Kubernetes stack. In addition to the core Kubernetes components that are included in the standard EKS-optimized AMIs, the EKS-optimized accelerated AMIs include the kernel modules and drivers required to run the NVIDIA GPU `G` and `P` EC2 instances, and the {aws} GPU link:machine-learning/inferentia/[Inferentia,type="marketing"] and link:machine-learning/trainium/[Trainium,type="marketing"] EC2 instances in EKS clusters. -The Amazon EKS optimized accelerated Amazon Linux AMIs are built on top of the standard Amazon EKS optimized Amazon Linux AMIs. For details on these AMIs, see <>. -The following text describes how to enable {aws} Neuron-based workloads. +The table below shows the supported GPU instance types for each EKS-optimized accelerated AMI variant. See the EKS-optimized https://github.com/awslabs/amazon-eks-ami/releases[AL2023 releases] and https://github.com/bottlerocket-os/bottlerocket/blob/develop/CHANGELOG.md[Bottlerocket releases] on GitHub for the latest updates to the AMI variants. -.To enable {aws} Neuron (ML accelerator) based workloads -For details on training and inference workloads using [.noloc]`Neuron` in Amazon EKS, see the following references: +[%header,cols="2,4"] +|=== +|EKS AMI variant | EC2 instance types -* https://awsdocs-neuron.readthedocs-hosted.com/en/latest/containers/kubernetes-getting-started.html[Containers - Kubernetes - Getting Started] in the _{aws} [.noloc]`Neuron` Documentation_ -* https://github.com/aws-neuron/aws-neuron-eks-samples/blob/master/README.md#training[Training] in {aws} [.noloc]`Neuron` EKS Samples on GitHub -* <> +|AL2023 x86_64 NVIDIA +|p6-b200, p5, p5e, p5en, p4d, p4de, p3, p3dn, gr6, g6, g6e, g6f, gr6f, g5, g4dn -The following procedure describes how to run a workload on a GPU based instance with the Amazon EKS optimized accelerated AMIs. +|AL2023 ARM NVIDIA +|p6e-gb200, g5g -. After your GPU nodes join your cluster, you must apply the https://github.com/NVIDIA/k8s-device-plugin[NVIDIA device plugin for Kubernetes] as a [.noloc]`DaemonSet` on your cluster. Replace [.replaceable]`vX.X.X` with your desired https://github.com/NVIDIA/k8s-device-plugin/releases[NVIDIA/k8s-device-plugin] version before running the following command. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl apply -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/vX.X.X/deployments/static/nvidia-device-plugin.yml ----- -. You can verify that your nodes have allocatable GPUs with the following command. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl get nodes "-o=custom-columns=NAME:.metadata.name,GPU:.status.allocatable.nvidia\.com/gpu" ----- -. Create a file named `nvidia-smi.yaml` with the following contents. Replace [.replaceable]`tag` with your desired tag for https://hub.docker.com/r/nvidia/cuda/tags[nvidia/cuda]. This manifest launches an https://developer.nvidia.com/cuda-zone[NVIDIA CUDA] container that runs `nvidia-smi` on a node. -+ -[source,yaml,subs="verbatim,attributes"] ----- -apiVersion: v1 -kind: Pod -metadata: - name: nvidia-smi -spec: - restartPolicy: OnFailure - containers: - - name: nvidia-smi - image: nvidia/cuda:tag - args: - - "nvidia-smi" - resources: - limits: - nvidia.com/gpu: 1 ----- -. Apply the manifest with the following command. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl apply -f nvidia-smi.yaml ----- -. After the [.noloc]`Pod` has finished running, view its logs with the following command. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl logs nvidia-smi ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] +|AL2023 x86_64 Neuron +|inf1, inf2, trn1, trn2 + +|Bottlerocket x86_64 aws-k8s-nvidia +|p6-b200, p5, p5e, p5en, p4d, p4de, p3, p3dn, gr6, g6, g6e, g6f, gr6f, g5, g4dn + +|Bottlerocket aarch64/arm64 aws-k8s-nvidia +|g5g + +|Bottlerocket x86_64 aws-k8s +|inf1, inf2, trn1, trn2 +|=== + +[#eks-amis-nvidia] +== EKS-optimized NVIDIA AMIs + +By using the EKS-optimized NVIDIA AMIs, you agree to https://s3.amazonaws.com/EULA/NVidiaEULAforAWS.pdf[NVIDIA's Cloud End User License Agreement (EULA)]. + +To find the latest EKS-optimized NVIDIA AMIs, see <> and <>. + +When using Amazon Elastic Fabric Adaptor (EFA) with the EKS-optimized AL2023 or Bottlerocket NVIDIA AMIs, you must install the EFA device plugin separately. For more information, see <>. + +[#eks-amis-nvidia-al2023] +== EKS AL2023 NVIDIA AMIs + +When using the https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/overview.html[NVIDIA GPU operator] with the EKS-optimized AL2023 NVIDIA AMIs, you must disable the operator installation of the driver and toolkit, as these are already included in the EKS AMIs. The EKS-optimized AL2023 NVIDIA AMIs do not include the NVIDIA Kubernetes device plugin or the NVIDIA DRA driver, and these must be installed separately. For more information, see <>. + +In addition to the standard EKS AMI components, the EKS-optimized AL2023 NVIDIA AMIs include the following components. + +* NVIDIA driver +* NVIDIA CUDA user mode driver +* NVIDIA container toolkit +* NVIDIA fabric manager +* NVIDIA persistenced +* NVIDIA IMEX driver +* NVIDIA NVLink Subnet Manager +* EFA minimal (kernel module and rdma-core) + +For details on the NVIDIA CUDA user mode driver and the CUDA runtime/libraries used within application containers, see the https://docs.nvidia.com/deploy/cuda-compatibility/why-cuda-compatibility.html#why-cuda-compatibility[NVIDIA documentation]. The CUDA version shown from `nvidia-smi` is the version of the NVIDIA CUDA user mode driver installed on the host, which must be compatible with the CUDA runtime/libraries used in application containers. + +To track the status of the EKS-optimized NVIDIA AMIs upgrade to NVIDIA driver 580 version, see https://github.com/awslabs/amazon-eks-ami/issues/2470[GitHub issue #2470]. The NVIDIA 580 driver is required to use CUDA 13+. + +See the EKS AL2023 NVIDIA AMI https://github.com/awslabs/amazon-eks-ami/blob/main/templates/al2023/provisioners/install-nvidia-driver.sh[installation script] and https://github.com/awslabs/amazon-eks-ami/blob/main/templates/al2023/runtime/gpu/nvidia-kmod-load.sh[kernel loading script] for details on how the EKS AMIs configure the NVIDIA dependencies. See the EKS-optimized https://github.com/awslabs/amazon-eks-ami/releases[AL2023 releases] on GitHub to see the component versions included in the AMIs. You can find the list of installed packages and their versions on a running EC2 instance with the `dnf list installed` command. + +When building custom AMIs with the EKS-optimized AMIs as the base, it is not recommended or supported to run an operating system upgrade (ie. `dnf upgrade`) or upgrade any of the Kubernetes or GPU packages that are included in the EKS-optimized AMIs, as this risks breaking component compatibility. If you do upgrade the operating system or packages that are included in the EKS-optimized AMIs, it is recommended to thoroughly test in a development or staging environment before deploying to production. + +When building custom AMIs for GPU instances, it is recommended to build separate custom AMIs for each instance type generation and family that you will run. The EKS-optimized accelerated AMIs selectively install drivers and packages at runtime based on the underlying instance type generation and family. For more information, see the EKS AMI scripts for https://github.com/awslabs/amazon-eks-ami/blob/main/templates/al2023/provisioners/install-nvidia-driver.sh[installation] and https://github.com/awslabs/amazon-eks-ami/blob/main/templates/al2023/runtime/gpu/nvidia-kmod-load.sh[runtime]. + +[#eks-amis-nvidia-bottlerocket] +== EKS Bottlerocket NVIDIA AMIs + +When using the https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/overview.html[NVIDIA GPU operator] with the EKS-optimized Bottlerocket NVIDIA AMIs, you must disable the operator installation of the driver, toolkit, and device plugin as these are already included in the EKS AMIs. + +In addition to the standard EKS AMI components, the EKS-optimized Bottlerocket NVIDIA AMIs include the following components. The minimal dependencies for EFA (kernel module and rdma-core) are installed in all Bottlerocket variants. + +* NVIDIA Kubernetes device plugin +* NVIDIA driver +* NVIDIA CUDA user mode driver +* NVIDIA container toolkit +* NVIDIA fabric manager +* NVIDIA persistenced +* NVIDIA IMEX driver +* NVIDIA NVLink Subnet Manager +* NVIDIA MIG manager + +For details on the NVIDIA CUDA user mode driver and the CUDA runtime/libraries used within application containers, see the https://docs.nvidia.com/deploy/cuda-compatibility/why-cuda-compatibility.html#why-cuda-compatibility[NVIDIA documentation]. The CUDA version shown from `nvidia-smi` is the version of the NVIDIA CUDA user mode driver installed on the host, which must be compatible with the CUDA runtime/libraries used in application containers. + +See the Bottlerocket Version Information in the https://bottlerocket.dev/en/[Bottlerocket documentation] for details on the installed packages and their versions. The EKS-optimized Bottlerocket NVIDIA AMIs support kernel 6.12 and NVIDIA driver 580 version for Kubernetes versions 1.33 and above. The NVIDIA 580 driver is required to use CUDA 13+. + +[#eks-amis-neuron] +== EKS-optimized Neuron AMIs + +For details on how to run training and inference workloads using Neuron with Amazon EKS, see the following references: + +* https://awsdocs-neuron.readthedocs-hosted.com/en/latest/containers/kubernetes-getting-started.html[Containers - Kubernetes - Getting Started] in the {aws} Neuron Documentation +* https://github.com/aws-neuron/aws-neuron-eks-samples/blob/master/README.md#training[Training example] in {aws} Neuron EKS Samples on GitHub +* <> + +To find the latest EKS-optimized Neuron AMIs, see <> and <>. + +When using Amazon Elastic Fabric Adaptor (EFA) with the EKS-optimized AL2023 or Bottlerocket Neuron AMIs, you must install the EFA device plugin separately. For more information, see <>. + +[#eks-amis-neuron-al2023] +== EKS AL2023 Neuron AMIs + +The EKS-optimized AL2023 Neuron AMIs do not include the Neuron Kubernetes device plugin or the https://awsdocs-neuron.readthedocs-hosted.com/en/latest/containers/tutorials/k8s-neuron-scheduler.html[Neuron Kubernetes scheduler extension], and these must be installed separately. For more information, see <>. + +In addition to the standard EKS AMI components, the EKS-optimized AL2023 Neuron AMIs include the following components. + +* Neuron driver (aws-neuronx-dkms) +* Neuron tools (aws-neuronx-tools) +* EFA minimal (kernel module and rdma-core) + +See the EKS AL2023 Neuron AMI https://github.com/awslabs/amazon-eks-ami/blob/main/templates/al2023/provisioners/install-neuron-driver.sh[installation script] for details on how the EKS AMIs configure the Neuron dependencies. See the EKS-optimized https://github.com/awslabs/amazon-eks-ami/releases[AL2023 releases] on GitHub to see the component versions included in the AMIs. You can find the list of installed packages and their versions on a running EC2 instance with the `dnf list installed` command. + +[#eks-amis-neuron-bottlerocket] +== EKS Bottlerocket Neuron AMIs + +The standard Bottlerocket variants (aws-k8s) include the Neuron dependencies that are automatically detected and loaded when running on {aws} Inferentia or Trainium EC2 instances. + +The EKS-optimized Bottlerocket AMIs do not include the Neuron Kubernetes device plugin or the https://awsdocs-neuron.readthedocs-hosted.com/en/latest/containers/tutorials/k8s-neuron-scheduler.html[Neuron Kubernetes scheduler extension], and these must be installed separately. For more information, see <>. + +In addition to the standard EKS AMI components, the EKS-optimized Bottlerocket Neuron AMIs include the following components. + +* Neuron driver (aws-neuronx-dkms) +* EFA minimal (kernel module and rdma-core) + +When using the EKS-optimized Bottlerocket AMIs with Neuron instances, the following must be configured in the Bottlerocket user-data. This setting allows the container to take ownership of the mounted Neuron device based on the `runAsUser` and `runAsGroup` values provided in the workload specification. For more information on Neuron support in Bottlerocket, see the https://github.com/bottlerocket-os/bottlerocket/blob/develop/QUICKSTART-EKS.md#neuron-support[Quickstart on EKS readme] on GitHub. + +[source,toml] ---- -Mon Aug 6 20:23:31 20XX -+-----------------------------------------------------------------------------+ -| NVIDIA-SMI XXX.XX Driver Version: XXX.XX | -|-------------------------------+----------------------+----------------------+ -| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | -| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | -|===============================+======================+======================| -| 0 Tesla V100-SXM2... On | 00000000:00:1C.0 Off | 0 | -| N/A 46C P0 47W / 300W | 0MiB / 16160MiB | 0% Default | -+-------------------------------+----------------------+----------------------+ -+-----------------------------------------------------------------------------+ -| Processes: GPU Memory | -| GPU PID Type Process name Usage | -|=============================================================================| -| No running processes found | -+-----------------------------------------------------------------------------+ +[settings] +[settings.kubernetes] +device-ownership-from-security-context = true ---- + +See the https://github.com/bottlerocket-os/bottlerocket-kernel-kit/blob/develop/CHANGELOG.md[Bottlerocket kernel kit changelog] for information on the Neuron driver version included in the EKS-optimized Bottlerocket AMIs. \ No newline at end of file diff --git a/latest/ug/ml/ml-eks-windows-optimized-ami.adoc b/latest/ug/ml/ml-eks-windows-optimized-ami.adoc index 9dcf0425b..6b1e46213 100644 --- a/latest/ug/ml/ml-eks-windows-optimized-ami.adoc +++ b/latest/ug/ml/ml-eks-windows-optimized-ami.adoc @@ -1,10 +1,9 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[ml-eks-windows-optimized-ami,ml-eks-windows-optimized-ami.title]] +[#ml-eks-windows-optimized-ami] = Run GPU-accelerated containers (Windows on EC2 G-Series) -:info_titleabbrev: Run Windows GPU AMIs - -include::../attributes.txt[] +:info_titleabbrev: Set up Windows GPU AMIs [IMPORTANT] ==== @@ -15,9 +14,9 @@ Learn how to run GPU-accelerated Windows container workloads on Amazon EKS (Elas There are two main approaches to setting up GPU-acceleration for your Windows containers: -* **Option 1**: <> with the required GPU drivers pre-installed. +* *Option 1*: <> with the required GPU drivers pre-installed. ** Use this approach when you need a consistent, pre-configured environment ready to run GPU-accelerated Windows containers, and you're able to invest the additional effort to build and maintain the custom AMI. -* **Option 2**: Install the necessary GPU drivers on your EKS worker nodes after launching your instance. +* *Option 2*: Install the necessary GPU drivers on your EKS worker nodes after launching your instance. ** Use this approach when you want a simpler setup process and don't mind installing the GPU drivers on each new worker node. More suited to a development environment when you are evaluating or prototyping GPU-accelerated workloads. Both approaches can be leveraged using the steps detailed in this guide. @@ -31,7 +30,7 @@ This guide provides steps to install and set up GPU-acceleration for your Window * There are some known limitations to be aware of before running GPU-accelerated Windows containers. Please see the <> section for more information. -[[ml-eks-windows-ami-prerequisites,ml-eks-windows-ami-prerequisites.title]] +[#ml-eks-windows-ami-prerequisites] == Prerequisites To enable GPU acceleration for your Windows containers on Amazon EKS, you'll need to prepare the following requirements before proceeding: @@ -41,35 +40,35 @@ To enable GPU acceleration for your Windows containers on Amazon EKS, you'll nee * Provision Windows nodes in the G-family of instance types, such as link:ec2/instance-types/g4/[G4,type="marketing"] or link:ec2/instance-types/g5/[G5,type="marketing"]. * Provision Windows nodes with a container runtime with containerd `1.7.x` or `2.x.x`. (See <> to verify the containerd version in your Amazon EKS Optimized AMI.) -[[ml-eks-windows-ami-install-gpu-driver,ml-eks-windows-ami-install-gpu-driver.title]] +[#ml-eks-windows-ami-install-gpu-driver] == Install the GPU driver on each Windows Windows node To install the NVIDIA GRID drivers on your EKS worker nodes, follow the steps outlined in link:AWSEC2/latest/UserGuide/install-nvidia-driver.html[NVIDIA drivers for your Amazon EC2 instance,type="documentation"]. Navigate to link:AWSEC2/latest/UserGuide/install-nvidia-driver#nvidia-GRID-driver[Installation options - Option 3: GRID drivers,type="documentation"] and follow the installation steps. -**Install for Windows Server Core** +*Install for Windows Server Core* For Windows Server Core, which doesn’t have a desktop experience, install NVIDIA GRID drivers silently by using the following commands: -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- $nvidiaInstallerFilePath = nvidia-driver-installer.exe # Replace with path to installer $installerArguments = "-s -clean -noreboot -noeula" Start-Process -FilePath $nvidiaInstallerFilePath -ArgumentList $installerArguments -Wait -NoNewWindow -PassThru ---- -**Verify your installation** +*Verify your installation* Run the following PowerShell command to show diagnostic information about the GPUs on the instance: -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- nvidia-smi ---- This command displays the NVIDIA driver version, as well as information about the GPU hardware. Ensure that the output of this command matches the NVIDIA GRID driver version you expected to be installed. -[[ml-eks-windows-ami-deploy-gpu-driver,ml-eks-windows-ami-deploy-gpu-driver.title]] +[#ml-eks-windows-ami-deploy-gpu-driver] == Deploy the GPU device plugin on each node To enable discovery and exposure of the GPU resources to containers on your Windows nodes, you will need a device plugin. @@ -86,49 +85,49 @@ The device plugin DaemonSet will run on every node as a host process container w When running GPU-accelerated containers, the device plugin supports two modes: -* **Single-tenancy mode**: This mode dedicates all GPU resources to a single container on the instance. Install the device plugins with single-tenancy support using the following command. See README.md for more information. +* *Single-tenancy mode*: This mode dedicates all GPU resources to a single container on the instance. Install the device plugins with single-tenancy support using the following command. See README.md for more information. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- kubectl apply -f "https://raw.githubusercontent.com/TensorWorks/directx-device-plugins/main/deployments/default-daemonsets.yml" ---- -* **Multi-tenancy mode**: This mode allows sharing GPU resources among multiple containers on the instance. Install the device plugins with multi-tenancy support using the following command. See README.md for more information. +* *Multi-tenancy mode*: This mode allows sharing GPU resources among multiple containers on the instance. Install the device plugins with multi-tenancy support using the following command. See README.md for more information. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- kubectl apply -f "https://raw.githubusercontent.com/TensorWorks/directx-device-plugins/main/deployments/multitenancy-inline.yml" ---- + Alternatively, use a ConfigMap to specify the multi-tenancy. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- kubectl apply -f "https://raw.githubusercontent.com/TensorWorks/directx-device-plugins/main/deployments/multitenancy-configmap.yml" ---- -[[ml-eks-windows-ami-verify-device-plugin,ml-eks-windows-ami-verify-device-plugin.title]] +[#ml-eks-windows-ami-verify-device-plugin] === Verifying the device plugin deployment -After you have deployed the device plugin, run the following command to verify the DirectX Device Plugin is running correctly on your all your Windows nodes. -[source,bash,subs="verbatim,attributes,quotes"] +After you have deployed the device plugin, replace `` and run the following command to verify the DirectX Device Plugin is running correctly on your all your Windows nodes. +[source,bash,subs="verbatim,attributes"] ---- -kubectl get ds device-plugin-wddm -n [.replaceable]`` +kubectl get ds device-plugin-wddm -n ---- -[[ml-eks-windows-ami-verify-container-deployment,ml-eks-windows-ami-verify-container-deployment.title]] +[#ml-eks-windows-ami-verify-container-deployment] === Verifying containers are ready for deployment Once the device plugin DaemonSet is running on the GPU-powered Windows worker nodes, use the following command to verify that each node has allocatable GPUs. The corresponding number should match the number of DirectX devices on each node. -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- kubectl get nodes "-o=custom-columns=NAME:.metadata.name,DirectX:.status.allocatable.directx\.microsoft\.com/display" ---- -[[ml-eks-windows-ami-run-with-gpu-acceleration,ml-eks-windows-ami-run-with-gpu-acceleration.title]] +[#ml-eks-windows-ami-run-with-gpu-acceleration] == Running Windows containers with GPU-acceleration Before launching your pods, specify the resource name `directx.microsoft.com/display` in `.spec.containers[].resources`. @@ -137,7 +136,7 @@ This will indicate that your containers require GPU-enabled capabilities, and th As an example, see the sample command below which launches a `Job` to run Monte Carlo simulation to estimate the value of pi. This example is from the https://github.com/TensorWorks/DirectX-Device-Plugins[Kubernetes Device Plugins for DirectX] GitHub repository, which has https://github.com/TensorWorks/DirectX-Device-Plugins/tree/main/examples[multiple examples] to choose from that you can run to test your Windows node GPU capabilities. -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- cat < - -[.topic] -[[ml-get-started,ml-get-started.title]] -= Get started with ML -:info_doctype: section -:info_title: Get started deploying Machine Learning tools on EKS -:info_titleabbrev: Get started with ML -:info_abstract: Choose the Machine Learning on EKS tools and platforms that best suit your needs, then use quick start procedures to deploy them to the {aws} cloud. - -include::../attributes.txt[] - - -[abstract] --- -Choose the Machine Learning on EKS tools and platforms that best suit your needs, then use quick start procedures to deploy ML workloads and EKS clusters to the {aws} cloud. --- - -To jump into Machine Learning on EKS, start by choosing from these prescriptive patterns to quickly get an EKS cluster and ML software and hardware ready to begin running ML workloads. Most of these patterns are based on Terraform blueprints that are available from the https://awslabs.github.io/data-on-eks/docs/introduction/intro[Data on Amazon EKS] site. Before you begin, here are few things to keep in mind: - -* GPUs or Neuron instances are required to run these procedures. Lack of availability of these resources can cause these procedures to fail during cluster creation or node autoscaling. -* Neuron SDK (Tranium and Inferentia-based instances) can save money and are more available than NVIDIA GPUs. So, when your workloads permit it, we recommend that you consider using Neutron for your Machine Learning workloads (see https://awsdocs-neuron.readthedocs-hosted.com/en/latest/[Welcome to {aws} Neuron]). -* Some of the getting started experiences here require that you get data via your own https://huggingface.co/[Hugging Face] account. - -To get started, choose from the following selection of patterns that are designed to get you started setting up infrastructure to run your Machine Learning workloads: - -* *https://awslabs.github.io/data-on-eks/docs/blueprints/ai-ml/jupyterhub[JupyterHub on EKS]*: Explore the https://awslabs.github.io/data-on-eks/docs/blueprints/ai-ml/jupyterhub[JupyterHub blueprint], which showcases Time Slicing and MIG features, as well as multi-tenant configurations with profiles. This is ideal for deploying large-scale JupyterHub platforms on EKS. -* *https://aws.amazon.com/ai/machine-learning/neuron/[Large Language Models on {aws} Neuron and RayServe]*: Use https://aws.amazon.com/ai/machine-learning/neuron/[{aws} Neuron] to run large language models (LLMs) on Amazon EKS and {aws} Trainium and {aws} Inferentia accelerators. See https://awslabs.github.io/data-on-eks/docs/gen-ai/inference/Neuron/vllm-ray-inf2[Serving LLMs with RayServe and vLLM on {aws} Neuron] for instructions on setting up a platform for making inference requests, with components that include: -+ -** {aws} Neuron SDK toolkit for deep learning -** {aws} Inferentia and Trainium accelerators -** vLLM - variable-length language model (see the https://docs.vllm.ai/en/latest/[vLLM] documentation site) -** RayServe scalable model serving library (see the https://docs.ray.io/en/latest/serve/index.html[Ray Serve: Scalable and Programmable Serving] site) -** Llama-3 language model, using your own https://huggingface.co/[Hugging Face] account. -** Observability with {aws} CloudWatch and Neuron Monitor -** Open WebUI -* *https://awslabs.github.io/data-on-eks/docs/gen-ai/inference/GPUs/vLLM-NVIDIATritonServer[Large Language Models on NVIDIA and Triton]*: Deploy multiple large language models (LLMs) on Amazon EKS and NVIDIA GPUs. See https://awslabs.github.io/data-on-eks/docs/gen-ai/inference/GPUs/vLLM-NVIDIATritonServer[Deploying Multiple Large Language Models with NVIDIA Triton Server and vLLM] for instructions for setting up a platform for making inference requests, with components that include: -+ -** NVIDIA Triton Inference Server (see the https://github.com/triton-inference-server/server[Triton Inference Server] GitHub site) -** vLLM - variable-length language model (see the https://docs.vllm.ai/en/latest/[vLLM] documentation site) -** Two language models: mistralai/Mistral-7B-Instruct-v0.2 and meta-llama/Llama-2-7b-chat-hf, using your own https://huggingface.co/[Hugging Face] account. - -== Continuing with ML on EKS - -Along with choosing from the blueprints described on this page, there are other ways you can proceed through the ML on EKS documentation if you prefer. For example, you can: - -* *Try tutorials for ML on EKS* – Run other end-to-end tutorials for building and running your own Machine Learning models on EKS. See <>. - -To improve your work with ML on EKS, refer to the following: - -* *Prepare for ML* – Learn how to prepare for ML on EKS with features like custom AMIs and GPU reservations. See <>. diff --git a/latest/ug/ml/ml-prepare-for-cluster.adoc b/latest/ug/ml/ml-prepare-for-cluster.adoc deleted file mode 100644 index e813b68b5..000000000 --- a/latest/ug/ml/ml-prepare-for-cluster.adoc +++ /dev/null @@ -1,48 +0,0 @@ -//!!NODE_ROOT
- -[.topic] -[[ml-prepare-for-cluster,ml-prepare-for-cluster.title]] -= Prepare for ML clusters -:info_doctype: section -:info_title: Prepare to create an EKS cluster for Machine Learning -:info_titleabbrev: Prepare for ML -:info_abstract: Learn how to make decisions about CPU, AMIs, and tooling before creating an EKS cluster for ML. - -include::../attributes.txt[] - - -[abstract] --- -Learn how to make decisions about CPU, AMIs, and tooling before creating an EKS cluster for ML. --- - -There are ways that you can enhance your Machine Learning on EKS experience. -Following pages in this section will help you: - -* Understand your choices for using ML on EKS and -* Help in preparation of your EKS and ML environment. - -In particular, this will help you: - -* *Choose AMIs*: {aws} offers multiple customized AMIs for running ML workloads on EKS. See <> and <>. -* *Customize AMIs*: You can further modify {aws} custom AMIs to add other software and drivers needed for your particular use cases. See <>. -* *Reserve GPUs*: Because of the demand for GPUs, to ensure that the GPUs you need are available when you need them, you can reserve the GPUs you need in advance. See <>. -* *Add EFA*: Add the Elastic Fabric Adapter to improve network performance for inter-node cluster communications. See <>. -* *Use AWSInferentia workloads*: Create an EKS cluster with Amazon EC2 Inf1 instances. See <>. - -[.topiclist] -[[Topic List]] - -include::ml-eks-optimized-ami.adoc[leveloffset=+1] - -include::ml-eks-windows-optimized-ami.adoc[leveloffset=+1] - -include::capacity-blocks-mng.adoc[leveloffset=+1] - -include::capacity-blocks.adoc[leveloffset=+1] - -include::node-taints-managed-node-groups.adoc[leveloffset=+1] - -include::node-efa.adoc[leveloffset=+1] - -include::inferentia-support.adoc[leveloffset=+1] diff --git a/latest/ug/ml/ml-realtime-inference-cluster.adoc b/latest/ug/ml/ml-realtime-inference-cluster.adoc new file mode 100644 index 000000000..ed0fd2fec --- /dev/null +++ b/latest/ug/ml/ml-realtime-inference-cluster.adoc @@ -0,0 +1,1312 @@ +include::../attributes.txt[] + +[.topic] +[#ml-realtime-inference-cluster] += Best Practices Cluster Setup Guide for Real-Time Inference on Amazon EKS +:info_titleabbrev: Create cluster + +[abstract] +-- +Learn how to set up an Amazon EKS cluster optimized for real-time online inference workloads using GPU-accelerated nodes, Karpenter for autoscaling, and integrate {aws} services to serve a model. +-- + +== Introduction + +This guide offers a hands-on walkthrough for setting up an Amazon Elastic Kubernetes Service (EKS) cluster optimized for real-time online inference workloads, incorporating best practices curated by {aws} experts throughout. It uses an opinionated EKS Quickstart Architecture—a curated set of drivers, instance types, and configurations aligned with {aws} best practices for models, accelerators, and scaling. This approach helps you bypass the task of selecting cluster settings, allowing you to get a functional, pre-configured cluster up and running quickly. Along the way, we'll deploy sample workloads to validate your setup, explain key architectural concepts (such as decoupling CPU-bound tasks from GPU-intensive computations), address common questions (e.g., why choose Bottlerocket AMI over AL2023?), and outline next steps to extend your cluster's capabilities. + +Designed specifically for Machine Learning (ML) and Artificial Intelligence (AI) engineers, platform administrators, operators, and data/AI specialists who are new to the {aws} and EKS ecosystem, this guide assumes familiarity with Kubernetes but no prior EKS experience. It is designed to help you understand the steps and processes needed to get real-time online inference workloads up and running. The guide shows you the essentials of creating a single-node inference cluster, including provisioning GPU resources, integrating storage for model artifacts, enabling secure {aws} service access, and exposing inference endpoints. Throughout, it emphasizes low-latency, resilient design for user-facing applications like fraud detection, real-time chatbots, and sentiment analysis in customer feedback systems. + +In this guide, we focus exclusively on setting up a foundational, prescriptive starting point using G5 EC2 instances. If you're seeking {aws} Inferentia-specific cluster configurations or end-to-end workflows, see <> or our workshops in <>. + +== Before you begin + +Before you start, make sure you have performed the following tasks: + +* link:eks/latest/userguide/setting-up.html[Setup your environment for Amazon EKS,type="documentation"] +* https://eksctl.io/installation/[Install the latest version of eksctl] +* https://helm.sh/docs/intro/install/[Install Helm] +* https://docs.docker.com/get-started/[(Optional) Install Docker] +* https://org.ngc.nvidia.com/setup/installers/cli[(Optional) Install the NGC CLI] + +== Architecture + +Real-time online inference refers to the process of using a trained machine learning model to generate predictions or outputs on incoming data streams with minimal latency. For example, it enables real-time fraud detection, classification of images, or the generation of knowledge graphs in response to user inputs. The architecture of a real-time online inference system delivers low-latency machine learning predictions in user-facing applications by decoupling CPU-bound web traffic handling from GPU-intensive AI computations. This process typically lives within a larger application ecosystem, and often includes backend, frontend, vector, and model services, with a focus on specialized components to enable independent scaling, parallel development, and resilience against failures. Isolating inference tasks on dedicated GPU hardware and leveraging interfaces like APIs and WebSockets ensures high concurrency, fast processing of models like transformers, and user interactions through the frontend. Note that although vector databases and Retrieval Augmented Generation (RAG) pipelines often play a big part in real-time inference systems, these components are not covered in this guide. At a minimum, a typical architecture often includes: + +* *Frontend Service*: Serves as the user-facing interface, handling client-side logic, rendering dynamic content, and facilitating real-time interactions, it communicates with the backend service to initiate inference requests and display results, often initiating requests to the backend service which uses WebSockets for streaming updates or APIs for structured data exchange. This service typically does not require a dedicated load balancer, as it can be hosted on content delivery networks (CDNs) like {aws} CloudFront for static assets or served directly from web servers, with scaling handled via auto-scaling groups if needed for dynamic content. +* *Backend Service*: Acts as the application's orchestrator, managing business logic such as user authentication, data validation, and service coordination (e.g., via APIs for RESTful endpoints or WebSockets for persistent connections). It communicates with the inference service, scales independently on multi-core CPUs and RAM to handle high web traffic without relying on GPUs, and often requires a load balancer (such as {aws} Application Load Balancer or Network Load Balancer) to distribute incoming requests across multiple instances, especially in high-concurrency scenarios. An ingress controller can further manage external access and routing rules for enhanced security and traffic shaping. +* *Inference Service*: Serves as the core for AI computations, running on GPUs with sufficient VRAM (e.g., 8-12 GB for models like DistilBERT) to perform vector embeddings, knowledge extraction, and model inference (e.g., exposed through APIs for batch requests or WebSockets for real-time streaming) using custom or open-source models. This isolation prevents dependency conflicts, allows model updates without downtime, and enables horizontal scaling with load balancing for multiple concurrent requests. To expose the model service effectively, it typically sits behind a load balancer to distribute GPU-bound workloads across replicated instances, while an ingress resource or controller (such as ALB Ingress Controller in {aws}) handles external routing, SSL termination, and path-based forwarding to ensure secure and efficient access without overwhelming individual GPUs. + +== Solution Overview + +Real-time online inference systems require a high-performance, resilient architecture that can deliver ultra-low latency while handling unpredictable, high volume traffic bursts. This solution overview explains how the following {aws} components work together in the Amazon EKS cluster we will create to ensure our cluster is able to host and manage machine learning models that provide immediate predictions on live data with minimal delay for end-users. + +* https://aws.amazon.com/ec2/instance-types/g5/[Amazon G5 EC2 Instances] — For GPU-intensive inference tasks, we are using g5.xlarge and g5.2xlarge G5 EC2 instance types, which feature a single (1) NVIDIA A10G GPU with 24GB of memory (e.g., 8 billion parameters at FP16). Based on the NVIDIA Ampere Architecture, these GPUs are powered by NVIDIA A10G Tensor Core GPUs and 2nd generation AMD EPYC processors, support 4-8 vCPUs, up to 10 Gbps network bandwidth, and 250-450 GB of local NVMe SSD storage, ensuring fast data movement and compute power for complex models, making them ideal for low-latency, high-throughput inference tasks. Choosing an EC2 instance type is application-specific, depends on your model (e.g., image, video, text model), and your latency and throughput requirements. For instance, if using an image and or video model, you may want to use https://aws.amazon.com/ec2/instance-types/p5/[P5 EC2 instances] for optimal, real-time latency. We recommend starting out with https://aws.amazon.com/ec2/instance-types/g5/[G5 EC2 instances] as it provides a good starting point for getting up and running quickly, then evaluating whether it's the right fit for your workloads through performance benchmark testing. For more advanced cases, consider https://aws.amazon.com/ec2/instance-types/g6/[G6 EC2 instances]. +* https://aws.amazon.com/ec2/instance-types/m7g/[Amazon EC2 M7g Instances] — For CPU-intensive tasks like data preprocessing, API request handling, hosting the Karpenter controller, add-ons, and other system components, we are using the m5.xlarge M7g EC2 instance type. M7g instances are ARM-based instance which features 4 vCPUs, 16 GB of memory, up to 12.5 Gbps network bandwidth, and is powered by {aws} Graviton3 processors. Choosing an EC2 instance type is application-specific and depends on your workload's compute, memory, and scalability requirements. For compute-optimized workloads, you might consider https://aws.amazon.com/ec2/instance-types/c7g/[C7g EC2 instances], which also use Graviton3 processors but are optimized for higher compute performance than M7g instances for certain use cases. Alternatively, newer https://aws.amazon.com/ec2/instance-types/c8g/[C8g EC2 instances] (where available) provide up to 30% better compute performance than C7g instances. We recommend starting out with M7g EC2 instances for their cost efficiency and compatibility with a wide range of workloads (e.g., application servers, microservices, gaming servers, mid-size data stores), then evaluating whether it's the right fit for your workloads through performance benchmark testing. +* link:eks/latest/userguide/s3-csi.html[Amazon S3 Mountpoint CSI Driver,type="documentation"] — For workloads on single-GPU instances where multiple pods share a GPU (e.g., multiple pods scheduled on the same node to utilize its GPU resources), we are using the Mountpoint S3 CSI Driver to optimize memory usage—essential for tasks like large-model inference in cost-sensitive, low-complexity setups. It exposes Amazon S3 buckets as a POSIX-like file system available to the Kubernetes cluster, which allows inference pods to read model artifacts (e.g., model weights) directly into memory without having to download them first, and input datasets using standard file operations. Additionally, S3 has virtually unlimited storage capacity and accelerates data-intensive inference workloads. Choosing a storage CSI driver is application-specific, and depends on your workload's throughput and latency requirements. Though the link:eks/latest/userguide/fsx-openzfs-csi.html[FSx for OpenZFS CSI Driver,type="documentation"] offers sub-millisecond latency for random I/O or fully POSIX-compliant shared persistent volumes across nodes, we recommend starting out with the Mountpoint S3 CSI Driver due to its scalability, lower costs for large datasets, and built-in integration with S3-managed object storage for read-heavy inference patterns (e.g., streaming model inputs), then evaluating whether it's the right fit for your workloads through performance benchmark testing. +* link:eks/latest/userguide/pod-identities.html[EKS Pod Identity Agent,type="documentation"] — To enable access to {aws} services, we are using the EKS Pod Identity Agent, which uses a single service principal and facilitates pod-level IAM role associations within the Amazon EKS cluster. EKS Pod Identity offers a streamlined alternative to the traditional link:eks/latest/userguide/iam-roles-for-service-accounts.html[IAM Roles for Service Accounts (IRSA),type="documentation"] approach by utilizing a single service principal (pods.eks.amazonaws.com) instead of relying on individual OIDC providers for each cluster, which makes it easier to assign permissions. Additionally, it enables roles to be reused across multiple clusters and it supports advanced features like link:eks/latest/userguide/pod-id-abac.html[IAM role session tags,type="documentation"] and link:eks/latest/userguide/pod-id-assign-target-role.html[Target IAM roles,type="documentation"]. +* link:eks/latest/userguide/node-health.html[EKS Node Monitoring Agent,type="documentation"] — To ensure continuous availability and reliability of inference services, we are using the EKS Node Monitoring Agent with Auto Repair, which automatically detects and replaces unhealthy nodes, minimizing downtime. It continuously monitors nodes for hardware, kernel, networking, and storage issues using enhanced health checks (e.g., KernelReady, NetworkingReady). For GPU nodes, it detects accelerator-specific failures, initiating graceful remediation by cordoning unhealthy nodes, waiting 10 minutes for transient GPU issues to resolve, and replacing nodes after 30 minutes for persistent failures. +* link:eks/latest/userguide/eks-optimized-ami-bottlerocket.html[Bottlerocket AMI,type="documentation"] — To provide a security-hardened foundation for our EKS cluster, we are using the Bottlerocket AMI, which includes only the essential components required to run containers and offers minimal boot times for fast scaling. Choosing a node AMI is application-specific and depends on your workload's customization, security, and scalability requirements. Though the link:eks/latest/userguide/eks-optimized-ami.html[AL2023 AMI,type="documentation"] provides greater flexibility for host-level installations and customizations (e.g., specifying a dedicated cache directory in a PV/PVC without any additional node configurations), we recommend starting out with the Bottlerocket AMI for its smaller footprint and built-in optimization for containerized workloads (e.g., microservices, inference servers, scalable APIs), then evaluating whether it's the right fit for your workloads through performance benchmark testing. +* link:eks/latest/userguide/lbc-helm.html[{aws} Load Balancer Controller (LBC),type="documentation"] — To expose real-time inference endpoints, we are using the {aws} Load Balancer Controller, which automatically provisions and manages Application Load Balancers (ALBs) for HTTP/HTTPS traffic and Network Load Balancers (NLBs) for TCP/UDP traffic based on Kubernetes Ingress and Service resources, enabling the integration of inference models with external clients. Additionally, it supports features like path-based routing to distribute inference requests across multiple pods or nodes, ensuring scalability during traffic spikes and minimizing latency through {aws}-native optimizations like connection multiplexing and health checks. + +== 1. Create your EKS cluster + +In this step, we create a cluster with CPU nodes and a managed node group using an {aws} CloudFormation-powered eksctl https://eksctl.io/usage/creating-and-managing-clusters/[ClusterConfig] template. Initializing the cluster with only CPU nodes allows us to use Karpenter exclusively to manage CPU-intensive and GPU nodes for optimized resource allocation using Karpenter NodePools which we create in later steps. To support our real-time inference workloads, we provision the cluster with the EKS Bottlerocket AMI, EKS Node Monitoring Agent, EKS Pod Identity Agent, Mountpoint S3 CSI Driver, {aws} Load Balancer Controller (LBC), and link:eks/latest/userguide/managing-kube-proxy.html[kube-proxy,type="documentation"], link:eks/latest/userguide/managing-vpc-cni.html[vpc-cni,type="documentation"], and link:eks/latest/userguide/managing-coredns.html[coredns,type="documentation"] drivers. The m7g.xlarge instances will be used for CPU system tasks, including hosting the Karpenter controller, add-ons, and other system components. + +By default, `eksctl` will create a dedicated VPC for the cluster with a CIDR block of `192.168.0.0/16`. The VPC includes three public subnets and three private subnets, each distributed across three different Availability Zones (or two AZs in the `us-east-1` region) which is the ideal method for deploying Kubernetes workloads. The template also deploys an internet gateway, providing internet access to the public subnets through default routes in their route tables and a single NAT gateway in one of the public subnets, with default routes in the private subnets' route tables directing outbound traffic through the NAT gateway for internet access. To learn more about this setup, see https://docs.aws.amazon.com/eks/latest/best-practices/subnets.html#_deploy_nodes_to_private_subnets[Deploy Nodes to Private Subnets]. + +=== Check your credentials + +Check whether your {aws} CLI credentials are valid and can authenticate with {aws} services: + +[source,bash] +---- +aws sts get-caller-identity +---- +If successful, the CLI will return details about your {aws} identity (UserId, Account, and Arn). + +=== Check instance availability + +G5 instance types are not available in all regions. Check your nearest region. For example: + +[source,bash] +---- +aws ec2 describe-instance-types --instance-types g5.xlarge g5.2xlarge --region us-east-1 +---- +If successful, the G5 instance type is available in the region you specified. + +The Bottlerocket AMI is not available in all regions. Check by retrieving a Bottlerocket AMI ID for your nearest region. For example: + +[source,bash] +---- +aws ssm get-parameter --name /aws/service/bottlerocket/aws-k8s-1.33/arm64/latest/image_id \ + --region us-east-1 --query "Parameter.Value" --output text +---- +If successful, the Bottlerocket AMI is available in the region you specified. + +=== Prepare your environment + +First, set the following environment variables in a new terminal window. *Note*: Be sure to substitute the sample placeholders with your unique values, including cluster name, your desired region, https://github.com/kubernetes-sigs/karpenter/releases[Karpenter release version], and link:eks/latest/userguide/kubernetes-versions.html[Kubernetes version,type="documentation"]. + +[TIP] +==== +Some variables (such as `${AWS_REGION}` and `${K8S_VERSION}`) are defined early in the block and then referenced in later commands for consistency and to avoid repetition. Make sure to run the commands in sequence so that these values are properly exported and available for use in subsequent definitions. +==== + +[source,bash] +---- +export TEMPOUT="$(mktemp)" +export K8S_VERSION=1.33 +export KARPENTER_VERSION="1.5.0" +export AWS_REGION="us-east-1" +export EKS_CLUSTER_NAME="eks-rt-inference-${AWS_REGION}" +export S3_BUCKET_NAME="eks-rt-inference-models-${AWS_REGION}-$(date +%s)" +export NVIDIA_BOTTLEROCKET_AMI="$(aws ssm get-parameter --name /aws/service/bottlerocket/aws-k8s-${K8S_VERSION}-nvidia/x86_64/latest/image_id --query Parameter.Value --output text)" +export STANDARD_BOTTLEROCKET_AMI="$(aws ssm get-parameter --name /aws/service/bottlerocket/aws-k8s-${K8S_VERSION}/arm64/latest/image_id --query Parameter.Value --output text)" +export AWS_ACCOUNT_ID="$(aws sts get-caller-identity --query Account --output text)" +export ALIAS_VERSION="$(aws ssm get-parameter --name "/aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2023/x86_64/standard/recommended/image_id" --query Parameter.Value | xargs aws ec2 describe-images --query 'Images[0].Name' --image-ids | sed -r 's/^.*(v[[:digit:]]+).*$/\1/')" +---- + +=== Create required roles and policies + +Karpenter needs specific IAM roles and policies (e.g., Karpenter controller IAM role, instance profile, and policies) to manage EC2 instances as Kubernetes worker nodes. It uses these roles to perform actions like launching and terminating EC2 instances, tagging resources, and interacting with other {aws} services. Create the Karpenter roles and policies using the Karpenter's https://raw.githubusercontent.com/aws/karpenter-provider-aws/v1.5.0/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml[cloudformation.yaml]: + +[source,bash] +---- +curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/v${KARPENTER_VERSION}/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml > "${TEMPOUT}" \ +&& aws cloudformation deploy \ + --stack-name "Karpenter-${EKS_CLUSTER_NAME}" \ + --template-file "${TEMPOUT}" \ + --capabilities CAPABILITY_NAMED_IAM \ + --parameter-overrides "ClusterName=${EKS_CLUSTER_NAME}" +---- + +The {aws} LBC needs permission to provision and manage {aws} load balancers, such as creating ALBs for Ingress resources or NLBs for services of type `LoadBalancer`. We'll specify this permissions policy during cluster creation. During cluster creation, we will create the service account with eksctl in the ClusterConfig. Create the LBC IAM policy: + +[source,bash] +---- +aws iam create-policy \ + --policy-name AWSLoadBalancerControllerIAMPolicy \ + --policy-document "$(curl -fsSL https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.14.1/docs/install/iam_policy.json)" +---- + +When the Mountpoint S3 CSI Driver is installed, its DaemonSet pods are configured to use a service account for execution. The Mountpoint for Mountpoint S3 CSI driver needs permission to interact with the Amazon S3 bucket you create later in this guide. We'll specify this permissions policy during cluster creation. During cluster creation, we will create the service account with eksctl in the ClusterConfig. Create the S3 IAM policy: + +[source,bash] +---- +aws iam create-policy \ + --policy-name S3CSIDriverPolicy \ + --policy-document "{\"Version\": \"2012-10-17\", \"Statement\": [{\"Effect\": \"Allow\", \"Action\": [\"s3:GetObject\", \"s3:PutObject\", \"s3:AbortMultipartUpload\", \"s3:DeleteObject\", \"s3:ListBucket\"], \"Resource\": [\"arn:aws:s3:::${S3_BUCKET_NAME}\", \"arn:aws:s3:::${S3_BUCKET_NAME}/*\"]}]}" +---- + +*Note*: if a role already exists with this name, give the role a different name. The role we create in this step is specific to your cluster and your S3 bucket. + +=== Create the cluster + +In this template, eksctl automatically creates a Kubernetes service account for EKS Pod Identity, Node Monitoring Agent, CoreDNS, Kubeproxy, the VPC CNI Plugin. As of today, the Mountpoint S3 CSI Driver is not available for EKS Pod Identity, so we create an IAM Roles for Service Account (IRSA) and an link:eks/latest/userguide/enable-iam-roles-for-service-accounts.html[OIDC endpoint,type="documentation"]. In addition, we create a service account for the {aws} Load Balancer Controller (LBC). For access to Bottlerocket nodes, eksctl automatically attaches AmazonSSMManagedInstanceCore for Bottlerocket to allow secure shell sessions via SSM. + +In the same terminal where you set your environment variables, run the following command block to create the cluster: + +[source,bash] +---- +eksctl create cluster -f - < 12m v1.33.0-eks-802817d m7g.xlarge +ip-192-168-7-15.ec2.internal Ready 12m v1.33.0-eks-802817d m7g.xlarge +---- + +Verify all the Pod Identity associations and how they map a role to a service account in a namespace in the cluster with the following command: + +[source,bash] +---- +eksctl get podidentityassociation --cluster ${EKS_CLUSTER_NAME} --region ${AWS_REGION} +---- + +The output should show the IAM roles for Karpenter ("karpenter") and the {aws} LBC ("aws-load-balancer-controller"). + +Verify the DaemonSets are available: + +[source,bash] +---- +kubectl get daemonsets -n kube-system +---- + +The expected output should look like this: + +[source] +---- +NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE +aws-node 3 3 3 3 3 12m +dcgm-server 0 0 0 0 0 kubernetes.io/os=linux 12m +eks-node-monitoring-agent 3 3 3 3 3 kubernetes.io/os=linux 12m +eks-pod-identity-agent 3 3 3 3 3 12m +kube-proxy 3 3 3 3 3 12m +s3-csi-node 2 2 2 2 2 kubernetes.io/os=linux 12m +---- + +Verify all addons are installed on the cluster: + +[source,bash] +---- +eksctl get addons --cluster ${EKS_CLUSTER_NAME} --region ${AWS_REGION} +---- + +The expected output should look like this: + +[source] +---- +NAME VERSION STATUS ISSUES IAMROLE UPDATE AVAILABLE CONFIGURATION VALUES POD IDENTITY ASSOCIATION ROLES +aws-mountpoint-s3-csi-driver v1.15.0-eksbuild.1 ACTIVE 0 arn:aws:iam::143095308808:role/eksctl-eks-rt-inference-us-east-1-addon-aws-m-Role1-RAUjk4sJnc0L +coredns v1.12.1-eksbuild.2 ACTIVE 0 +eks-node-monitoring-agent v1.3.0-eksbuild.2 ACTIVE 0 +eks-pod-identity-agent v1.3.7-eksbuild.2 ACTIVE 0 +kube-proxy v1.33.0-eksbuild.2 ACTIVE 0 +metrics-server v0.7.2-eksbuild.3 ACTIVE 0 +vpc-cni v1.19.5-eksbuild.1 ACTIVE 0 +---- + +== 3. Install Karpenter + +Install the Karpenter controller on your CPU worker nodes (`cpu-worker`) to optimize costs and conserve GPU resources. We'll be installing it in the "kube-system" namespace and specifying the "karpenter" service account we defined during cluster creation. Additionally, this command configures the cluster name and a Spot Instance interruption queue for CPU nodes. Karpenter will use IRSA to assume this IAM role. + +[source,bash] +---- +# Logout of helm registry before pulling from public ECR +helm registry logout public.ecr.aws + +# Install Karpenter +helm upgrade --install karpenter oci://public.ecr.aws/karpenter/karpenter --version "${KARPENTER_VERSION}" --namespace "kube-system" --create-namespace \ + --set "settings.clusterName=${EKS_CLUSTER_NAME}" \ + --set "settings.interruptionQueue=${EKS_CLUSTER_NAME}" \ + --set controller.resources.requests.cpu=1 \ + --set controller.resources.requests.memory=1Gi \ + --set controller.resources.limits.cpu=1 \ + --set controller.resources.limits.memory=1Gi \ + --set serviceAccount.annotations."eks\.amazonaws\.com/role-arn"="arn:aws:iam::${AWS_ACCOUNT_ID}:role/${EKS_CLUSTER_NAME}-karpenter" \ + --wait +---- + +The expected output should look like this: + +[source] +---- +Release "karpenter" does not exist. Installing it now. +Pulled: public.ecr.aws/karpenter/karpenter:1.5.0 +Digest: sha256:9a155c7831fbff070669e58500f68d7ccdcf3f7c808dcb4c21d3885aa20c0a1c +NAME: karpenter +LAST DEPLOYED: Thu Jun 19 09:57:06 2025 +NAMESPACE: kube-system +STATUS: deployed +REVISION: 1 +TEST SUITE: None +---- + +Verify that Karpenter is running: + +[source,bash] +---- +kubectl get pods -n kube-system -l app.kubernetes.io/name=karpenter +---- + +The expected output should look like this: + +[source] +---- +NAME READY STATUS RESTARTS AGE +karpenter-555895dc-865bc 1/1 Running 0 5m58s +karpenter-555895dc-j7tk9 1/1 Running 0 5m58s +---- + +== 4. Setup Karpenter NodePools + +In this step, we configure mutually exclusive CPU and GPU https://karpenter.sh/docs/concepts/nodepools/[Karpenter NodePools]. The `limits` field in the NodePool spec constrains the maximum total resources (e.g., CPU, memory, GPUs) that each NodePool can consume across all provisioned nodes, preventing additional node provisioning if these limits are exceeded. While NodePools support broad instance categories (e.g., `c`, `g`), specifying specific https://karpenter.sh/docs/concepts/nodepools/#instance-types[instance types], https://karpenter.sh/docs/concepts/nodepools/#capacity-type[capacity types], and resource https://karpenter.sh/docs/concepts/nodepools/#speclimits[limits] help you more easily estimate costs for your on-demand workloads. In these NodePools, we use a diverse set of instance types within the G5 instance family. This allows Karpenter to automatically select the most appropriate instance type based on pod resource requests, optimizing resource utilization while respecting the NodePool's total limits. To learn more, see https://docs.aws.amazon.com/eks/latest/best-practices/karpenter.html#_creating_nodepools[Creating NodePools]. + +=== Setup the GPU NodePool + +In this NodePool, we set resource limits to manage the provisioning of nodes with GPU capabilities. These limits are designed to cap the total resources across all nodes in the pool, allowing for up to 10 instances in total. Each instance can be either g5.xlarge (4 vCPUs, 16 GiB memory, 1 GPU) or g5.2xlarge (8 vCPUs, 32 GiB memory, 1 GPU), as long as the total vCPUs do not exceed 80, total memory does not exceed 320GiB, and total GPUs do not exceed 10. For example, the pool can provision 10 g5.2xlarge instances (80 vCPUs, 320 GiB, 10 GPUs), or 10 g5.xlarge instances (40 vCPUs, 160 GiB, 10 GPUs), or a mix such as 5 g5.xlarge and 5 g5.2xlarge (60 vCPUs, 240 GiB, 10 GPUs), ensuring flexibility based on workload demands while respecting resource constraints. + +Additionally, we specify the ID of the Nvidia variant of the Bottlerocket AMI. Finally, we set a https://karpenter.sh/docs/concepts/disruption/#nodepool-disruption-budgets[disruption policy] to remove empty nodes after 30 minutes (`consolidateAfter: 30m`) and set a maximum node lifetime of 30 days (`expireAfter: 720h`) to optimize costs and maintain node health for GPU-intensive tasks. To learn more, see https://docs.aws.amazon.com/eks/latest/best-practices/aiml-compute.html#_disable_karpenter_consolidation_for_interruption_sensitive_workloads[Disable Karpenter Consolidation for interruption sensitive workloads], and https://docs.aws.amazon.com/eks/latest/best-practices/aiml-compute.html#_use_ttlsecondsafterfinished_to_auto_clean_up_kubernetes_jobs[Use ttlSecondsAfterFinished to Auto Clean-Up Kubernetes Jobs]. + +[source,bash] +---- +cat < Setup > Generate API Key > Generate Personal Key > NGC Catalog). +* https://org.ngc.nvidia.com/setup/installers/cli[Download and install the NGC CLI] (Linux/macOS/Windows) and configure the CLI using: `ngc config set`. Enter your API key when prompted; set org to `nvidia` and hit Enter to accept defaults for others. If successful, you should see something like: `Successfully saved NGC configuration to /Users/your-username/.ngc/config`. + +=== Verify service account permissions + +Before we start, check the Kubernetes service account permissions: + +[source,bash] +---- +kubectl get serviceaccount s3-csi-driver-sa -n kube-system -o yaml +---- + +During cluster creation, we attached the S3CSIDriverPolicy to an IAM role and annotated the service account ("s3-csi-driver-sa"). The Mountpoint S3 CSI driver pods inherits the IAM role's permissions when interacting with S3. The expected output should look like this: + +[source,yaml] +---- +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::143095308808:role/eksctl-eks-rt-inference-us-east-1-addon-aws-m-Role1-fpXXjRYdKN8r + creationTimestamp: "2025-07-17T03:55:29Z" + labels: + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-mountpoint-s3-csi-driver + app.kubernetes.io/managed-by: EKS + app.kubernetes.io/name: aws-mountpoint-s3-csi-driver + name: s3-csi-driver-sa + namespace: kube-system + resourceVersion: "2278" + uid: 50b36272-6716-4c68-bdc3-c4054df1177c +---- + +=== Add a toleration + +The S3 CSI Driver runs as a DaemonSet on all nodes. Pods use the CSI driver on those nodes to mount S3 volumes. To allow it to schedule on our GPU nodes which have taints, add a toleration to the DaemonSet: + +[source,bash] +---- +kubectl patch daemonset s3-csi-node -n kube-system --type='json' -p='[{"op": "add", "path": "/spec/template/spec/tolerations/-", "value": {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}}]' +---- + +The expected output should look like this: + +[source] +---- +daemonset.apps/s3-csi-node patched +---- + +=== Upload model weights to S3 + +In this step, you'll create an Amazon S3 bucket, download the GPUNet-0 model weights from NVIDIA GPU Cloud (NGC), and upload them to the bucket. These weights will be accessed by our application at runtime for inference. + +Create your Amazon S3 bucket: + +[source,bash] +---- +aws s3 mb s3://${S3_BUCKET_NAME} --region ${AWS_REGION} +---- + +Enable https://docs.aws.amazon.com/AmazonS3/latest/userguide/Versioning.html[S3 Versioning] for the bucket, to prevent accidental deletions and overwrites from causing immediate and permanent data loss: + +[source,bash] +---- +aws s3api put-bucket-versioning --bucket ${S3_BUCKET_NAME} --versioning-configuration Status=Enabled +---- + +Apply a lifecycle rule to the bucket to remove overwritten or deleted object versions 14 days after they become non-current, remove expired delete markers, and remove incomplete multi-part uploads after 7 days. To learn more, see https://docs.aws.amazon.com/AmazonS3/latest/userguide/lifecycle-configuration-examples.html[Examples of S3 Lifecycle configurations]. + +[source,bash] +---- +aws s3api put-bucket-lifecycle-configuration --bucket $S3_BUCKET_NAME --lifecycle-configuration '{"Rules":[{"ID":"LifecycleRule","Status":"Enabled","Filter":{},"Expiration":{"ExpiredObjectDeleteMarker":true},"NoncurrentVersionExpiration":{"NoncurrentDays":14},"AbortIncompleteMultipartUpload":{"DaysAfterInitiation":7}}]}' +---- + +Download the GPUNet-0 model weights from NGC. For example, on macOS: + +[source,bash] +---- +ngc registry model download-version nvidia/dle/gpunet_0_pyt_ckpt:21.12.0_amp --dest ~/downloads +---- + +[NOTE] +==== +You may need to adjust this download command for your operating system. For this command to work on a Linux system, you likely need to create the directory as part of the command (e.g., `mkdir ~/downloads`). +==== + +The expected output should look like this: + +[source,json] +---- +{ + "download_end": "2025-07-18 08:22:39", + "download_start": "2025-07-18 08:22:33", + "download_time": "6s", + "files_downloaded": 1, + "local_path": "/Users/your-username/downloads/gpunet_0_pyt_ckpt_v21.12.0_amp", + "size_downloaded": "181.85 MB", + "status": "Completed", + "transfer_id": "gpunet_0_pyt_ckpt[version=21.12.0_amp]" +} +---- + +Rename the checkpoint file to match the expected naming in our application code in later steps (no extraction is needed, as it's a standard PyTorch *.pth.tar checkpoint containing the model state dictionary): + +[source,bash] +---- +mv ~/downloads/gpunet_0_pyt_ckpt_v21.12.0_amp/0.65ms.pth.tar gpunet-0.pth +---- + +Enable the https://aws.amazon.com/blogs/storage/improving-amazon-s3-throughput-for-the-aws-cli-and-boto3-with-the-aws-common-runtime/[{aws} Common Runtime] in the {aws} CLI to optimize S3 throughput: + +[source,bash] +---- +aws configure set s3.preferred_transfer_client crt +---- + +Upload the model weights to your S3 bucket: + +[source,bash] +---- +aws s3 cp gpunet-0.pth s3://${S3_BUCKET_NAME}/gpunet-0.pth +---- + +The expected output should look like this: + +[source] +---- +upload: ./gpunet-0.pth to s3://eks-rt-inference-models-us-east-1-1752722786/gpunet-0.pth +---- + +=== Create the Model Service + +In this step, you'll set up a FastAPI web application for GPU-accelerated image classification using the GPUNet-0 vision model. The application downloads model weights from Amazon S3 at runtime, fetches the model architecture from NVIDIA's repository for caching, and downloads ImageNet class labels via HTTP. The application includes image preprocessing transforms and exposes two endpoints: a root GET for status check and a POST `/predict` endpoint that accepts an image URL. + +We serve the model using FastAPI with PyTorch, loading weights from Amazon S3 at runtime in a containerized setup for quick prototyping and Kubernetes deployment. For other methods like optimized batching or high-throughput engines, see https://docs.aws.amazon.com/eks/latest/best-practices/aiml-performance.html#_serving_ml_models[Serving ML Models]. + +==== Create the application + +Create a directory for your application files such as `model-testing`, then change directories into it and add the following code to a new file named `app.py`: + +[source,python] +---- +import os +import torch +import json +import requests +from fastapi import FastAPI, HTTPException +from PIL import Image +from io import BytesIO, StringIO +import torchvision.transforms as transforms +from torch.nn.functional import softmax +import warnings +from contextlib import redirect_stdout, redirect_stderr +import argparse +import boto3 +app = FastAPI() + +# Suppress specific warnings from the model code (quantization is optional and unused here) +warnings.simplefilter("ignore", UserWarning) + +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + +# Load model code from cache (if present) +# Use backed cache directory +torch.hub.set_dir('/cache/torch/hub') + +# Allowlist for secure deserialization (handles potential issues in older checkpoints) +torch.serialization.add_safe_globals([argparse.Namespace]) +# Load the model architecture only on container startup (changed to pretrained=False) +# Precision (FP32 for full accuracy, could be 'fp16' for speed on Ampere+ GPUs) +with redirect_stdout(StringIO()), redirect_stderr(StringIO()): + gpunet = torch.hub.load('NVIDIA/DeepLearningExamples:torchhub', 'nvidia_gpunet', pretrained=False, model_type='GPUNet-0', model_math='fp32') + +# Download weights from S3 if not present, then load them +model_path = os.getenv('MODEL_PATH', '/cache/torch/hub/checkpoints/gpunet-0.pth') +os.makedirs(os.path.dirname(model_path), exist_ok=True) # Ensure checkpoints dir exists +if not os.path.exists(model_path): + s3 = boto3.client('s3') + s3.download_file(os.getenv('S3_BUCKET_NAME'), 'gpunet-0.pth', model_path) +checkpoint = torch.load(model_path, map_location=device, weights_only=True) +gpunet.load_state_dict(checkpoint['state_dict']) +# Move to GPU/CPU +gpunet.to(device) +gpunet.eval() + +# Preprocessing +preprocess = transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), +]) + +# Load ImageNet labels +labels_url = "https://s3.amazonaws.com/deep-learning-models/image-models/imagenet_class_index.json" +response = requests.get(labels_url) +json_data = json.loads(response.text) +labels = [json_data[str(i)][1].replace('_', ' ') for i in range(1000)] + +# Required, FastAPI root +@app.get("/") +async def hello(): + return {"status": "hello"} + +# Serve model requests +@app.post("/predict") +async def predict(image_url: str): + try: + response = requests.get(image_url) + response.raise_for_status() + img = Image.open(BytesIO(response.content)).convert("RGB") + input_tensor = preprocess(img).unsqueeze(0).to(device) + + with torch.no_grad(): + output = gpunet(input_tensor) + + probs = softmax(output, dim=1)[0] + top5_idx = probs.topk(5).indices.cpu().numpy() + top5_probs = probs.topk(5).values.cpu().numpy() + + results = [{ "label": labels[idx], "probability": float(prob) } for idx, prob in zip(top5_idx, top5_probs)] + + return {"predictions": results} + except Exception as e: + raise HTTPException(status_code=400, detail=str(e)) +---- + + +==== Create the Dockerfile + +The following Dockerfile creates a container image for our application utilizing the GPUNet model from the https://github.com/NVIDIA/DeepLearningExamples[NVIDIA Deep Learning Examples for Tensor Cores] GitHub repository. + +We reduce container image size by using a runtime-only PyTorch base, installing only essential packages with cache cleanup, pre-caching model code, and avoiding "baking" weights in the container image to enable faster pulls and updates. To learn more, see https://docs.aws.amazon.com/eks/latest/best-practices/aiml-performance.html#_reducing_container_image_sizes[Reducing Container Image Sizes]. + +In the same directory as `app.py`, create the `Dockerfile`: + +[source,dockerfile] +---- +FROM pytorch/pytorch:2.4.0-cuda12.4-cudnn9-runtime + +# Install required system packages required for git cloning +RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/* + +# Install application dependencies +RUN pip install --no-cache-dir fastapi uvicorn requests pillow boto3 timm==0.5.4 + +# Pre-cache the GPUNet code from Torch Hub (without weights) +# Clone the repository containing the GPUNet code +RUN mkdir -p /cache/torch/hub && \ + cd /cache/torch/hub && \ + git clone --branch torchhub --depth 1 https://github.com/NVIDIA/DeepLearningExamples NVIDIA_DeepLearningExamples_torchhub + +COPY app.py /app/app.py + +WORKDIR /app + +CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "80"] +---- + +==== Test the application +From the same directory as your `app.py` and `Dockerfile`, build the container image for the inference application, targeting AMD64 architecture: + +[source,bash] +---- +docker build --platform linux/amd64 -t gpunet-inference-app . +---- + +Set environment variables for your {aws} credentials, and optionally an {aws} session token. For example: + +[source,bash] +---- +export AWS_REGION="us-east-1" +export AWS_ACCESS_KEY_ID=ABCEXAMPLESCUJFEIELSMUHHAZ +export AWS_SECRET_ACCESS_KEY=123EXAMPLEMZREoQXr8XkiicsOgWDQ5TpUsq0/Z +---- + +Run the container locally, injecting {aws} credentials as environment variables for S3 access. For example: + +[source,bash] +---- +docker run --platform linux/amd64 -p 8080:80 \ + -e S3_BUCKET_NAME=${S3_BUCKET_NAME} \ + -e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \ + -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \ + -e AWS_DEFAULT_REGION=${AWS_REGION} \ + gpunet-inference-app +---- + +The expected output should look like this: + +[source] +---- +INFO: Started server process [1] +INFO: Waiting for application startup. +INFO: Application startup complete. +INFO: Uvicorn running on http://0.0.0.0:80 (Press CTRL+C to quit) +---- + +In a new terminal window, test the inference endpoint by sending a sample POST request with a public image URL as a query parameter: + +[source,bash] +---- +curl -X POST "http://localhost:8080/predict?image_url=http://images.cocodataset.org/test-stuff2017/000000024309.jpg" +---- + +The expected output should be a JSON response with top-5 predictions, similar to this (actual labels and probabilities may vary slightly based on the image and model precision): + +[source,json] +---- +{"predictions":[{"label":"desk","probability":0.28885871171951294},{"label":"laptop","probability":0.24679335951805115},{"label":"notebook","probability":0.08539070934057236},{"label":"library","probability":0.030645888298749924},{"label":"monitor","probability":0.02989606373012066}]} +---- + +Quit the application using "Ctrl + C". + +=== Push the container to Amazon ECR + +In this step, we upload the container image for the GPUNet-0 model service to https://docs.aws.amazon.com/AmazonECR/latest/userguide/what-is-ecr.html[Amazon Elastic Container Registry (ECR)], making it available for deployment on Amazon EKS. This process involves creating a new ECR repository to store the image, authenticating with ECR, then tagging and pushing the container image to our registry. + +First, navigate back to the directory where you set your environment variables at the beginning of this guide. For example: + +[source,bash] +---- +cd .. +---- + +Create a repository in Amazon ECR: + +[source,bash] +---- +aws ecr create-repository --repository-name gpunet-inference-app --region ${AWS_REGION} +---- + +Log into Amazon ECR: + +[source,bash] +---- +aws ecr get-login-password --region ${AWS_REGION} | docker login --username AWS --password-stdin ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com +---- + +The expected output should look like this: + +[source] +---- +Login Succeeded +---- + +Tag the image: + +[source,bash] +---- +docker tag gpunet-inference-app:latest ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/gpunet-inference-app:latest +---- + +Push the image to your Amazon ECR repository: + +[source,bash] +---- +docker push ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/gpunet-inference-app:latest +---- + +This last step takes several minutes to complete. + +== 7. (Optional) Expose the Model Service + +In this step, you'll expose your real-time inference model service externally on Amazon EKS using the {aws} Load Balancer Controller (LBC). This involves setting up the LBC, mounting model weights from Amazon S3 as a persistent volume using the Mountpoint S3 CSI Driver, deploying a GPU-accelerated application pod, creating a service and ingress to provision an Application Load Balancer (ALB), and testing the endpoint. + +First, verify the Pod Identity association for the {aws} LBC, confirming that the service account is properly linked to the required IAM role: + +[source,bash] +---- +eksctl get podidentityassociation --cluster ${EKS_CLUSTER_NAME} --namespace kube-system --service-account-name aws-load-balancer-controller +---- + +The expected output should look like this: + +[source] +---- +ASSOCIATION ARN NAMESPACE SERVICE ACCOUNT NAME IAM ROLE ARN OWNER ARN +arn:aws:eks:us-east-1:143095308808:podidentityassociation/eks-rt-inference-us-east-1/a-buavluu2wp1jropya kube-system aws-load-balancer-controller arn:aws:iam::143095308808:role/AmazonEKSLoadBalancerControllerRole +---- + +=== Tag your cluster security group + +The {aws} Load Balancer Controller only supports a single security group with the tag key `karpenter.sh/discovery: "${EKS_CLUSTER_NAME}"` for Karpenter's security group selection. When creating a cluster with eksctl, the default cluster security group (which has the `"kubernetes.io/cluster/: owned"` tag) is not automatically tagged with `karpenter.sh/discovery` tags. This tag is essential for Karpenter to discover and attach this security group to the nodes it provisions. Attaching this security group ensures compatibility with the {aws} Load Balancer Controller (LBC), allowing it to automatically manage inbound traffic rules for services exposed via Ingress, such as the model service in these steps. + +Export the VPC ID for your cluster: + +[source,bash] +---- +CLUSTER_VPC_ID="$(aws eks describe-cluster --name ${EKS_CLUSTER_NAME} --query cluster.resourcesVpcConfig.vpcId --output text)" +---- + +Export the default security group for your cluster: + +[source,bash] +---- +CLUSTER_SG_ID="$(aws ec2 describe-security-groups --filters Name=vpc-id,Values=$CLUSTER_VPC_ID Name=tag-key,Values=kubernetes.io/cluster/${EKS_CLUSTER_NAME} --query 'SecurityGroups[].[GroupId]' --output text)" +---- + +Add the `karpenter.sh/discovery` tag to the default cluster security group. This will allow our CPU and GPU EC2NodeClass selectors to use it: + +[source,bash] +---- +aws ec2 create-tags --resources ${CLUSTER_SG_ID} --tags Key=karpenter.sh/discovery,Value=${EKS_CLUSTER_NAME} +---- + +Verify the tag was added: + +[source,bash] +---- +aws ec2 describe-security-groups --group-ids ${CLUSTER_SG_ID} --query "SecurityGroups[].Tags" +---- + +Among the results, you should see the following with the tag and your cluster name. For example: + +[source,json] +---- +{ + "Key": "karpenter.sh/discovery", + "Value": "eks-rt-inference-us-east-1" +} +---- + +=== Setup the {aws} Load Balancer Controller (LBC) + +The {aws} LBC is essential for managing ingress traffic to AI/ML workloads on Amazon EKS, ensuring access to inference endpoints or data processing pipelines. By integrating with {aws} Application Load Balancers (ALB) and Network Load Balancers (NLB), the LBC dynamically routes traffic to containerized applications, such as those running large language models, computer vision models, or real-time inference services. Since we've already created the service account and the Pod Identity Association during cluster creation, we set the `serviceAccount.name` to match what's defined in our cluster config (`aws-load-balancer-controller`). + +Add the {aws}-owned *eks-charts* Helm chart repository: + +[source,bash] +---- +helm repo add eks https://aws.github.io/eks-charts +---- + +Refresh your local Helm repositories with the most recent charts: + +[source,bash] +---- +helm repo update eks +---- + +Deploy the {aws} LBC using Helm, specifying the EKS cluster name and referencing the pre-created service account: + +[source,bash] +---- +helm install aws-load-balancer-controller eks/aws-load-balancer-controller \ + -n kube-system \ + --set clusterName=${EKS_CLUSTER_NAME} \ + --set serviceAccount.create=false \ + --set serviceAccount.name=aws-load-balancer-controller +---- + +The expected output should look like this: + +[source,bash,subs="verbatim,attributes"] +---- +NAME: aws-load-balancer-controller +LAST DEPLOYED: Wed Jul 9 15:03:31 2025 +NAMESPACE: kube-system +STATUS: deployed +REVISION: 1 +TEST SUITE: None +NOTES: +{aws} Load Balancer controller installed! +---- + +=== Mount the model in a persistent volume + +In this step, you'll mount model weights from your Amazon S3 bucket using a PersistentVolume (PV) backed by the Mountpoint for Amazon S3 CSI driver. This allows Kubernetes pods to access S3 objects as local files, eliminating resource-intensive downloads to ephemeral pod storage or init containers—ideal for large, multi-gigabyte model weights. + +The PV mounts the entire bucket root (no path specified in `volumeAttributes`), supports concurrent read-only access by multiple pods, and exposes files like the model weights (`/models/gpunet-0.pth`) inside the container for inference. This ensures the fallback "download" in our application (`app.py`) does not trigger because the file exists via the mount. By decoupling the model from the container image, this enables shared access and independent model version updates without image rebuilds. + +==== Create the PersistentVolume (PV) + +Create a PersistentVolume (PV) resource to mount the S3 bucket containing your model weights, enabling read-only access for multiple pods without downloading files at runtime: + +[source,yaml] +---- +cat < - -[.topic] -[[ml-tutorials,ml-tutorials.title]] -= Try tutorials for deploying Machine Learning workloads on EKS -:info_doctype: section -:info_title: Try tutorials for deploying Machine Learning workloads and platforms on EKS -:info_titleabbrev: Try tutorials for ML on EKS -:info_abstract: Learn how to deploy Machine Learning workloads on EKS - -include::../attributes.txt[] - -If you are interested in setting up Machine Learning platforms and frameworks in EKS, explore the tutorials described in this page. -These tutorials cover everything from patterns for making the best use of GPU processors to choosing modeling tools to building frameworks for specialized industries. - -== Build generative AI platforms on EKS - -* https://aws.amazon.com/blogs/containers/deploy-generative-ai-models-on-amazon-eks/[Deploy Generative AI Models on Amazon EKS] -* https://aws.amazon.com/blogs/containers/building-multi-tenant-jupyterhub-platforms-on-amazon-eks/[Building multi-tenant JupyterHub Platforms on Amazon EKS] -* https://aws.amazon.com/blogs/containers/run-spark-rapids-ml-workloads-with-gpus-on-amazon-emr-on-eks/[Run Spark-RAPIDS ML workloads with GPUs on Amazon EMR on EKS] - -== Run specialized generative AI frameworks on EKS - -* https://aws.amazon.com/blogs/hpc/accelerate-drug-discovery-with-nvidia-bionemo-framework-on-amazon-eks/[Accelerate drug discovery with NVIDIA BioNeMo Framework on Amazon EKS] -* https://aws.amazon.com/blogs/containers/host-the-whisper-model-with-streaming-mode-on-amazon-eks-and-ray-serve/[Host the Whisper Model with Streaming Mode on Amazon EKS and Ray Serve] -* https://aws.amazon.com/blogs/machine-learning/accelerate-your-generative-ai-distributed-training-workloads-with-the-nvidia-nemo-framework-on-amazon-eks/[Accelerate your generative AI distributed training workloads with the NVIDIA NeMo Framework on Amazon EKS] -* https://aws.amazon.com/blogs/publicsector/virtualizing-satcom-operations-aws/[Virtualizing satellite communication operations with {aws}] -* https://aws.amazon.com/blogs/opensource/running-torchserve-on-amazon-elastic-kubernetes-service/[Running TorchServe on Amazon Elastic Kubernetes Service] - -== Maximize NVIDIA GPU performance for ML on EKS - -* Implement GPU sharing to efficiently use NVIDIA GPUs for your EKS clusters: -+ -https://aws.amazon.com/blogs/containers/gpu-sharing-on-amazon-eks-with-nvidia-time-slicing-and-accelerated-ec2-instances/[GPU sharing on Amazon EKS with NVIDIA time-slicing and accelerated EC2 instances] - -* Use Multi-Instance GPUs (MIGs) and NIM microservices to run more pods per GPU on your EKS clusters: -+ -https://aws.amazon.com/blogs/containers/maximizing-gpu-utilization-with-nvidias-multi-instance-gpu-mig-on-amazon-eks-running-more-pods-per-gpu-for-enhanced-performance/[Maximizing GPU utilization with NVIDIA's Multi-Instance GPU (MIG) on Amazon EKS: Running more pods per GPU for enhanced performance] - -* Leverage NVIDIA NIM microservices to optimize inference workloads using optimized microservices to deploy AI models at scale: -+ -https://aws.amazon.com/blogs/hpc/deploying-generative-ai-applications-with-nvidia-nims-on-amazon-eks/[Part 1: Deploying generative AI applications with NVIDIA NIMs on Amazon EKS] -+ -https://aws.amazon.com/blogs/hpc/deploying-generative-ai-applications-with-nvidia-nim-microservices-on-amazon-elastic-kubernetes-service-amazon-eks-part-2/[Part 2: Deploying Generative AI Applications with NVIDIA NIM Microservices on Amazon Elastic Kubernetes Service (Amazon EKS)] - -* https://aws.amazon.com/blogs/containers/scaling-a-large-language-model-with-nvidia-nim-on-amazon-eks-with-karpenter/[Scaling a Large Language Model with NVIDIA NIM on Amazon EKS with Karpenter] - - -* https://aws.amazon.com/blogs/machine-learning/build-and-deploy-a-scalable-machine-learning-system-on-kubernetes-with-kubeflow-on-aws/[Build and deploy a scalable machine learning system on Kubernetes with Kubeflow on {aws}] - -== Run video encoding workloads on EKS - -* https://aws.amazon.com/blogs/containers/delivering-video-content-with-fractional-gpus-in-containers-on-amazon-eks/[Delivering video content with fractional GPUs in containers on Amazon EKS] - -== Accelerate image loading for inference workloads - -* https://aws.amazon.com/blogs/containers/how-h2o-ai-optimized-and-secured-their-ai-ml-infrastructure-with-karpenter-and-bottlerocket/[How H2O.ai optimized and secured their AI/ML infrastructure with Karpenter and Bottlerocket] - -== Testimonials for ML on EKS - -* https://aws.amazon.com/blogs/containers/quora-3x-faster-machine-learning-25-lower-costs-with-nvidia-triton-on-amazon-eks/[Quora achieved 3x lower latency and 25% lower Costs by modernizing model serving with Nvidia Triton on Amazon EKS] - -== Monitoring ML workloads - -* https://aws.amazon.com/blogs/mt/monitoring-gpu-workloads-on-amazon-eks-using-aws-managed-open-source-services/[Monitoring GPU workloads on Amazon EKS using {aws} managed open-source services] -* https://aws.amazon.com/blogs/machine-learning/enable-pod-based-gpu-metrics-in-amazon-cloudwatch/[Enable pod-based GPU metrics in Amazon CloudWatch] - -== Announcements for ML on EKS - -* https://aws.amazon.com/blogs/containers/bottlerocket-support-for-nvidia-gpus/[Bottlerocket support for NVIDIA GPUs] -* https://aws.amazon.com/blogs/aws/new-ec2-instances-g5-with-nvidia-a10g-tensor-core-gpus/[New – EC2 Instances (G5) with NVIDIA A10G Tensor Core GPUs] -* https://aws.amazon.com/blogs/containers/utilizing-nvidia-multi-instance-gpu-mig-in-amazon-ec2-p4d-instances-on-amazon-elastic-kubernetes-service-eks/[Utilizing NVIDIA Multi-Instance GPU (MIG) in Amazon EC2 P4d Instances on Amazon Elastic Kubernetes Service] -* https://aws.amazon.com/blogs/aws/new-gpu-equipped-ec2-p4-instances-for-machine-learning-hpc/[New – GPU-Equipped EC2 P4 Instances for Machine Learning & HPC] -* https://aws.amazon.com/blogs/machine-learning/amazon-ec2-p5e-instances-are-generally-available/[Amazon EC2 P5e instances are generally available] -* https://aws.amazon.com/blogs/containers/deploying-managed-p4d-instances-in-amazon-elastic-kubernetes-service/[Deploying managed P4d Instances in Amazon Elastic Kubernetes Service with NVIDIA GPUDirectRDMA] -* https://aws.amazon.com/blogs/machine-learning/establishing-an-ai-ml-center-of-excellence/[Establishing an AI/ML center of excellence] diff --git a/latest/ug/ml/node-efa.adoc b/latest/ug/ml/node-efa.adoc index 934678c52..ab5460cff 100644 --- a/latest/ug/ml/node-efa.adoc +++ b/latest/ug/ml/node-efa.adoc @@ -1,62 +1,55 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + :AWSEC2-latest-UserGuide-using-eni-html-network-cards: AWSEC2/latest/UserGuide/using-eni.html#network-cards [.topic] -[[node-efa,node-efa.title]] -= Run machine learning training on Amazon EKS with [.noloc]`Elastic Fabric Adapter` -:info_doctype: section -:info_title: Add Elastic Fabric \ - Adapter to EKS clusters for ML training -:info_titleabbrev: Prepare training clusters with EFA -:info_abstract: Learn how to integrate Elastic Fabric Adapter (EFA) with Amazon EKS to run machine \ - learning training workloads requiring high inter-node communications at scale using \ - p4d instances with GPUDirect RDMA and NVIDIA Collective Communications Library \ - (NCCL). - -include::../attributes.txt[] +[#node-efa] += Run machine learning training on Amazon EKS with Elastic Fabric Adapter +:info_titleabbrev: Set up training clusters with EFA [abstract] -- -Learn how to integrate Elastic Fabric Adapter (EFA) with Amazon EKS to run machine learning training workloads requiring high inter-node communications at scale using p4d instances with [.noloc]`GPUDirect RDMA` and [.noloc]`NVIDIA Collective Communications Library (NCCL)`. +Learn how to integrate Elastic Fabric Adapter (EFA) with Amazon EKS to run machine learning training workloads requiring high inter-node communications at scale using p4d instances with GPUDirect RDMA and NVIDIA Collective Communications Library (NCCL). -- -This topic describes how to integrate Elastic Fabric Adapter (EFA) with [.noloc]`Pods` deployed in your Amazon EKS cluster. Elastic Fabric Adapter (EFA) is a network interface for Amazon EC2 instances that enables you to run applications requiring high levels of inter-node communications at scale on {aws}. Its custom-built operating system bypass hardware interface enhances the performance of inter-instance communications, which is critical to scaling these applications. With EFA, High Performance Computing (HPC) applications using the Message Passing Interface (MPI) and Machine Learning (ML) applications using NVIDIA Collective Communications Library (NCCL) can scale to thousands of CPUs or GPUs. As a result, you get the application performance of on-premises HPC clusters with the on-demand elasticity and flexibility of the {aws} cloud. Integrating EFA with applications running on Amazon EKS clusters can reduce the time to complete large scale distributed training workloads without having to add additional instances to your cluster. For more information about EFA, link:hpc/efa/[Elastic Fabric Adapter,type="marketing"]. +This topic describes how to integrate Elastic Fabric Adapter (EFA) with Pods deployed in your Amazon EKS cluster. Elastic Fabric Adapter (EFA) is a network interface for Amazon EC2 instances that enables you to run applications requiring high levels of inter-node communications at scale on {aws}. Its custom-built operating system bypass hardware interface enhances the performance of inter-instance communications, which is critical to scaling these applications. With EFA, High Performance Computing (HPC) applications using the Message Passing Interface (MPI) and Machine Learning (ML) applications using NVIDIA Collective Communications Library (NCCL) can scale to thousands of CPUs or GPUs. As a result, you get the application performance of on-premises HPC clusters with the on-demand elasticity and flexibility of the {aws} cloud. Integrating EFA with applications running on Amazon EKS clusters can reduce the time to complete large scale distributed training workloads without having to add additional instances to your cluster. For more information about EFA, link:hpc/efa/[Elastic Fabric Adapter,type="marketing"]. -[[efa-instances,efa-instances.title]] +[#efa-instances] == Instance types with EFA -The _{aws} EFA Kubernetes Device Plugin_ supports all Amazon EC2 instance types that have EFA. To see a list of all instance types that have EFA, see link:AWSEC2/latest/UserGuide/efa.html#efa-instance-types[Supported instance types,type="documentation"] in the _Amazon EC2 User Guide_. However, to run ML applications quickly, we recommend that an instance has hardware acceleration chips such as [.noloc]`nVidia` GPUs, link:machine-learning/inferentia/[{aws} Inferentia,type="marketing"] chips, or link:machine-learning/trainium/[{aws} Trainium,type="marketing"] chips, in addition to the EFA. To see a list of instance types that have hardware acceleration chips and EFA, see link:AWSEC2/latest/UserGuide/efa.html#efa-instance-types[Accelerated computing,type="documentation"] in the _Amazon EC2 User Guide_. +The _{aws} EFA Kubernetes Device Plugin_ supports all Amazon EC2 instance types that have EFA. To see a list of all instance types that have EFA, see link:AWSEC2/latest/UserGuide/efa.html#efa-instance-types[Supported instance types,type="documentation"] in the _Amazon EC2 User Guide_. However, to run ML applications quickly, we recommend that an instance has hardware acceleration chips such as nVidia GPUs, link:machine-learning/inferentia/[{aws} Inferentia,type="marketing"] chips, or link:machine-learning/trainium/[{aws} Trainium,type="marketing"] chips, in addition to the EFA. To see a list of instance types that have hardware acceleration chips and EFA, see link:AWSEC2/latest/UserGuide/efa.html#efa-instance-types[Accelerated computing,type="documentation"] in the _Amazon EC2 User Guide_. As you compare instance types to choose between them, consider the number of EFA network cards available for that instance type as well as the number of accelerator cards, amount of CPU, and amount of memory. You can assign up to one EFA per network card. An EFA counts as a network interface.. To see how many EFA are available for each instance types that have EFA, see the link:AWSEC2/latest/UserGuide/using-eni.html#network-cards[Network cards,type="documentation"] list in the _Amazon EC2 User Guide_. -[[efa-only-interfaces,efa-only-interfaces.title]] +[#efa-only-interfaces] == EFA and EFA-only interfaces An _Elastic Fabric Adapter (EFA)_ is a network interface that combines the capabilities of an Elastic Network Adapter (ENA) and an OS-bypass interface, powered by the {aws} Scalable Reliable Datagram (SRD) protocol. The EFA functionalities allow applications to communicate directly with the hardware for low-latency transport. You can choose to access only the EFA capabilities using _EFA-only_ interfaces, limiting communication to interfaces within the same Availability Zone. To create nodes that can have EFA-only interfaces, you must use a custom EC2 Launch Template and set the `InterfaceType` to `efa-only`. In your custom Launch Template, you can't set the network card `0` to an EFA-only interface, as that is the primary network card and network interface of the EC2 instance. You must have VPC CNI version `1.18.5` or later for EFA-only interfaces. If you are using Amazon Linux 2, ami version has to be `v20240928` or later for EfA-only interfaces. -The following procedure guides you to create an EKS cluster with `eksctl` with nodes that have [.noloc]`nVidia` GPUs and EFA interfaces. You can't use `eksctl` to create nodes and node groups that use EFA-only interfaces. +The following procedure guides you to create an EKS cluster with `eksctl` with nodes that have nVidia GPUs and EFA interfaces. You can't use `eksctl` to create nodes and node groups that use EFA-only interfaces. -[[efa-prereqs,efa-prereqs.title]] +[#efa-prereqs] == Prerequisites * An existing Amazon EKS cluster. If you don't have an existing cluster, create one using <>.. Your cluster must be deployed in a VPC that has at least one private subnet with enough available IP addresses to deploy nodes in. The private subnet must have outbound internet access provided by an external device, such as a NAT gateway. + If you plan to use `eksctl` to create your node group, `eksctl` can also create a cluster for you. -* Version `2.12.3` or later or version `1.27.160` or later of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device or {aws} CloudShell. To check your current version, use `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or [.noloc]`Homebrew` for [.noloc]`macOS` are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the _{aws} Command Line Interface User Guide_. The {aws} CLI version that is installed in {aws} CloudShell might also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the _{aws} CloudShell User Guide_. -* The `kubectl` command line tool is installed on your device or {aws} CloudShell. The version can be the same as or up to one minor version earlier or later than the [.noloc]`Kubernetes` version of your cluster. For example, if your cluster version is `1.29`, you can use `kubectl` version `1.28`, `1.29`, or `1.30` with it. To install or upgrade `kubectl`, see <>. -* You must have the [.noloc]`Amazon VPC CNI plugin for Kubernetes` version `1.7.10` or later installed before launching worker nodes that support multiple Elastic Fabric Adapters, such as the `p4d` or `p5`. For more information about updating your [.noloc]`Amazon VPC CNI plugin for Kubernetes` version, see <>. +* Version `2.12.3` or later or version `1.27.160` or later of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device or {aws} CloudShell. To check your current version, use `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or Homebrew for macOS are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the _{aws} Command Line Interface User Guide_. The {aws} CLI version that is installed in {aws} CloudShell might also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the _{aws} CloudShell User Guide_. +* The `kubectl` command line tool is installed on your device or {aws} CloudShell. The version can be the same as or up to one minor version earlier or later than the Kubernetes version of your cluster. For example, if your cluster version is `1.29`, you can use `kubectl` version `1.28`, `1.29`, or `1.30` with it. To install or upgrade `kubectl`, see <>. +* You must have the Amazon VPC CNI plugin for Kubernetes version `1.7.10` or later installed before launching worker nodes that support multiple Elastic Fabric Adapters, such as the `p4d` or `p5`. For more information about updating your Amazon VPC CNI plugin for Kubernetes version, see <>. +* For p6-b200 instances, you must use EFA Device Plugin version v0.5.6 or later. [IMPORTANT] ==== -An important consideration required for adopting EFA with [.noloc]`Kubernetes` is configuring and managing [.noloc]`Huge Pages` as a resource in the cluster. For more information, see https://kubernetes.io/docs/tasks/manage-hugepages/scheduling-hugepages/[Manage Huge Pages] in the [.noloc]`Kubernetes` documentation. Amazon EC2 instances with the EFA driver installed pre-allocate 5128 2MiB Huge Pages, which you can request as resources to consume in your job specifications. +An important consideration required for adopting EFA with Kubernetes is configuring and managing Huge Pages as a resource in the cluster. For more information, see https://kubernetes.io/docs/tasks/manage-hugepages/scheduling-hugepages/[Manage Huge Pages] in the Kubernetes documentation. Amazon EC2 instances with the EFA driver installed pre-allocate 5128 2MiB Huge Pages, which you can request as resources to consume in your job specifications. ==== -[[efa-create-nodegroup,efa-create-nodegroup.title]] +[#efa-create-nodegroup] == Create node group The following procedure helps you create a node group with a `p4d.24xlarge` backed node group with EFA interfaces and GPUDirect RDMA, and run an example NVIDIA Collective Communications Library (NCCL) test for multi-node NCCL Performance using EFAs. The example can be used a template for distributed deep learning training on Amazon EKS using EFAs. @@ -90,7 +83,7 @@ us-west-2a us-west-2c us-west-2b Note the Availability Zones returned for use in later steps. When you deploy nodes to a cluster, your VPC must have subnets with available IP addresses in one of the Availability Zones returned in the output. . Create a node group using `eksctl`. You need version `{eksctl-min-version}` or later of the `eksctl` command line tool installed on your device or {aws} CloudShell. To install or update `eksctl`, see https://eksctl.io/installation[Installation] in the `eksctl` documentation. + -.. Copy the following contents to a file named [.replaceable]`efa-cluster.yaml`. Replace the [.replaceable]`example values` with your own. You can replace [.replaceable]`p5.48xlarge` with a different instance, but if you do, make sure that the values for `availabilityZones` are Availability Zones that were returned for the instance type in step 1. +.. Copy the following contents to a file named [.replaceable]`efa-cluster.yaml`. Replace the example values with your own. You can replace `p5.48xlarge` with a different instance, but if you do, make sure that the values for `availabilityZones` are Availability Zones that were returned for the instance type in step 1. + [source,yaml,subs="verbatim,attributes"] ---- @@ -132,31 +125,131 @@ If you don't have an existing cluster, you can run the following command to crea eksctl create cluster -f efa-cluster.yaml ---- + -NOTE: Because the instance type used in this example has GPUs, `eksctl` automatically installs the NVIDIA Kubernetes device plugin on each instance for you. -. Deploy the EFA Kubernetes device plugin. +[NOTE] +==== +Because the instance type used in this example has GPUs, `eksctl` automatically installs the NVIDIA Kubernetes device plugin on each instance for you when using Amazon Linux 2. This is not necessary for Bottlerocket, as the NVIDIA device plugin is built into Bottlerocket's EKS NVIDIA variant. When `efaEnabled` is set to `true` in the nodegroup configuration, `eksctl` will also automatically deploy the EFA device plugin on the nodes. +==== + +[#efa-bottlerocket] +=== Using Bottlerocket with EFA + +Bottlerocket AMI version 1.28.0 and later include official support for EFA. To use Bottlerocket for EFA-enabled nodes, specify `amiFamily: Bottlerocket` in your configuration. If you need to use a custom AMI ID, you must use standard `nodeGroups` instead of `managedNodeGroups`. + +Here's an example configuration: + +[source,yaml,subs="verbatim,attributes"] +---- +apiVersion: eksctl.io/v1alpha5 +kind: ClusterConfig + +metadata: + name: my-efa-bottlerocket-cluster + region: region-code + version: "1.XX" + +iam: + withOIDC: true + +availabilityZones: ["us-west-2a", "us-west-2c"] + +managedNodeGroups: + - name: my-efa-bottlerocket-ng + instanceType: p5.48xlarge + minSize: 1 + desiredCapacity: 2 + maxSize: 3 + availabilityZones: ["us-west-2a"] + volumeSize: 300 + privateNetworking: true + efaEnabled: true + amiFamily: Bottlerocket + bottlerocket: + enableAdminContainer: true + settings: + kernel: + sysctl: + "vm.nr_hugepages": "3000" # Configures 3000 * 2Mi = 6000Mi hugepages +---- + +The `vm.nr_hugepages` sysctl setting above configures the number of 2Mi hugepages. In this example, 3000 means 3000 * 2Mi = 6000Mi of hugepages. + +[#verify-efa-device-plugin] +=== Verify EFA device plugin installation + +When you create a node group with `efaEnabled: true`, `eksctl` automatically deploys the EFA Kubernetes device plugin for you. You can verify that the device plugin is installed and functioning correctly: + +. Check the DaemonSet status: ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get daemonsets -n kube-system +---- ++ +Sample output: ++ +[source,bash,subs="verbatim,attributes"] +---- +NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE +aws-efa-k8s-device-plugin-daemonset 2 2 2 2 2 6m16s +... +---- ++ +Here, the EFA device plugin DaemonSet is running on two nodes. Both are READY and AVAILABLE. + +. Next, verify the pods created by the DaemonSet: + -The EFA Kubernetes device plugin detects and advertises EFA interfaces as allocatable resources to Kubernetes. An application can consume the extended resource type `vpc.amazonaws.com/efa` in a [.noloc]`Pod` request spec just like CPU and memory. For more information, see https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#consuming-extended-resources[Consuming extended resources] in the [.noloc]`Kubernetes` documentation. Once requested, the plugin automatically assigns and mounts an EFA interface to the [.noloc]`Pod`. Using the device plugin simplifies EFA setup and does not require a [.noloc]`Pod` to run in privileged mode. +[source,bash,subs="verbatim,attributes"] +---- +kubectl get pods -n kube-system -l name=aws-efa-k8s-device-plugin +---- ++ +Sample output: + [source,bash,subs="verbatim,attributes"] ---- -helm repo add eks https://aws.github.io/eks-charts -helm install aws-efa-k8s-device-plugin --namespace kube-system eks/aws-efa-k8s-device-plugin +NAME READY STATUS RESTARTS AGE +aws-efa-k8s-device-plugin-daemonset-d68bs 1/1 Running 0 6m16s +aws-efa-k8s-device-plugin-daemonset-w4l8t 1/1 Running 0 6m16s ---- ++ +The EFA device plugin pods are in a Running state, confirming that the plugin is successfully deployed and operational. +. Verify resource registration: ++ +You can confirm that the `vpc.amazonaws.com/efa` resource is registered with the kubelet by describing the nodes: ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl describe nodes +---- ++ +If the EFA resource is properly registered, you will see it listed under the node's Capacity and Allocatable resources. For example: ++ +[source,bash,subs="verbatim,attributes"] +---- +Capacity: + ... + vpc.amazonaws.com/efa: 4 +Allocatable: + ... + vpc.amazonaws.com/efa: 4 +---- ++ +This output confirms that the node recognizes the EFA resource, making it available for pods that request it. -[[efa-application,efa-application.title]] +[#efa-application] == (Optional) Test the performance of the EFA We recommend that you test the EFA setup. You can use the https://github.com/aws-samples/awsome-distributed-training/tree/main/micro-benchmarks/nccl-tests[NCCL Tests] in the `aws-samples/awsome-distributed-training` repository on GitHub. https://github.com/NVIDIA/nccl-tests[NCCL Tests] evaluate the performance of the network using the Nvidia Collective Communication Library. The following steps submit NCCL tests on Amazon EKS. . Deploy the Kubeflow MPI Operator: + -For the NCCL tests you can apply the Kubeflow MPI Operator. The MPI Operator makes it easy to run Allreduce-style distributed training on Kubernetes. For more information, see https://github.com/kubeflow/mpi-operator[MPI Operator] on [.noloc]`GitHub`. +For the NCCL tests you can apply the Kubeflow MPI Operator. The MPI Operator makes it easy to run Allreduce-style distributed training on Kubernetes. For more information, see https://github.com/kubeflow/mpi-operator[MPI Operator] on GitHub. . Run the multi-node NCCL Performance Test to verify GPUDirectRDMA/EFA: + -To verify NCCL performance with [.noloc]`GPUDirectRDMA` over EFA, run the standard NCCL Performance test. For more information, see the official https://github.com/NVIDIA/nccl-tests.git[NCCL-Tests] repo on [.noloc]`GitHub`. +To verify NCCL performance with GPUDirectRDMA over EFA, run the standard NCCL Performance test. For more information, see the official https://github.com/NVIDIA/nccl-tests.git[NCCL-Tests] repo on GitHub. + -Complete the following steps to run a two node [.noloc]`NCCL Performance Test`. In the example [.noloc]`NCCL` test job, each worker requests eight GPUs, 5210Mi of `hugepages-2Mi`, four EFAs, and 8000Mi of memory, which effectively means each worker consumes all the resources of a `p5.48xlarge` instance. +Complete the following steps to run a two node NCCL Performance Test. In the example NCCL test job, each worker requests eight GPUs, 5210Mi of `hugepages-2Mi`, four EFAs, and 8000Mi of memory, which effectively means each worker consumes all the resources of a `p5.48xlarge` instance. + .. Create the MPIJob manifest: + @@ -286,7 +379,7 @@ mpijob.kubeflow.org/nccl-tests created ---- .. Verify that the job started pods: + -View your running [.noloc]`Pods`. +View your running Pods. + [source,bash,subs="verbatim,attributes"] ---- @@ -303,14 +396,14 @@ nccl-tests-worker-0 1/1 Running 0 2m49s nccl-tests-worker-1 1/1 Running 0 2m49s ---- + -The MPI Operator creates a launcher [.noloc]`Pod` and 2 worker [.noloc]`Pods` (one on each node). +The MPI Operator creates a launcher Pod and 2 worker Pods (one on each node). .. Verify that the job is running successfully with the logs: + -View the log for the `nccl-tests-launcher` [.noloc]`Pod`. Replace [.replaceable]`nbql9` with the value from your output. +View the log for the `nccl-tests-launcher` Pod. Replace [.replaceable]`nbql9` with the value from your output. + [source,bash,subs="verbatim,attributes"] ---- kubectl logs -f nccl-tests-launcher-nbql9 ---- -If the test completed successfully, you can deploy your applications that use the [.noloc]`Nvidia Collective Communication Library`. +If the test completed successfully, you can deploy your applications that use the Nvidia Collective Communication Library. diff --git a/latest/ug/ml/node-taints-managed-node-groups.adoc b/latest/ug/ml/node-taints-managed-node-groups.adoc index 4e3d3cf86..2c443ac82 100644 --- a/latest/ug/ml/node-taints-managed-node-groups.adoc +++ b/latest/ug/ml/node-taints-managed-node-groups.adoc @@ -1,28 +1,32 @@ -//!!NODE_ROOT
include::../attributes.txt[] + [.topic] -[[node-taints-managed-node-groups,node-taints-managed-node-groups.title]] -= Prevent [.noloc]`Pods` from being scheduled on specific nodes -:info_titleabbrev: Taint GPU nodes +[#node-taints-managed-node-groups] += Recipe: Prevent pods from being scheduled on specific nodes +:info_titleabbrev: Prevent pods from being scheduled on specific nodes [abstract] -- -Taints and tolerations work together to ensure that [.noloc]`Pods` aren't scheduled onto inappropriate nodes. This can be particularly useful for nodes running on GPU hardware. +Taints and tolerations work together to ensure that Pods aren't scheduled onto inappropriate nodes. This can be particularly useful for nodes running on GPU hardware. -- -Nodes with specialized processors, such as GPUs, can be more expensive to run than nodes running on more standard machines. -For that reason, you may want to protect those nodes from having workloads that don't require special hardware from being deployed to those nodes. -One way to do that is with taints. +== Overview + +Nodes with specialized processors, such as GPUs, can be more expensive to run than nodes on standard machines. To protect these nodes from workloads that don't require special hardware, you can use Kubernetes taints. Taints mark nodes to repel pods that don't have matching tolerations, ensuring only compatible workloads are scheduled. For more information, see https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/[Taints and Tolerations] in the Kubernetes documentation. + +Kubernetes node taints can be applied to new and existing managed node groups using the {aws-management-console} or through the Amazon EKS API. This recipe shows how to apply taints to Amazon EKS managed node groups using the {aws} CLI. For information on creating a node group with a taint using the {aws-management-console}, see <>. -Amazon EKS supports configuring [.noloc]`Kubernetes` taints through managed node groups. Taints and tolerations work together to ensure that [.noloc]`Pods` aren't scheduled onto inappropriate nodes. One or more taints can be applied to a node. This marks that the node shouldn't accept any [.noloc]`Pods` that don't tolerate the taints. Tolerations are applied to [.noloc]`Pods` and allow, but don't require, the [.noloc]`Pods` to schedule onto nodes with matching taints. For more information, see https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/[Taints and Tolerations] in the [.noloc]`Kubernetes` documentation. +== Prerequisites -[.noloc]`Kubernetes` node taints can be applied to new and existing managed node groups using the {aws-management-console} or through the Amazon EKS API. +* An link:eks/latest/userguide/getting-started.html[existing Amazon EKS cluster,type="documentation"]. +* link:eks/latest/userguide/setting-up.html[{aws} CLI installed and configured,type="documentation"] with appropriate permissions. +== Steps +=== Step 1: Create a node group with taints + +Use the `aws eks create-nodegroup` command to create a new managed node group with taints. This example applies a taint with key `dedicated`, value `gpuGroup`, and effect `NO_SCHEDULE`. -* For information on creating a node group with a taint using the {aws-management-console}, see <>. -* The following is an example of creating a node group with a taint using the {aws} CLI: -+ [source,bash,subs="verbatim,attributes"] ---- aws eks create-nodegroup \ @@ -46,20 +50,26 @@ aws eks create-nodegroup \ }' ---- -For more information and examples of usage, see https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands#taint[taint] in the [.noloc]`Kubernetes` reference documentation. +For more information and examples, see https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands#taint[taint] in the Kubernetes reference documentation. -[NOTE] -==== +=== Step 2: Update taints on an existing node group +Use the link:cli/latest/reference/eks/update-nodegroup-config.html[aws eks update-nodegroup-config,type="documentation"] {aws} CLI command to add, remove, or replace taints for managed node groups. + +[source,bash,subs="verbatim,attributes"] +---- +aws eks update-nodegroup-config + --cluster-name my-cluster + --nodegroup-name node-taints-example + --taints 'removeTaints=[{key=dedicated,value=gpuGroup,effect=NO_SCHEDULE}]' +---- + +== Notes * Taints can be updated after you create the node group using the `UpdateNodegroupConfig` API. * The taint key must begin with a letter or number. It can contain letters, numbers, hyphens (`-`), periods (`.`), and underscores (`_`). It can be up to 63 characters long. * Optionally, the taint key can begin with a DNS subdomain prefix and a single `/`. If it begins with a DNS subdomain prefix, it can be 253 characters long. * The value is optional and must begin with a letter or number. It can contain letters, numbers, hyphens (`-`), periods (`.`), and underscores (`_`). It can be up to 63 characters long. -* When using [.noloc]`Kubernetes` directly or the {aws-management-console}, the taint effect must be `NoSchedule`, `PreferNoSchedule`, or `NoExecute`. However, when using the {aws} CLI or API, the taint effect must be `NO_SCHEDULE`, `PREFER_NO_SCHEDULE`, or `NO_EXECUTE`. +* When using Kubernetes directly or the {aws-management-console}, the taint effect must be `NoSchedule`, `PreferNoSchedule`, or `NoExecute`. However, when using the {aws} CLI or API, the taint effect must be `NO_SCHEDULE`, `PREFER_NO_SCHEDULE`, or `NO_EXECUTE`. * A maximum of 50 taints are allowed per node group. * If taints that were created using a managed node group are removed manually from a node, then Amazon EKS doesn't add the taints back to the node. This is true even if the taints are specified in the managed node group configuration. - -==== - -You can use the link:cli/latest/reference/eks/update-nodegroup-config.html[aws eks update-nodegroup-config,type="documentation"] {aws} CLI command to add, remove, or replace taints for managed node groups. diff --git a/latest/ug/networking/alternate-cni-plugins.adoc b/latest/ug/networking/alternate-cni-plugins.adoc new file mode 100644 index 000000000..0a924b3a7 --- /dev/null +++ b/latest/ug/networking/alternate-cni-plugins.adoc @@ -0,0 +1,57 @@ +include::../attributes.txt[] + +[.topic] +[#alternate-cni-plugins] += Alternate CNI plugins for Amazon EKS clusters +:info_titleabbrev: Alternate CNI plugins + +[abstract] +-- +Learn how to use alternate network and security plugins on Amazon EKS to customize networking for your Kubernetes clusters on Amazon EC2 nodes. +-- + +The https://github.com/aws/amazon-vpc-cni-plugins[Amazon VPC CNI plugin for Kubernetes] is the only CNI plugin supported by Amazon EKS with Amazon EC2 nodes. Amazon EKS supports the core capabilities of Cilium and Calico for Amazon EKS Hybrid Nodes. Amazon EKS runs upstream Kubernetes, so you can install alternate compatible CNI plugins to Amazon EC2 nodes in your cluster. If you have Fargate nodes in your cluster, the Amazon VPC CNI plugin for Kubernetes is already on your Fargate nodes. It's the only CNI plugin you can use with Fargate nodes. An attempt to install an alternate CNI plugin on Fargate nodes fails. + +If you plan to use an alternate CNI plugin on Amazon EC2 nodes, we recommend that you obtain commercial support for the plugin or have the in-house expertise to troubleshoot and contribute fixes to the CNI plugin project. + +Amazon EKS maintains relationships with a network of partners that offer support for alternate compatible CNI plugins. For details about the versions, qualifications, and testing performed, see the following partner documentation. + +[%header,cols="3"] +|=== +|Partner +|Product +|Documentation + + +|Tigera +|https://www.tigera.io/partners/aws/[Calico] +|https://docs.projectcalico.org/getting-started/kubernetes/managed-public-cloud/eks[Installation instructions] + +|Isovalent +|https://cilium.io[Cilium] +|https://docs.cilium.io/en/stable/gettingstarted/k8s-install-default/[Installation instructions] + +|Juniper +|https://www.juniper.net/us/en/products/sdn-and-orchestration/contrail/cloud-native-contrail-networking.html[Cloud-Native Contrail Networking (CN2)] +|https://www.juniper.net/documentation/us/en/software/cn-cloud-native23.2/cn-cloud-native-eks-install-and-lcm/index.html[Installation instructions] + +|VMware +|https://antrea.io/[Antrea] +|https://antrea.io/docs/main/docs/eks-installation[Installation instructions] +|=== + +Amazon EKS aims to give you a wide selection of options to cover all use cases. + + +[#alternate-network-policy-plugins] +== Alternate compatible network policy plugins + +https://www.tigera.io/project-calico[Calico] is a widely adopted solution for container networking and security. Using Calico on EKS provides a fully compliant network policy enforcement for your EKS clusters. Additionally, you can opt to use Calico's networking, which conserve IP addresses from your underlying VPC. https://www.tigera.io/tigera-products/calico-cloud/[Calico Cloud] enhances the features of Calico Open Source, providing advanced security and observability capabilities. + +Traffic flow to and from Pods with associated security groups are not subjected to Calico network policy enforcement and are limited to Amazon VPC security group enforcement only. + +If you use Calico network policy enforcement, we recommend that you set the environment variable `ANNOTATE_POD_IP` to `true` to avoid a known issue with Kubernetes. To use this feature, you must add `patch` permission for pods to the `aws-node` ClusterRole. Note that adding patch permissions to the `aws-node` DaemonSet increases the security scope for the plugin. For more information, see https://github.com/aws/amazon-vpc-cni-k8s/?tab=readme-ov-file#annotate_pod_ip-v193[ANNOTATE_POD_IP] in the VPC CNI repo on GitHub. + +== Considerations for Amazon EKS Auto Mode + +Amazon EKS Auto Mode does not support alternate CNI plugins or network policy plugins. For more information, see <>. \ No newline at end of file diff --git a/latest/ug/networking/aws-load-balancer-controller.adoc b/latest/ug/networking/aws-load-balancer-controller.adoc new file mode 100644 index 000000000..70334ed94 --- /dev/null +++ b/latest/ug/networking/aws-load-balancer-controller.adoc @@ -0,0 +1,76 @@ +include::../attributes.txt[] + +[.topic] +[#aws-load-balancer-controller] += Route internet traffic with {aws} Load Balancer Controller +:info_titleabbrev: {aws} Load Balancer Controller + +[abstract] +-- +Learn how to configure and use the {aws} Load Balancer Controller to expose Kubernetes cluster apps to the internet with {aws} Elastic Load Balancing for Kubernetes services and ingresses. +-- + +The {aws} Load Balancer Controller manages {aws} Elastic Load Balancers for a Kubernetes cluster. You can use the controller to expose your cluster apps to the internet. The controller provisions {aws} load balancers that point to cluster Service or Ingress resources. In other words, the controller creates a single IP address or DNS name that points to multiple pods in your cluster. + +image::images/lbc-overview.png["Architecture diagram. Illustration of traffic coming from internet users, to Amazon Load Balancer. Amazon Load Balancer distributes traffic to pods in the cluster.",scaledwidth=50%] + +The controller watches for Kubernetes Ingress or Service resources. In response, it creates the appropriate {aws} Elastic Load Balancing resources. You can configure the specific behavior of the load balancers by applying annotations to the Kubernetes resources. For example, you can attach {aws} security groups to load balancers using annotations. + +The controller provisions the following resources: + +*Kubernetes `Ingress`*:: +The LBC creates an link:elasticloadbalancing/latest/application/introduction.html[{aws} Application Load Balancer (ALB),type="documentation"] when you create a Kubernetes `Ingress`. https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/guide/ingress/annotations/[Review the annotations you can apply to an Ingress resource.] + + +*Kubernetes service of the `LoadBalancer` type*:: +The LBC creates an link:elasticloadbalancing/latest/network/introduction.html[{aws} Network Load Balancer (NLB),type="documentation"]when you create a Kubernetes service of type `LoadBalancer`. https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/guide/service/annotations/[Review the annotations you can apply to a Service resource.] ++ +In the past, the Kubernetes network load balancer was used for _instance_ targets, but the LBC was used for _IP_ targets. With the {aws} Load Balancer Controller version `2.3.0` or later, you can create NLBs using either target type. For more information about NLB target types, see link:elasticloadbalancing/latest/network/load-balancer-target-groups.html#target-type[Target type,type="documentation"] in the User Guide for Network Load Balancers. + +The controller is an https://github.com/kubernetes-sigs/aws-load-balancer-controller[open-source project] managed on GitHub. + +Before deploying the controller, we recommend that you review the prerequisites and considerations in <> and <>. In those topics, you will deploy a sample app that includes an {aws} load balancer. + +*Kubernetes `Gateway` API*:: +With the {aws} Load Balancer Controller version `2.14.0` or later, the LBC creates an link:elasticloadbalancing/latest/application/introduction.html[{aws} Application Load Balancer (ALB),type="documentation"] when you create a Kubernetes `Gateway`. Kubernetes Gateway standardizes more configuration than Ingress, which needed custom annotations for many common options. https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/guide/gateway/gateway/[Review the configuration that you can apply to an Gateway resource.] For more information about the `Gateway` API, see link:https://kubernetes.io/docs/concepts/services-networking/gateway/[Gateway API] in the Kubernetes documentation. + +[#lbc-overview] +== Install the controller + +You can use one of the following procedures to install the {aws} Load Balancer Controller: + +* If you are new to Amazon EKS, we recommend that you use Helm for the installation because it simplifies the {aws} Load Balancer Controller installation. For more information, see <>. +* For advanced configurations, such as clusters with restricted network access to public container registries, use Kubernetes Manifests. For more information, see <>. + + +[#lbc-deprecated] +== Migrate from deprecated controller versions + +* If you have deprecated versions of the {aws} Load Balancer Controller installed, see <>. +* Deprecated versions cannot be upgraded. They must be removed and a current version of the {aws} Load Balancer Controller installed. ++ +[[lbc-deprecated-list]] +* Deprecated versions include: ++ +** {aws} ALB Ingress Controller for Kubernetes ("Ingress Controller"), a predecessor to the {aws} Load Balancer Controller. +** Any `0.1.[.replaceable]``x``` version of the {aws} Load Balancer Controller + + +[#lbc-legacy] +== Legacy cloud provider + +Kubernetes includes a legacy cloud provider for {aws}. The legacy cloud provider is capable of provisioning {aws} load balancers, similar to the {aws} Load Balancer Controller. The legacy cloud provider creates Classic Load Balancers. If you do not install the {aws} Load Balancer Controller, Kubernetes will default to using the legacy cloud provider. You should install the {aws} Load Balancer Controller and avoid using the legacy cloud provider. + +[IMPORTANT] +==== + +In versions 2.5 and newer, the {aws} Load Balancer Controller becomes the default controller for Kubernetes _service_ resources with the `type: LoadBalancer` and makes an {aws} Network Load Balancer (NLB) for each service. It does this by making a mutating webhook for services, which sets the `spec.loadBalancerClass` field to `service.k8s.aws/nlb` for new services of `type: LoadBalancer`. You can turn off this feature and revert to using the https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/guide/service/annotations/#legacy-cloud-provider[legacy Cloud Provider] as the default controller, by setting the helm chart value `enableServiceMutatorWebhook` to `false`. The cluster won't provision new Classic Load Balancers for your services unless you turn off this feature. Existing Classic Load Balancers will continue to work. + +==== + +include::lbc-helm.adoc[leveloffset=+1] + +include::lbc-manifest.adoc[leveloffset=+1] + +include::lbc-remove.adoc[leveloffset=+1] + diff --git a/latest/ug/networking/cni-custom-network-tutorial.adoc b/latest/ug/networking/cni-custom-network-tutorial.adoc new file mode 100644 index 000000000..632c45313 --- /dev/null +++ b/latest/ug/networking/cni-custom-network-tutorial.adoc @@ -0,0 +1,606 @@ +include::../attributes.txt[] + +[.topic] +[#cni-custom-network-tutorial] += Customize the secondary network interface in Amazon EKS nodes +:info_titleabbrev: Secondary interface + +[abstract] +-- +Learn how your Pods can use different security groups and subnets than the primary elastic network interface of the Amazon EC2 node that they run on. +-- + +Complete the following before you start the tutorial: + +* Review the considerations +* Familiarity with how the Amazon VPC CNI plugin for Kubernetes creates secondary network interfaces and assigns IP addresses to Pods. For more information, see https://github.com/aws/amazon-vpc-cni-k8s#eni-allocation[ENI Allocation] on GitHub. +* Version `2.12.3` or later or version `1.27.160` or later of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device or {aws} CloudShell. To check your current version, use `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or Homebrew for macOS are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the _{aws} Command Line Interface User Guide_. The {aws} CLI version that is installed in {aws} CloudShell might also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the _{aws} CloudShell User Guide_. +* The `kubectl` command line tool is installed on your device or {aws} CloudShell. To install or upgrade `kubectl`, see <>. +* We recommend that you complete the steps in this topic in a Bash shell. If you aren't using a Bash shell, some script commands such as line continuation characters and the way variables are set and used require adjustment for your shell. Additionally, the quoting and escaping rules for your shell might be different. For more information, see link:cli/latest/userguide/cli-usage-parameters-quoting-strings.html[Using quotation marks with strings in the {aws} CLI,type="documentation"] in the {aws} Command Line Interface User Guide. + +For this tutorial, we recommend using the example values, except where it's noted to replace them. You can replace any example value when completing the steps for a production cluster. We recommend completing all steps in the same terminal. This is because variables are set and used throughout the steps and won't exist in different terminals. + +The commands in this topic are formatted using the conventions listed in link:cli/latest/userguide/welcome-examples.html[Using the {aws} CLI examples,type="documentation"]. If you're running commands from the command line against resources that are in a different {aws} Region than the default {aws} Region defined in the {aws} CLI link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-profiles[profile,type="documentation"] that you're using, then you need to add `--region us-west-2` to the commands, replacing `us-west-2` with your {aws} region. + +When you want to deploy custom networking to your production cluster, skip to <>. + +[#custom-networking-create-cluster] +== Step 1: Create a test VPC and cluster + +The following procedures help you create a test VPC and cluster and configure custom networking for that cluster. We don't recommend using the test cluster for production workloads because several unrelated features that you might use on your production cluster aren't covered in this topic. For more information, see <>. + +. Run the following command to define the `account_id` variable. ++ +[source,bash,subs="verbatim,attributes"] +---- +account_id=$(aws sts get-caller-identity --query Account --output text) +---- +. Create a VPC. ++ +.. If you are deploying to a test system, create a VPC using an Amazon EKS {aws} CloudFormation template. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws cloudformation create-stack --stack-name my-eks-custom-networking-vpc \ + --template-url https://s3.us-west-2.amazonaws.com/amazon-eks/cloudformation/2020-10-29/amazon-eks-vpc-private-subnets.yaml \ + --parameters ParameterKey=VpcBlock,ParameterValue=192.168.0.0/24 \ + ParameterKey=PrivateSubnet01Block,ParameterValue=192.168.0.64/27 \ + ParameterKey=PrivateSubnet02Block,ParameterValue=192.168.0.96/27 \ + ParameterKey=PublicSubnet01Block,ParameterValue=192.168.0.0/27 \ + ParameterKey=PublicSubnet02Block,ParameterValue=192.168.0.32/27 +---- ++ +.. The {aws} CloudFormation stack takes a few minutes to create. To check on the stack's deployment status, run the following command. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws cloudformation describe-stacks --stack-name my-eks-custom-networking-vpc --query Stacks\[\].StackStatus --output text +---- ++ +Don't continue to the next step until the output of the command is `CREATE_COMPLETE`. +.. Define variables with the values of the private subnet IDs created by the template. ++ +[source,bash,subs="verbatim,attributes"] +---- +subnet_id_1=$(aws cloudformation describe-stack-resources --stack-name my-eks-custom-networking-vpc \ + --query "StackResources[?LogicalResourceId=='PrivateSubnet01'].PhysicalResourceId" --output text) +subnet_id_2=$(aws cloudformation describe-stack-resources --stack-name my-eks-custom-networking-vpc \ + --query "StackResources[?LogicalResourceId=='PrivateSubnet02'].PhysicalResourceId" --output text) +---- +.. Define variables with the Availability Zones of the subnets retrieved in the previous step. ++ +[source,bash,subs="verbatim,attributes"] +---- +az_1=$(aws ec2 describe-subnets --subnet-ids $subnet_id_1 --query 'Subnets[*].AvailabilityZone' --output text) +az_2=$(aws ec2 describe-subnets --subnet-ids $subnet_id_2 --query 'Subnets[*].AvailabilityZone' --output text) +---- +. Create a cluster IAM role. ++ +.. Run the following command to create an IAM trust policy JSON file. ++ +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:7576a0cb-6ba7-436b-b801-ff94846ef6f7[] +---- +.. Create the Amazon EKS cluster IAM role. If necessary, preface `eks-cluster-role-trust-policy.json` with the path on your computer that you wrote the file to in the previous step. The command associates the trust policy that you created in the previous step to the role. To create an IAM role, the link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] that is creating the role must be assigned the `iam:CreateRole` action (permission). ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam create-role --role-name myCustomNetworkingAmazonEKSClusterRole --assume-role-policy-document file://"eks-cluster-role-trust-policy.json" +---- +.. Attach the Amazon EKS managed policy named link:aws-managed-policy/latest/reference/AmazonEKSClusterPolicy.html#AmazonEKSClusterPolicy-json[AmazonEKSClusterPolicy,type="documentation"] to the role. To attach an IAM policy to an link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"], the principal that is attaching the policy must be assigned one of the following IAM actions (permissions): `iam:AttachUserPolicy` or `iam:AttachRolePolicy`. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam attach-role-policy --policy-arn {arn-aws}iam::aws:policy/AmazonEKSClusterPolicy --role-name myCustomNetworkingAmazonEKSClusterRole +---- +. Create an Amazon EKS cluster and configure your device to communicate with it. ++ +.. Create a cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks create-cluster --name my-custom-networking-cluster \ + --role-arn {arn-aws}iam::$account_id:role/myCustomNetworkingAmazonEKSClusterRole \ + --resources-vpc-config subnetIds="$subnet_id_1","$subnet_id_2" +---- ++ +NOTE: You might receive an error that one of the Availability Zones in your request doesn't have sufficient capacity to create an Amazon EKS cluster. If this happens, the error output contains the Availability Zones that can support a new cluster. Retry creating your cluster with at least two subnets that are located in the supported Availability Zones for your account. For more information, see <>. +.. The cluster takes several minutes to create. To check on the cluster's deployment status, run the following command. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks describe-cluster --name my-custom-networking-cluster --query cluster.status +---- ++ +Don't continue to the next step until the output of the command is `"ACTIVE"`. +.. Configure `kubectl` to communicate with your cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks update-kubeconfig --name my-custom-networking-cluster +---- + + +[#custom-networking-configure-vpc] +== Step 2: Configure your VPC + +This tutorial requires the VPC created in <>. For a production cluster, adjust the steps accordingly for your VPC by replacing all of the example values with your own. + +. Confirm that your currently-installed Amazon VPC CNI plugin for Kubernetes is the latest version. To determine the latest version for the Amazon EKS add-on type and update your version to it, see <>. To determine the latest version for the self-managed add-on type and update your version to it, see <>. +. Retrieve the ID of your cluster VPC and store it in a variable for use in later steps. ++ +[source,bash,subs="verbatim,attributes"] +---- +vpc_id=$(aws eks describe-cluster --name my-custom-networking-cluster --query "cluster.resourcesVpcConfig.vpcId" --output text) +---- +. Associate an additional Classless Inter-Domain Routing (CIDR) block with your cluster's VPC. The CIDR block can't overlap with any existing associated CIDR blocks. ++ +.. View the current CIDR blocks associated to your VPC. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws ec2 describe-vpcs --vpc-ids $vpc_id \ + --query 'Vpcs[*].CidrBlockAssociationSet[*].{CIDRBlock: CidrBlock, State: CidrBlockState.State}' --out table +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +---------------------------------- +| DescribeVpcs | ++-----------------+--------------+ +| CIDRBlock | State | ++-----------------+--------------+ +| 192.168.0.0/24 | associated | ++-----------------+--------------+ +---- +.. Associate an additional CIDR block to your VPC. Replace the CIDR block value in the following command. For more information, see link:vpc/latest/userguide/modify-vpcs.html#add-ipv4-cidr[Associate additional IPv4 CIDR blocks with your VPC,type="documentation"] in the Amazon VPC User Guide. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws ec2 associate-vpc-cidr-block --vpc-id $vpc_id --cidr-block 192.168.1.0/24 +---- +.. Confirm that the new block is associated. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws ec2 describe-vpcs --vpc-ids $vpc_id --query 'Vpcs[*].CidrBlockAssociationSet[*].{CIDRBlock: CidrBlock, State: CidrBlockState.State}' --out table +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +---------------------------------- +| DescribeVpcs | ++-----------------+--------------+ +| CIDRBlock | State | ++-----------------+--------------+ +| 192.168.0.0/24 | associated | +| 192.168.1.0/24 | associated | ++-----------------+--------------+ +---- + ++ +Don't proceed to the next step until your new CIDR block's `State` is `associated`. +. Create as many subnets as you want to use in each Availability Zone that your existing subnets are in. Specify a CIDR block that's within the CIDR block that you associated with your VPC in a previous step. ++ +.. Create new subnets. Replace the CIDR block values in the following command. The subnets must be created in a different VPC CIDR block than your existing subnets are in, but in the same Availability Zones as your existing subnets. In this example, one subnet is created in the new CIDR block in each Availability Zone that the current private subnets exist in. The IDs of the subnets created are stored in variables for use in later steps. The `Name` values match the values assigned to the subnets created using the Amazon EKS VPC template in a previous step. Names aren't required. You can use different names. ++ +[source,bash,subs="verbatim,attributes"] +---- +new_subnet_id_1=$(aws ec2 create-subnet --vpc-id $vpc_id --availability-zone $az_1 --cidr-block 192.168.1.0/27 \ + --tag-specifications 'ResourceType=subnet,Tags=[{Key=Name,Value=my-eks-custom-networking-vpc-PrivateSubnet01},{Key=kubernetes.io/role/internal-elb,Value=1}]' \ + --query Subnet.SubnetId --output text) +new_subnet_id_2=$(aws ec2 create-subnet --vpc-id $vpc_id --availability-zone $az_2 --cidr-block 192.168.1.32/27 \ + --tag-specifications 'ResourceType=subnet,Tags=[{Key=Name,Value=my-eks-custom-networking-vpc-PrivateSubnet02},{Key=kubernetes.io/role/internal-elb,Value=1}]' \ + --query Subnet.SubnetId --output text) +---- ++ +IMPORTANT: By default, your new subnets are implicitly associated with your VPC's link:vpc/latest/userguide/VPC_Route_Tables.html#RouteTables[main route table,type="documentation"]. This route table allows communication between all the resources that are deployed in the VPC. However, it doesn't allow communication with resources that have IP addresses that are outside the CIDR blocks that are associated with your VPC. You can associate your own route table to your subnets to change this behavior. For more information, see link:vpc/latest/userguide/VPC_Route_Tables.html#subnet-route-tables[Subnet route tables,type="documentation"] in the Amazon VPC User Guide. +.. View the current subnets in your VPC. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws ec2 describe-subnets --filters "Name=vpc-id,Values=$vpc_id" \ + --query 'Subnets[*].{SubnetId: SubnetId,AvailabilityZone: AvailabilityZone,CidrBlock: CidrBlock}' \ + --output table +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +---------------------------------------------------------------------- +| DescribeSubnets | ++------------------+--------------------+----------------------------+ +| AvailabilityZone | CidrBlock | SubnetId | ++------------------+--------------------+----------------------------+ +| us-west-2d | 192.168.0.0/27 | subnet-example1 | +| us-west-2a | 192.168.0.32/27 | subnet-example2 | +| us-west-2a | 192.168.0.64/27 | subnet-example3 | +| us-west-2d | 192.168.0.96/27 | subnet-example4 | +| us-west-2a | 192.168.1.0/27 | subnet-example5 | +| us-west-2d | 192.168.1.32/27 | subnet-example6 | ++------------------+--------------------+----------------------------+ +---- ++ +You can see the subnets in the `192.168.1.0` CIDR block that you created are in the same Availability Zones as the subnets in the `192.168.0.0` CIDR block. + + +[#custom-networking-configure-kubernetes] +== Step 3: Configure Kubernetes resources +. Set the `AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG` environment variable to `true` in the `aws-node` DaemonSet. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl set env daemonset aws-node -n kube-system AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG=true +---- +. Retrieve the ID of your <> and store it in a variable for use in the next step. Amazon EKS automatically creates this security group when you create your cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +cluster_security_group_id=$(aws eks describe-cluster --name my-custom-networking-cluster --query cluster.resourcesVpcConfig.clusterSecurityGroupId --output text) +---- +. [[custom-networking-create-eniconfig]]Create an `ENIConfig` custom resource for each subnet that you want to deploy Pods in. ++ +.. Create a unique file for each network interface configuration. ++ +The following commands create separate `ENIConfig` files for the two subnets that were created in a previous step. The value for `name` must be unique. The name is the same as the Availability Zone that the subnet is in. The cluster security group is assigned to the `ENIConfig`. ++ +[source,yaml,subs="verbatim,attributes"] +---- +cat >$az_1.yaml <$az_2.yaml <> later in this tutorial. ++ +NOTE: If you don't specify a valid security group for use with a production cluster and you're using: + +*** version `1.8.0` or later of the Amazon VPC CNI plugin for Kubernetes, then the security groups associated with the node's primary elastic network interface are used. +*** a version of the Amazon VPC CNI plugin for Kubernetes that's earlier than `1.8.0`, then the default security group for the VPC is assigned to secondary network interfaces. + ++ +[IMPORTANT] +==== +* `AWS_VPC_K8S_CNI_EXTERNALSNAT=false` is a default setting in the configuration for the Amazon VPC CNI plugin for Kubernetes. If you're using the default setting, then traffic that is destined for IP addresses that aren't within one of the CIDR blocks associated with your VPC use the security groups and subnets of your node's primary network interface. The subnets and security groups defined in your `ENIConfigs` that are used to create secondary network interfaces aren't used for this traffic. For more information about this setting, see <>. +* If you also use security groups for Pods, the security group that's specified in a `SecurityGroupPolicy` is used instead of the security group that's specified in the `ENIConfigs`. For more information, see <>. +==== + ++ +.. Apply each custom resource file that you created to your cluster with the following commands. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl apply -f $az_1.yaml +kubectl apply -f $az_2.yaml +---- +. Confirm that your `ENIConfigs` were created. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get ENIConfigs +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +NAME AGE +us-west-2a 117s +us-west-2d 105s +---- +. If you're enabling custom networking on a production cluster and named your `ENIConfigs` something other than the Availability Zone that you're using them for, then skip to the <> to deploy Amazon EC2 nodes. ++ +Enable Kubernetes to automatically apply the `ENIConfig` for an Availability Zone to any new Amazon EC2 nodes created in your cluster. ++ +.. For the test cluster in this tutorial, skip to the <>. ++ +For a production cluster, check to see if an annotation with the key `k8s.amazonaws.com/eniConfig` for the `https://github.com/aws/amazon-vpc-cni-k8s#eni_config_annotation_def[ENI_CONFIG_ANNOTATION_DEF]` environment variable exists in the container spec for the `aws-node` DaemonSet. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl describe daemonset aws-node -n kube-system | grep ENI_CONFIG_ANNOTATION_DEF +---- ++ +If output is returned, the annotation exists. If no output is returned, then the variable is not set. For a production cluster, you can use either this setting or the setting in the following step. If you use this setting, it overrides the setting in the following step. In this tutorial, the setting in the next step is used. +.. [[custom-networking-automatically-apply-eniconfig]]Update your `aws-node` DaemonSet to automatically apply the `ENIConfig` for an Availability Zone to any new Amazon EC2 nodes created in your cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl set env daemonset aws-node -n kube-system ENI_CONFIG_LABEL_DEF=topology.kubernetes.io/zone +---- + + +[#custom-networking-deploy-nodes] +== Step 4: Deploy Amazon EC2 nodes +. Create a node IAM role. ++ +.. Run the following command to create an IAM trust policy JSON file. ++ +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:47721391-ab4d-4986-9a2a-a52174dc2256[] +---- +.. Create an IAM role and store its returned Amazon Resource Name (ARN) in a variable for use in a later step. ++ +[source,bash,subs="verbatim,attributes"] +---- +node_role_arn=$(aws iam create-role --role-name myCustomNetworkingNodeRole --assume-role-policy-document file://"node-role-trust-relationship.json" \ + --query Role.Arn --output text) +---- +.. Attach three required IAM managed policies to the IAM role. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam attach-role-policy \ + --policy-arn {arn-aws}iam::aws:policy/AmazonEKSWorkerNodePolicy \ + --role-name myCustomNetworkingNodeRole +aws iam attach-role-policy \ + --policy-arn {arn-aws}iam::aws:policy/AmazonEC2ContainerRegistryReadOnly \ + --role-name myCustomNetworkingNodeRole +aws iam attach-role-policy \ + --policy-arn {arn-aws}iam::aws:policy/AmazonEKS_CNI_Policy \ + --role-name myCustomNetworkingNodeRole +---- ++ +IMPORTANT: For simplicity in this tutorial, the link:aws-managed-policy/latest/reference/AmazonEKS_CNI_Policy.html[AmazonEKS_CNI_Policy,type="documentation"] policy is attached to the node IAM role. In a production cluster however, we recommend attaching the policy to a separate IAM role that is used only with the Amazon VPC CNI plugin for Kubernetes. For more information, see <>. +. Create one of the following types of node groups. To determine the instance type that you want to deploy, see <>. For this tutorial, complete the *Managed*, *Without a launch template or with a launch template without an AMI ID specified* option. If you're going to use the node group for production workloads, then we recommend that you familiarize yourself with all of the <> and <> options before deploying the node group. ++ +** *Managed* – Deploy your node group using one of the following options: ++ +*** *Without a launch template or with a launch template without an AMI ID specified* – Run the following command. For this tutorial, use the example values. For a production node group, replace all example values with your own. The node group name can't be longer than 63 characters. It must start with letter or digit, but can also include hyphens and underscores for the remaining characters. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks create-nodegroup --cluster-name my-custom-networking-cluster --nodegroup-name my-nodegroup \ + --subnets $subnet_id_1 $subnet_id_2 --instance-types t3.medium --node-role $node_role_arn +---- +*** *With a launch template with a specified AMI ID* + ++ +.... Determine the Amazon EKS recommended number of maximum Pods for your nodes. Follow the instructions in <>, adding `--cni-custom-networking-enabled` to step 3 in that topic. Note the output for use in the next step. +.... In your launch template, specify an Amazon EKS optimized AMI ID, or a custom AMI built off the Amazon EKS optimized AMI, then <> and provide the following user data in the launch template. This user data passes arguments into the `NodeConfig` specification. For more information about NodeConfig, see the https://awslabs.github.io/amazon-eks-ami/nodeadm/doc/api/#nodeconfig[NodeConfig API reference]. You can replace `20` with either the value from the previous step (recommended) or your own value. ++ +[source,bash,subs="verbatim,attributes"] +---- +--- +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="BOUNDARY" +--BOUNDARY +Content-Type: application/node.eks.aws + +--- +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: my-cluster + ... + kubelet: + config: + maxPods: 20 +---- ++ +If you've created a custom AMI that is not built off the Amazon EKS optimized AMI, then you need to custom create the configuration yourself. +** *Self-managed* + ++ +... Determine the Amazon EKS recommended number of maximum Pods for your nodes. Follow the instructions in <>, adding `--cni-custom-networking-enabled` to step 3 in that topic. Note the output for use in the next step. +... Deploy the node group using the instructions in <>. + + ++ +[NOTE] +==== +If you want nodes in a production cluster to support a significantly higher number of Pods, run the script in <> again. Also, add the `--cni-prefix-delegation-enabled` option to the command. For example, `110` is returned for an `m5.large` instance type. For instructions on how to enable this capability, see <>. You can use this capability with custom networking. +==== ++ +. Node group creation takes several minutes. You can check the status of the creation of a managed node group with the following command. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks describe-nodegroup --cluster-name my-custom-networking-cluster --nodegroup-name my-nodegroup --query nodegroup.status --output text +---- ++ +Don't continue to the next step until the output returned is `ACTIVE`. +. [[custom-networking-annotate-eniconfig]]For the tutorial, you can skip this step. ++ +For a production cluster, if you didn't name your `ENIConfigs` the same as the Availability Zone that you're using them for, then you must annotate your nodes with the `ENIConfig` name that should be used with the node. This step isn't necessary if you only have one subnet in each Availability Zone and you named your `ENIConfigs` with the same names as your Availability Zones. This is because the Amazon VPC CNI plugin for Kubernetes automatically associates the correct `ENIConfig` with the node for you when you enabled it to do so in a <>. ++ +.. Get the list of nodes in your cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get nodes +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +NAME STATUS ROLES AGE VERSION +ip-192-168-0-126.us-west-2.compute.internal Ready 8m49s v1.22.9-eks-810597c +ip-192-168-0-92.us-west-2.compute.internal Ready 8m34s v1.22.9-eks-810597c +---- +.. Determine which Availability Zone each node is in. Run the following command for each node that was returned in the previous step, replacing the IP addresses based on the previous output. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws ec2 describe-instances --filters Name=network-interface.private-dns-name,Values=ip-192-168-0-126.us-west-2.compute.internal \ +--query 'Reservations[].Instances[].{AvailabilityZone: Placement.AvailabilityZone, SubnetId: SubnetId}' +---- ++ +An example output is as follows. ++ +[source,json,subs="verbatim,attributes",role="nocopy"] +---- +[ + { + "AvailabilityZone": "us-west-2d", + "SubnetId": "subnet-Example5" + } +] +---- +.. Annotate each node with the `ENIConfig` that you created for the subnet ID and Availability Zone. You can only annotate a node with one `ENIConfig`, though multiple nodes can be annotated with the same `ENIConfig`. Replace the example values with your own. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl annotate node ip-192-168-0-126.us-west-2.compute.internal k8s.amazonaws.com/eniConfig=EniConfigName1 +kubectl annotate node ip-192-168-0-92.us-west-2.compute.internal k8s.amazonaws.com/eniConfig=EniConfigName2 +---- +. [[custom-networking-terminate-existing-nodes]]If you had nodes in a production cluster with running Pods before you switched to using the custom networking feature, complete the following tasks: ++ +.. Make sure that you have available nodes that are using the custom networking feature. +.. Cordon and drain the nodes to gracefully shut down the Pods. For more information, see https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/[Safely Drain a Node] in the Kubernetes documentation. +.. Terminate the nodes. If the nodes are in an existing managed node group, you can delete the node group. Run the following command. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks delete-nodegroup --cluster-name my-custom-networking-cluster --nodegroup-name my-nodegroup +---- + ++ +Only new nodes that are registered with the `k8s.amazonaws.com/eniConfig` label use the custom networking feature. +. Confirm that Pods are assigned an IP address from a CIDR block that's associated to one of the subnets that you created in a previous step. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get pods -A -o wide +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES +kube-system aws-node-2rkn4 1/1 Running 0 7m19s 192.168.0.92 ip-192-168-0-92.us-west-2.compute.internal +kube-system aws-node-k96wp 1/1 Running 0 7m15s 192.168.0.126 ip-192-168-0-126.us-west-2.compute.internal +kube-system coredns-657694c6f4-smcgr 1/1 Running 0 56m 192.168.1.23 ip-192-168-0-92.us-west-2.compute.internal +kube-system coredns-657694c6f4-stwv9 1/1 Running 0 56m 192.168.1.28 ip-192-168-0-92.us-west-2.compute.internal +kube-system kube-proxy-jgshq 1/1 Running 0 7m19s 192.168.0.92 ip-192-168-0-92.us-west-2.compute.internal +kube-system kube-proxy-wx9vk 1/1 Running 0 7m15s 192.168.0.126 ip-192-168-0-126.us-west-2.compute.internal +---- ++ +You can see that the coredns Pods are assigned IP addresses from the `192.168.1.0` CIDR block that you added to your VPC. Without custom networking, they would have been assigned addresses from the `192.168.0.0` CIDR block, because it was the only CIDR block originally associated with the VPC. ++ +If a Pod's `spec` contains `hostNetwork=true`, it's assigned the primary IP address of the node. It isn't assigned an address from the subnets that you added. By default, this value is set to `false`. This value is set to `true` for the `kube-proxy` and Amazon VPC CNI plugin for Kubernetes (`aws-node`) Pods that run on your cluster. This is why the `kube-proxy` and the plugin's `aws-node` Pods aren't assigned 192.168.1.x addresses in the previous output. For more information about a Pod's `hostNetwork` setting, see https://kubernetes.io/docs/reference/generated/kubernetes-api/v{k8s-n}/#podspec-v1-core[PodSpec v1 core] in the Kubernetes API reference. + + +[#custom-network-delete-resources] +== Step 5: Delete tutorial resources + +After you complete the tutorial, we recommend that you delete the resources that you created. You can then adjust the steps to enable custom networking for a production cluster. + +. If the node group that you created was just for testing, then delete it. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks delete-nodegroup --cluster-name my-custom-networking-cluster --nodegroup-name my-nodegroup +---- +. Even after the {aws} CLI output says that the cluster is deleted, the delete process might not actually be complete. The delete process takes a few minutes. Confirm that it's complete by running the following command. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks describe-nodegroup --cluster-name my-custom-networking-cluster --nodegroup-name my-nodegroup --query nodegroup.status --output text +---- ++ +Don't continue until the returned output is similar to the following output. ++ +[source,bash,subs="verbatim,attributes"] +---- +An error occurred (ResourceNotFoundException) when calling the DescribeNodegroup operation: No node group found for name: my-nodegroup. +---- +. If the node group that you created was just for testing, then delete the node IAM role. ++ +.. Detach the policies from the role. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam detach-role-policy --role-name myCustomNetworkingNodeRole --policy-arn {arn-aws}iam::aws:policy/AmazonEKSWorkerNodePolicy +aws iam detach-role-policy --role-name myCustomNetworkingNodeRole --policy-arn {arn-aws}iam::aws:policy/AmazonEC2ContainerRegistryReadOnly +aws iam detach-role-policy --role-name myCustomNetworkingNodeRole --policy-arn {arn-aws}iam::aws:policy/AmazonEKS_CNI_Policy +---- +.. Delete the role. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam delete-role --role-name myCustomNetworkingNodeRole +---- +. Delete the cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks delete-cluster --name my-custom-networking-cluster +---- ++ +Confirm the cluster is deleted with the following command. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks describe-cluster --name my-custom-networking-cluster --query cluster.status --output text +---- ++ +When output similar to the following is returned, the cluster is successfully deleted. ++ +[source,bash,subs="verbatim,attributes"] +---- +An error occurred (ResourceNotFoundException) when calling the DescribeCluster operation: No cluster found for name: my-custom-networking-cluster. +---- +. Delete the cluster IAM role. ++ +.. Detach the policies from the role. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam detach-role-policy --role-name myCustomNetworkingAmazonEKSClusterRole --policy-arn {arn-aws}iam::aws:policy/AmazonEKSClusterPolicy +---- +.. Delete the role. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam delete-role --role-name myCustomNetworkingAmazonEKSClusterRole +---- +. Delete the subnets that you created in a previous step. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws ec2 delete-subnet --subnet-id $new_subnet_id_1 +aws ec2 delete-subnet --subnet-id $new_subnet_id_2 +---- +. Delete the VPC that you created. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws cloudformation delete-stack --stack-name my-eks-custom-networking-vpc +---- diff --git a/latest/ug/networking/cni-custom-network.adoc b/latest/ug/networking/cni-custom-network.adoc new file mode 100644 index 000000000..17e1e8105 --- /dev/null +++ b/latest/ug/networking/cni-custom-network.adoc @@ -0,0 +1,38 @@ +include::../attributes.txt[] + +[.topic] +[#cni-custom-network] += Deploy Pods in alternate subnets with custom networking +:info_titleabbrev: Custom networking + +[abstract] +-- +Learn how to enable custom networking for Amazon EKS Pods to deploy them in different subnets or use different security groups than the node's primary network interface, increasing IP address availability and network isolation. +-- + +*Applies to*: Linux `IPv4` Fargate nodes, Linux nodes with Amazon EC2 instances + +image::images/cn-image.png[Diagram of node with multiple network interfaces] + +By default, when the Amazon VPC CNI plugin for Kubernetes creates secondary link:AWSEC2/latest/UserGuide/using-eni.html[elastic network interfaces,type="documentation"] (network interfaces) for your Amazon EC2 node, it creates them in the same subnet as the node's primary network interface. It also associates the same security groups to the secondary network interface that are associated to the primary network interface. For one or more of the following reasons, you might want the plugin to create secondary network interfaces in a different subnet or want to associate different security groups to the secondary network interfaces, or both: + +* There's a limited number of `IPv4` addresses that are available in the subnet that the primary network interface is in. This might limit the number of Pods that you can create in the subnet. By using a different subnet for secondary network interfaces, you can increase the number of available `IPv4` addresses available for Pods. +* For security reasons, your Pods might need to use a different subnet or security groups than the node's primary network interface. +* The nodes are configured in public subnets, and you want to place the Pods in private subnets. The route table associated to a public subnet includes a route to an internet gateway. The route table associated to a private subnet doesn't include a route to an internet gateway. + +TIP: You can also add a new or existing subnet directly to your Amazon EKS Cluster, without using custom networking. For more information, see <>. + + +[#cni-custom-network-considerations] +== Considerations + +The following are considerations for using the feature. + +* With custom networking enabled, no IP addresses assigned to the primary network interface are assigned to Pods. Only IP addresses from secondary network interfaces are assigned to Pods. +* If your cluster uses the `IPv6` family, you can't use custom networking. +* If you plan to use custom networking only to help alleviate `IPv4` address exhaustion, you can create a cluster using the `IPv6` family instead. For more information, see <>. +* Even though Pods deployed to subnets specified for secondary network interfaces can use different subnet and security groups than the node's primary network interface, the subnets and security groups must be in the same VPC as the node. +* For Fargate, subnets are controlled through the Fargate profile. For more information, see <>. + +include::cni-custom-network-tutorial.adoc[leveloffset=+1] + diff --git a/latest/ug/networking/cni-iam-role.adoc b/latest/ug/networking/cni-iam-role.adoc new file mode 100644 index 000000000..dc44da635 --- /dev/null +++ b/latest/ug/networking/cni-iam-role.adoc @@ -0,0 +1,232 @@ +include::../attributes.txt[] + +[.topic] +[#cni-iam-role] += Configure Amazon VPC CNI plugin to use IRSA +:info_titleabbrev: Configure for IRSA + +[abstract] +-- +Learn how to configure the Amazon VPC CNI plugin for Kubernetes to use IAM roles for service accounts (IRSA) for Pod networking in Amazon EKS clusters. +-- + +The https://github.com/aws/amazon-vpc-cni-k8s[Amazon VPC CNI plugin for Kubernetes] is the networking plugin for Pod networking in Amazon EKS clusters. The plugin is responsible for allocating VPC IP addresses to Kubernetes pods and configuring the necessary networking for Pods on each node. + +[NOTE] +==== +The Amazon VPC CNI plugin also supports Amazon EKS Pod Identities. For more information, see <>. +==== + +The plugin: + +* Requires {aws} Identity and Access Management (IAM) permissions. If your cluster uses the `IPv4` family, the permissions are specified in the link:aws-managed-policy/latest/reference/AmazonEKS_CNI_Policy.html[`AmazonEKS_CNI_Policy`,type="documentation"] {aws} managed policy. If your cluster uses the `IPv6` family, then the permissions must be added to an IAM policy that you create; for instructions, see <>. You can attach the policy to the Amazon EKS node IAM role, or to a separate IAM role. For instructions to attach the policy to the Amazon EKS node IAM role, see <>. We recommend that you assign it to a separate role, as detailed in this topic. +* Creates and is configured to use a Kubernetes service account named `aws-node` when it's deployed. The service account is bound to a Kubernetes `clusterrole` named `aws-node`, which is assigned the required Kubernetes permissions. + + +[NOTE] +==== + +The Pods for the Amazon VPC CNI plugin for Kubernetes have access to the permissions assigned to the <>, unless you block access to IMDS. For more information, see https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node[Restrict access to the instance profile assigned to the worker node]. + +==== + +* Requires an existing Amazon EKS cluster. To deploy one, see <>. +* Requires an existing {aws} Identity and Access Management (IAM) OpenID Connect (OIDC) provider for your cluster. To determine whether you already have one, or to create one, see <>. + + +[#cni-iam-role-create-role] +== Step 1: Create the Amazon VPC CNI plugin for Kubernetes IAM role +. Determine the IP family of your cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks describe-cluster --name my-cluster | grep ipFamily +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +"ipFamily": "ipv4" +---- ++ +The output may return `ipv6` instead. +. Create the IAM role. You can use `eksctl` or `kubectl` and the {aws} CLI to create your IAM role. ++ +eksctl::: +** Create an IAM role and attach the IAM policy to the role with the command that matches the IP family of your cluster. The command creates and deploys an {aws} CloudFormation stack that creates an IAM role, attaches the policy that you specify to it, and annotates the existing `aws-node` Kubernetes service account with the ARN of the IAM role that is created. ++ +*** `IPv4` ++ +Replace [.replaceable]`my-cluster` with your own value. ++ +[source,bash,subs="verbatim,attributes"] +---- +eksctl create iamserviceaccount \ + --name aws-node \ + --namespace kube-system \ + --cluster my-cluster \ + --role-name AmazonEKSVPCCNIRole \ + --attach-policy-arn {arn-aws}iam::aws:policy/AmazonEKS_CNI_Policy \ + --override-existing-serviceaccounts \ + --approve +---- +*** `IPv6` ++ +Replace [.replaceable]`my-cluster` with your own value. Replace [.replaceable]`111122223333` with your account ID and replace [.replaceable]`AmazonEKS_CNI_IPv6_Policy` with the name of your `IPv6` policy. If you don't have an `IPv6` policy, see <> to create one. To use `IPv6` with your cluster, it must meet several requirements. For more information, see <>. ++ +[source,bash,subs="verbatim,attributes"] +---- +eksctl create iamserviceaccount \ + --name aws-node \ + --namespace kube-system \ + --cluster my-cluster \ + --role-name AmazonEKSVPCCNIRole \ + --attach-policy-arn {arn-aws}iam::111122223333:policy/AmazonEKS_CNI_IPv6_Policy \ + --override-existing-serviceaccounts \ + --approve +---- + + +kubectl and the {aws} CLI::: +... View your cluster's OIDC provider URL. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks describe-cluster --name my-cluster --query "cluster.identity.oidc.issuer" --output text +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +https://oidc.eks.region-code.amazonaws.com/id/EXAMPLED539D4633E53DE1B71EXAMPLE +---- ++ +If no output is returned, then you must <>. +... Copy the following contents to a file named [.replaceable]`vpc-cni-trust-policy.json`. Replace [.replaceable]`111122223333` with your account ID and [.replaceable]`EXAMPLED539D4633E53DE1B71EXAMPLE` with the output returned in the previous step. Replace [.replaceable]`region-code` with the {aws} Region that your cluster is in. ++ +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:7410ad54-dbf3-4644-81cd-1affe22fd796[] +---- +... Create the role. You can replace [.replaceable]`AmazonEKSVPCCNIRole` with any name that you choose. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam create-role \ + --role-name AmazonEKSVPCCNIRole \ + --assume-role-policy-document file://"vpc-cni-trust-policy.json" +---- +... Attach the required IAM policy to the role. Run the command that matches the IP family of your cluster. ++ +**** `IPv4` ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam attach-role-policy \ + --policy-arn {arn-aws}iam::aws:policy/AmazonEKS_CNI_Policy \ + --role-name AmazonEKSVPCCNIRole +---- +**** `IPv6` ++ +Replace [.replaceable]`111122223333` with your account ID and [.replaceable]`AmazonEKS_CNI_IPv6_Policy` with the name of your `IPv6` policy. If you don't have an `IPv6` policy, see <> to create one. To use `IPv6` with your cluster, it must meet several requirements. For more information, see <>. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam attach-role-policy \ + --policy-arn {arn-aws}iam::111122223333:policy/AmazonEKS_CNI_IPv6_Policy \ + --role-name AmazonEKSVPCCNIRole +---- +... Run the following command to annotate the `aws-node` service account with the ARN of the IAM role that you created previously. Replace the example values with your own values. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl annotate serviceaccount \ + -n kube-system aws-node \ + eks.amazonaws.com/role-arn={arn-aws}iam::111122223333:role/AmazonEKSVPCCNIRole +---- +. (Optional) Configure the {aws} Security Token Service endpoint type used by your Kubernetes service account. For more information, see <>. + + +[#cni-iam-role-redeploy-pods] +== Step 2: Re-deploy Amazon VPC CNI plugin for Kubernetes Pods +. Delete and re-create any existing Pods that are associated with the service account to apply the credential environment variables. The annotation is not applied to Pods that are currently running without the annotation. The following command deletes the existing `aws-node` DaemonSet Pods and deploys them with the service account annotation. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl delete Pods -n kube-system -l k8s-app=aws-node +---- +. Confirm that the Pods all restarted. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get pods -n kube-system -l k8s-app=aws-node +---- +. Describe one of the Pods and verify that the `AWS_WEB_IDENTITY_TOKEN_FILE` and `AWS_ROLE_ARN` environment variables exist. Replace [.replaceable]`cpjw7` with the name of one of your Pods returned in the output of the previous step. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl describe pod -n kube-system aws-node-cpjw7 | grep 'AWS_ROLE_ARN:\|AWS_WEB_IDENTITY_TOKEN_FILE:' +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +AWS_ROLE_ARN: {arn-aws}iam::111122223333:role/AmazonEKSVPCCNIRole + AWS_WEB_IDENTITY_TOKEN_FILE: /var/run/secrets/eks.amazonaws.com/serviceaccount/token + AWS_ROLE_ARN: {arn-aws}iam::111122223333:role/AmazonEKSVPCCNIRole + AWS_WEB_IDENTITY_TOKEN_FILE: /var/run/secrets/eks.amazonaws.com/serviceaccount/token +---- ++ +Two sets of duplicate results are returned because the Pod contains two containers. Both containers have the same values. ++ +If your Pod is using the {aws} Regional endpoint, then the following line is also returned in the previous output. ++ +[source,bash,subs="verbatim,attributes"] +---- +AWS_STS_REGIONAL_ENDPOINTS=regional +---- + + +[#remove-cni-policy-node-iam-role] +== Step 3: Remove the CNI policy from the node IAM role + +If your <> currently has the `AmazonEKS_CNI_Policy` IAM (`IPv4`) policyor an <>attached to it, and you've created a separate IAM role, attached the policy to it instead, and assigned it to the `aws-node` Kubernetes service account, then we recommend that you remove the policy from your node role with the {aws} CLI command that matches the IP family of your cluster. Replace [.replaceable]`AmazonEKSNodeRole` with the name of your node role. + + + +* `IPv4` ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam detach-role-policy --role-name AmazonEKSNodeRole --policy-arn {arn-aws}iam::aws:policy/AmazonEKS_CNI_Policy +---- +* `IPv6` ++ +Replace [.replaceable]`111122223333` with your account ID and [.replaceable]`AmazonEKS_CNI_IPv6_Policy` with the name of your `IPv6` policy. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam detach-role-policy --role-name AmazonEKSNodeRole --policy-arn {arn-aws}iam::111122223333:policy/AmazonEKS_CNI_IPv6_Policy +---- + + +[#cni-iam-role-create-ipv6-policy] +== Create IAM policy for clusters that use the `IPv6` family + +If you created a cluster that uses the `IPv6` family and the cluster has version `1.10.1` or later of the Amazon VPC CNI plugin for Kubernetes add-on configured, then you need to create an IAM policy that you can assign to an IAM role. If you have an existing cluster that you didn't configure with the `IPv6` family when you created it, then to use `IPv6`, you must create a new cluster. For more information about using `IPv6` with your cluster, see <>. + +. Copy the following text and save it to a file named `vpc-cni-ipv6-policy.json`. ++ +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:7d2817c2-2064-455b-93b9-4a819540317b[] +---- +. Create the IAM policy. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam create-policy --policy-name AmazonEKS_CNI_IPv6_Policy --policy-document file://vpc-cni-ipv6-policy.json +---- diff --git a/latest/ug/networking/cni-increase-ip-addresses-procedure.adoc b/latest/ug/networking/cni-increase-ip-addresses-procedure.adoc new file mode 100644 index 000000000..f7edf81bd --- /dev/null +++ b/latest/ug/networking/cni-increase-ip-addresses-procedure.adoc @@ -0,0 +1,215 @@ +include::../attributes.txt[] + +[.topic] +[#cni-increase-ip-addresses-procedure] += Increase the available IP addresses for your Amazon EKS node +:info_titleabbrev: Procedure + +You can increase the number of IP addresses that nodes can assign to Pods by assigning IP prefixes, rather than assigning individual secondary IP addresses to your nodes. + +== Prerequisites + +* You need an existing cluster. To deploy one, see <>. +* The subnets that your Amazon EKS nodes are in must have sufficient contiguous `/28` (for `IPv4` clusters) or `/80` (for `IPv6` clusters) Classless Inter-Domain Routing (CIDR) blocks. You can only have Linux nodes in an `IPv6` cluster. Using IP prefixes can fail if IP addresses are scattered throughout the subnet CIDR. We recommend the following: +** Using a subnet CIDR reservation so that even if any IP addresses within the reserved range are still in use, upon their release, the IP addresses aren't reassigned. This ensures that prefixes are available for allocation without segmentation. +** Use new subnets that are specifically used for running the workloads that IP prefixes are assigned to. Both Windows and Linux workloads can run in the same subnet when assigning IP prefixes. +* To assign IP prefixes to your nodes, your nodes must be {aws} Nitro-based. Instances that aren't Nitro-based continue to allocate individual secondary IP addresses, but have a significantly lower number of IP addresses to assign to Pods than Nitro-based instances do. +* *For clusters with Linux nodes only* – If your cluster is configured for the `IPv4` family, you must have version `1.9.0` or later of the Amazon VPC CNI plugin for Kubernetes add-on installed. You can check your current version with the following command. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl describe daemonset aws-node --namespace kube-system | grep Image | cut -d "/" -f 2 +---- ++ +If your cluster is configured for the `IPv6` family, you must have version `1.10.1` of the add-on installed. If your plugin version is earlier than the required versions, you must update it. For more information, see the updating sections of <>. +* *For clusters with Windows nodes only* +** You must have Windows support enabled for your cluster. For more information, see <>. + +[#cni-increase-ip-procedure] +== Assign IP address prefixes to nodes + +Configure your cluster to assign IP address prefixes to nodes. Complete the procedure that matches your node's operating system. + +=== Linux +. Enable the parameter to assign prefixes to network interfaces for the Amazon VPC CNI DaemonSet. When you deploy a cluster, version `1.10.1` or later of the Amazon VPC CNI plugin for Kubernetes add-on is deployed with it. If you created the cluster with the `IPv6` family, this setting was set to `true` by default. If you created the cluster with the `IPv4` family, this setting was set to `false` by default. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl set env daemonset aws-node -n kube-system ENABLE_PREFIX_DELEGATION=true +---- ++ +[IMPORTANT] +==== +Even if your subnet has available IP addresses, if the subnet does not have any contiguous `/28` blocks available, you will see the following error in the Amazon VPC CNI plugin for Kubernetes logs. + +[source,bash,subs="verbatim,attributes"] +---- +InsufficientCidrBlocks: The specified subnet does not have enough free cidr blocks to satisfy the request +---- + +This can happen due to fragmentation of existing secondary IP addresses spread out across a subnet. To resolve this error, either create a new subnet and launch Pods there, or use an Amazon EC2 subnet CIDR reservation to reserve space within a subnet for use with prefix assignment. For more information, see link:vpc/latest/userguide/subnet-cidr-reservation.html[Subnet CIDR reservations,type="documentation"] in the Amazon VPC User Guide. +==== ++ +. If you plan to deploy a managed node group without a launch template, or with a launch template that you haven't specified an AMI ID in, and you're using a version of the Amazon VPC CNI plugin for Kubernetes at or later than the versions listed in the prerequisites, then skip to the next step. Managed node groups automatically calculates the maximum number of Pods for you. ++ +If you're deploying a self-managed node group or a managed node group with a launch template that you have specified an AMI ID in, then you must determine the Amazon EKS recommend number of maximum Pods for your nodes. Follow the instructions in <>, adding `--cni-prefix-delegation-enabled` to step 3. Note the output for use in a later step. ++ +IMPORTANT: Managed node groups enforces a maximum number on the value of `maxPods`. For instances with less than 30 vCPUs the maximum number is 110 and for all other instances the maximum number is 250. This maximum number is applied whether prefix delegation is enabled or not. +. If you're using a cluster configured for `IPv6`, skip to the next step. ++ +Specify the parameters in one of the following options. To determine which option is right for you and what value to provide for it, see https://github.com/aws/amazon-vpc-cni-k8s/blob/master/docs/prefix-and-ip-target.md[WARM_PREFIX_TARGET, WARM_IP_TARGET, and MINIMUM_IP_TARGET] on GitHub. ++ +You can replace the example values with a value greater than zero. ++ +*** `WARM_PREFIX_TARGET` ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl set env ds aws-node -n kube-system WARM_PREFIX_TARGET=1 +---- +*** `WARM_IP_TARGET` or `MINIMUM_IP_TARGET` – If either value is set, it overrides any value set for `WARM_PREFIX_TARGET`. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl set env ds aws-node -n kube-system WARM_IP_TARGET=5 +---- ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl set env ds aws-node -n kube-system MINIMUM_IP_TARGET=2 +---- +. Create one of the following types of node groups with at least one Amazon EC2 Nitro Amazon Linux 2 instance type. For a list of Nitro instance types, see link:AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances[Instances built on the Nitro System,type="documentation"] in the Amazon EC2 User Guide. This capability is not supported on Windows. For the options that include [.replaceable]`110`, replace it with either the value from step 3 (recommended), or your own value. ++ +*** *Self-managed* – Deploy the node group using the instructions in <>. Specify the following text for the *BootstrapArguments* parameter. ++ +[source,bash,subs="verbatim,attributes"] +---- +--use-max-pods false --kubelet-extra-args '--max-pods=110' +---- ++ +If you're using `eksctl` to create the node group, you can use the following command. ++ +[source,bash,subs="verbatim,attributes"] +---- +eksctl create nodegroup --cluster my-cluster --managed=false --max-pods-per-node 110 +---- +*** *Managed* – Deploy your node group using one of the following options: ++ +**** *Without a launch template or with a launch template without an AMI ID specified* – Complete the procedure in <>. Managed node groups automatically calculates the Amazon EKS recommended `max-pods` value for you. +**** *With a launch template with a specified AMI ID* – In your launch template, specify an Amazon EKS optimized AMI ID, or a custom AMI built off the Amazon EKS optimized AMI, then <> and provide the following user data in the launch template. This user data passes arguments into the `bootstrap.sh` file. For more information about the bootstrap file, see https://github.com/awslabs/amazon-eks-ami/blob/main/templates/al2/runtime/bootstrap.sh[bootstrap.sh] on GitHub. ++ +[source,bash,subs="verbatim,attributes"] +---- +/etc/eks/bootstrap.sh my-cluster \ + --use-max-pods false \ + --kubelet-extra-args '--max-pods=110' +---- ++ +If you're using `eksctl` to create the node group, you can use the following command. ++ +[source,bash,subs="verbatim,attributes"] +---- +eksctl create nodegroup --cluster my-cluster --max-pods-per-node 110 +---- ++ +If you've created a custom AMI that is not built off the Amazon EKS optimized AMI, then you need to custom create the configuration yourself. + ++ +NOTE: If you also want to assign IP addresses to Pods from a different subnet than the instance's, then you need to enable the capability in this step. For more information, see <>. + +=== Windows +. Enable assignment of IP prefixes. ++ +.. Open the `amazon-vpc-cni` `ConfigMap` for editing. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl edit configmap -n kube-system amazon-vpc-cni -o yaml +---- +.. Add the following line to the `data` section. ++ +[source,yaml,subs="verbatim,attributes"] +---- + enable-windows-prefix-delegation: "true" +---- +.. Save the file and close the editor. +.. Confirm that the line was added to the `ConfigMap`. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get configmap -n kube-system amazon-vpc-cni -o "jsonpath={.data.enable-windows-prefix-delegation}" +---- ++ +If the returned output isn't `true`, then there might have been an error. Try completing the step again. ++ +[IMPORTANT] +==== +Even if your subnet has available IP addresses, if the subnet does not have any contiguous `/28` blocks available, you will see the following error in the Amazon VPC CNI plugin for Kubernetes logs. + +[source,bash,subs="verbatim,attributes"] +---- +InsufficientCidrBlocks: The specified subnet does not have enough free cidr blocks to satisfy the request +---- + +This can happen due to fragmentation of existing secondary IP addresses spread out across a subnet. To resolve this error, either create a new subnet and launch Pods there, or use an Amazon EC2 subnet CIDR reservation to reserve space within a subnet for use with prefix assignment. For more information, see link:vpc/latest/userguide/subnet-cidr-reservation.html[Subnet CIDR reservations,type="documentation"] in the Amazon VPC User Guide. +==== +. (Optional) Specify additional configuration for controlling the pre-scaling and dynamic scaling behavior for your cluster. For more information, see https://github.com/aws/amazon-vpc-resource-controller-k8s/blob/master/docs/windows/prefix_delegation_config_options.md[Configuration options with Prefix Delegation mode on Windows] on GitHub. ++ +.. Open the `amazon-vpc-cni` `ConfigMap` for editing. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl edit configmap -n kube-system amazon-vpc-cni -o yaml +---- +.. Replace the example values with a value greater than zero and add the entries that you require to the `data` section of the `ConfigMap`. If you set a value for either `warm-ip-target` or `minimum-ip-target`, the value overrides any value set for `warm-prefix-target`. ++ +[source,yaml,subs="verbatim,attributes"] +---- + warm-prefix-target: "1" + warm-ip-target: "5" + minimum-ip-target: "2" +---- +.. Save the file and close the editor. +. Create Windows node groups with at least one Amazon EC2 Nitro instance type. For a list of Nitro instance types, see link:AWSEC2/latest/WindowsGuide/instance-types.html#ec2-nitro-instances[Instances built on the Nitro System,type="documentation"] in the Amazon EC2 User Guide. By default, the maximum number of Pods that you can deploy to a node is 110. If you want to increase or decrease that number, specify the following in the user data for the bootstrap configuration. Replace [.replaceable]`max-pods-quantity` with your max pods value. ++ +[source,bash,subs="verbatim,attributes"] +---- +-KubeletExtraArgs '--max-pods=max-pods-quantity' +---- ++ +If you're deploying managed node groups, this configuration needs to be added in the launch template. For more information, see <>. For more information about the configuration parameters for Windows bootstrap script, see <>. + +[#cni-increase-ip-verify] +== Determine max Pods and available IP addresses + +. Once your nodes are deployed, view the nodes in your cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get nodes +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +NAME STATUS ROLES AGE VERSION +ip-192-168-22-103.region-code.compute.internal Ready 19m v1.XX.X-eks-6b7464 +ip-192-168-97-94.region-code.compute.internal Ready 19m v1.XX.X-eks-6b7464 +---- +. Describe one of the nodes to determine the value of `max-pods` for the node and the number of available IP addresses. Replace [.replaceable]`192.168.30.193` with the `IPv4` address in the name of one of your nodes returned in the previous output. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl describe node ip-192-168-30-193.region-code.compute.internal | grep 'pods\|PrivateIPv4Address' +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +pods: 110 +vpc.amazonaws.com/PrivateIPv4Address: 144 +---- ++ +In the previous output, `110` is the maximum number of Pods that Kubernetes will deploy to the node, even though [.replaceable]`144` IP addresses are available. \ No newline at end of file diff --git a/latest/ug/networking/cni-increase-ip-addresses.adoc b/latest/ug/networking/cni-increase-ip-addresses.adoc new file mode 100644 index 000000000..5b2032778 --- /dev/null +++ b/latest/ug/networking/cni-increase-ip-addresses.adoc @@ -0,0 +1,62 @@ +include::../attributes.txt[] + +[.topic] +[#cni-increase-ip-addresses] += Assign more IP addresses to Amazon EKS nodes with prefixes +:info_titleabbrev: Increase IP addresses + +[abstract] +-- +Learn how to significantly increase the number of IP addresses that you can assign to Pods by assigning IP prefixes with Amazon EKS, improving scalability and reducing launch delays for large and spiky workloads. +-- + +*Applies to*: Linux and Windows nodes with Amazon EC2 instances + +*Applies to*: Public and private subnets + +Each Amazon EC2 instance supports a maximum number of elastic network interfaces and a maximum number of IP addresses that can be assigned to each network interface. Each node requires one IP address for each network interface. All other available IP addresses can be assigned to `Pods`. Each `Pod` requires its own IP address. As a result, you might have nodes that have available compute and memory resources, but can't accommodate additional `Pods` because the node has run out of IP addresses to assign to `Pods`. + +You can increase the number of IP addresses that nodes can assign to `Pods` by assigning IP prefixes, rather than assigning individual secondary IP addresses to your nodes. Each prefix includes several IP addresses. If you don't configure your cluster for IP prefix assignment, your cluster must make more Amazon EC2 application programming interface (API) calls to configure network interfaces and IP addresses necessary for Pod connectivity. As clusters grow to larger sizes, the frequency of these API calls can lead to longer Pod and instance launch times. This results in scaling delays to meet the demand of large and spiky workloads, and adds cost and management overhead because you need to provision additional clusters and VPCs to meet scaling requirements. For more information, see https://github.com/kubernetes/community/blob/master/sig-scalability/configs-and-limits/thresholds.md[Kubernetes Scalability thresholds] on GitHub. + +[#cni-increase-ip-addresses-compatability] +== Compatibility with Amazon VPC CNI plugin for Kubernetes features + +You can use IP prefixes with the following features: + + + +* IPv4 Source Network Address Translation - For more information, see <>. +* IPv6 addresses to clusters, Pods, and services - For more information, see <>. +* Restricting traffic using Kubernetes network policies - For more information, see <>. + +The following list provides information about the Amazon VPC CNI plugin settings that apply. For more information about each setting, see https://github.com/aws/amazon-vpc-cni-k8s/blob/master/README.md[amazon-vpc-cni-k8s] on GitHub. + + + +* `WARM_IP_TARGET` +* `MINIMUM_IP_TARGET` +* `WARM_PREFIX_TARGET` + + +[#cni-increase-ip-addresses-considerations] +== Considerations + +Consider the following when you use this feature: + + + +* Each Amazon EC2 instance type supports a maximum number of Pods. If your managed node group consists of multiple instance types, the smallest number of maximum Pods for an instance in the cluster is applied to all nodes in the cluster. +* By default, the maximum number of `Pods` that you can run on a node is 110, but you can change that number. If you change the number and have an existing managed node group, the next AMI or launch template update of your node group results in new nodes coming up with the changed value. +* When transitioning from assigning IP addresses to assigning IP prefixes, we recommend that you create new node groups to increase the number of available IP addresses, rather than doing a rolling replacement of existing nodes. Running Pods on a node that has both IP addresses and prefixes assigned can lead to inconsistency in the advertised IP address capacity, impacting the future workloads on the node. For the recommended way of performing the transition, see link:eks/latest/best-practices/prefix-mode-linux.html[Prefix Delegation mode for Linux, type="documentation"] in the _Amazon EKS Best Practices Guide_. +* The security group scope is at the node-level - For more information, see link:vpc/latest/userguide/VPC_SecurityGroups.html[Security group,type="documentation"]. +* IP prefixes assigned to a network interface support high Pod density per node and have the best launch time. +* IP prefixes and IP addresses are associated with standard Amazon EC2 elastic network interfaces. Pods requiring specific security groups are assigned the primary IP address of a branch network interface. You can mix Pods getting IP addresses, or IP addresses from IP prefixes with Pods getting branch network interfaces on the same node. +* For clusters with Linux nodes only. ++ +** After you configure the add-on to assign prefixes to network interfaces, you can't downgrade your Amazon VPC CNI plugin for Kubernetes add-on to a version lower than `1.9.0` (or `1.10.1`) without removing all nodes in all node groups in your cluster. +** If you're also using security groups for Pods, with `POD_SECURITY_GROUP_ENFORCING_MODE`=``standard`` and `AWS_VPC_K8S_CNI_EXTERNALSNAT`=``false``, when your Pods communicate with endpoints outside of your VPC, the node's security groups are used, rather than any security groups you've assigned to your Pods. ++ +If you're also using <>, with `POD_SECURITY_GROUP_ENFORCING_MODE`=``strict``, when your `Pods` communicate with endpoints outside of your VPC, the `Pod's` security groups are used. + +include::cni-increase-ip-addresses-procedure.adoc[leveloffset=+1] + diff --git a/latest/ug/networking/cni-ipv6.adoc b/latest/ug/networking/cni-ipv6.adoc new file mode 100644 index 000000000..629619c11 --- /dev/null +++ b/latest/ug/networking/cni-ipv6.adoc @@ -0,0 +1,72 @@ +include::../attributes.txt[] + +[.topic] +[#cni-ipv6] += Learn about IPv6 addresses to clusters, Pods, and services +:info_titleabbrev: IPv6 + +[abstract] +-- +Learn how to deploy an `IPv6` cluster and nodes with Amazon EKS for assigning `IPv6` addresses to Pods and services instead of `IPv4`, leveraging IP prefix delegation and the latest Amazon VPC CNI plugin. +-- + +*Applies to*: Pods with Amazon EC2 instances and Fargate Pods + +By default, Kubernetes assigns `IPv4` addresses to your Pods and services. Instead of assigning `IPv4` addresses to your Pods and services, you can configure your cluster to assign `IPv6` addresses to them. Amazon EKS doesn't support dual-stacked Pods or services, even though Kubernetes does. As a result, you can't assign both `IPv4` and `IPv6` addresses to your Pods and services. + +You select which IP family you want to use for your cluster when you create it. You can't change the family after you create the cluster. + +For a tutorial to deploy an Amazon EKS `IPv6` cluster, see <>. + +//[#ipv6-considerations] +//===== Considerations + +The following are considerations for using the feature: + +== `IPv6` Feature support + +* *No Windows support*: Windows Pods and services aren't supported. +* *Nitro-based EC2 nodes required*: You can only use `IPv6` with {aws} Nitro-based Amazon EC2 or Fargate nodes. +* *EC2 and Fargate nodes supported*: You can use `IPv6` with <> with Amazon EC2 nodes and Fargate nodes. +* *Outposts not supported*: You can't use `IPv6` with <>. +* *FSx for Lustre is not supported*: The <> is not supported. +* *Custom networking not supported*: If you previously used <> to help alleviate IP address exhaustion, you can use `IPv6` instead. You can't use custom networking with `IPv6`. If you use custom networking for network isolation, then you might need to continue to use custom networking and the `IPv4` family for your clusters. + + +== IP address assignments + +* *Kubernetes services*: Kubernetes services are only assigned an `IPv6` addresses. They aren't assigned IPv4 addresses. +* *Pods*: Pods are assigned an IPv6 address and a host-local IPv4 address. The host-local IPv4 address is assigned by using a host-local CNI plugin chained with VPC CNI and the address is not reported to the Kubernetes control plane. It is only used when a pod needs to communicate with an external IPv4 resources in another Amazon VPC or the internet. The host-local IPv4 address gets SNATed (by VPC CNI) to the primary IPv4 address of the primary ENI of the worker node. +* *Pods and services*: Pods and services receive only `IPv6` addresses, not `IPv4` addresses. When Pods need to communicate with external `IPv4` endpoints, they use NAT on the node itself. This built-in NAT capability eliminates the need for link:vpc/latest/userguide/vpc-nat-gateway.html#nat-gateway-nat64-dns64[DNS64 and NAT64,type="documentation"]. For traffic requiring public internet access, the Pod's traffic is source network address translated to a public IP address. +* *Routing addresses*: When a Pod communicates outside the VPC, its original `IPv6` address is preserved (not translated to the node's `IPv6` address). This traffic is routed directly through an internet gateway or egress-only internet gateway. +* *Nodes*: All nodes are assigned an `IPv4` and `IPv6` address. +* *Fargate Pods*: Each Fargate Pod receives an `IPv6` address from the CIDR that's specified for the subnet that it's deployed in. The underlying hardware unit that runs Fargate Pods gets a unique `IPv4` and `IPv6` address from the CIDRs that are assigned to the subnet that the hardware unit is deployed in. + + +== How to use `IPv6` with EKS + +* *Create new cluster*: You must create a new cluster and specify that you want to use the `IPv6` family for that cluster. You can't enable the `IPv6` family for a cluster that you updated from a previous version. For instructions on how to create a new cluster, see Considerations . +* *Use recent VPC CNI*: Deploy Amazon VPC CNI version `1.10.1` or later. This version or later is deployed by default. After you deploy the add-on, you can't downgrade your Amazon VPC CNI add-on to a version lower than `1.10.1` without first removing all nodes in all node groups in your cluster. +* *Configure VPC CNI for `IPv6`*: If you use Amazon EC2 nodes, you must configure the Amazon VPC CNI add-on with IP prefix delegation and `IPv6`. If you choose the `IPv6` family when creating your cluster, the `1.10.1` version of the add-on defaults to this configuration. This is the case for both a self-managed or Amazon EKS add-on. For more information about IP prefix delegation, see <>. +* *Configure `IPv4` and `IPv6` addresses*: When you create a cluster, the VPC and subnets that you specify must have an `IPv6` CIDR block that's assigned to the VPC and subnets that you specify. They must also have an `IPv4` CIDR block assigned to them. This is because, even if you only want to use `IPv6`, a VPC still requires an `IPv4` CIDR block to function. For more information, see link:vpc/latest/userguide/working-with-vpcs.html#vpc-associate-ipv6-cidr[Associate an IPv6 CIDR block with your VPC,type="documentation"] in the Amazon VPC User Guide. +* *Auto-assign IPv6 addresses to nodes:* When you create your nodes, you must specify subnets that are configured to auto-assign `IPv6` addresses. Otherwise, you can't deploy your nodes. By default, this configuration is disabled. For more information, see link:vpc/latest/userguide/vpc-ip-addressing.html#subnet-ipv6[Modify the IPv6 addressing attribute for your subnet,type="documentation"] in the Amazon VPC User Guide. +* *Set route tables to use `IPv6`*: The route tables that are assigned to your subnets must have routes for `IPv6` addresses. For more information, see link:vpc/latest/userguide/vpc-migrate-ipv6.html[Migrate to IPv6,type="documentation"] in the Amazon VPC User Guide. +* *Set security groups for `IPv6`*: Your security groups must allow `IPv6` addresses. For more information, see link:vpc/latest/userguide/vpc-migrate-ipv6.html[Migrate to IPv6,type="documentation"] in the Amazon VPC User Guide. +* *Set up load balancer*: Use version `2.3.1` or later of the {aws} Load Balancer Controller to load balance HTTP applications using the <> or network traffic using the <> to `IPv6` Pods with either load balancer in IP mode, but not instance mode. For more information, see <>. +* *Add `IPv6` IAM policy*: You must attach an `IPv6` IAM policy to your node IAM or CNI IAM role. Between the two, we recommend that you attach it to a CNI IAM role. For more information, see <> and <>. +* *Evaluate all components*: Perform a thorough evaluation of your applications, Amazon EKS add-ons, and {aws} services that you integrate with before deploying `IPv6` clusters. This is to ensure that everything works as expected with `IPv6`. +* *Add `BootstrapArguments` self-managed node groups*: When creating a self-managed node group in a cluster that uses the `IPv6` family, user-data must include the following `BootstrapArguments` for the https://github.com/awslabs/amazon-eks-ami/blob/main/templates/al2/runtime/bootstrap.sh[bootstrap.sh] file that runs at node start up. Replace [.replaceable]`your-cidr` with the `IPv6` CIDR range of your cluster's VPC. ++ +[source,bash,subs="verbatim,attributes"] +---- +--ip-family ipv6 --service-ipv6-cidr your-cidr +---- ++ +If you don't know the `IPv6` `CIDR` range for your cluster, you can see it with the following command (requires the {aws} CLI version `2.4.9` or later). ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks describe-cluster --name my-cluster --query cluster.kubernetesNetworkConfig.serviceIpv6Cidr --output text +---- + +include::deploy-ipv6-cluster.adoc[leveloffset=+1] diff --git a/latest/ug/networking/cni-network-policy-configure.adoc b/latest/ug/networking/cni-network-policy-configure.adoc new file mode 100644 index 000000000..470e40ae5 --- /dev/null +++ b/latest/ug/networking/cni-network-policy-configure.adoc @@ -0,0 +1,279 @@ +include::../attributes.txt[] + +[.topic] +[#cni-network-policy-configure] += Restrict Pod network traffic with Kubernetes network policies +:info_titleabbrev: Restrict traffic + +[abstract] +-- +Learn how to deploy Kubernetes network policies on your Amazon EKS cluster. +-- + +You can use a Kubernetes network policy to restrict network traffic to and from your Pods. For more information, see https://kubernetes.io/docs/concepts/services-networking/network-policies/[Network Policies] in the Kubernetes documentation. + +You must configure the following in order to use this feature: + +. Set up policy enforcement at Pod startup. You do this in the `aws-node` container of the VPC CNI `DaemonSet`. +. Enable the network policy parameter for the add-on. +. Configure your cluster to use the Kubernetes network policy + +Before you begin, review the considerations. For more information, see <>. + +[#cni-network-policy-prereqs] +== Prerequisites + +The following are prerequisites for the feature: + +[#cni-network-policy-minimum] +=== Minimum cluster version +An existing Amazon EKS cluster. To deploy one, see <>. The cluster must be running one of the Kubernetes versions and platform versions listed in the following table. Note that any Kubernetes and platform versions later than those listed are also supported. You can check your current Kubernetes version by replacing [.replaceable]`my-cluster` in the following command with the name of your cluster and then running the modified command: + +[source,bash,subs="verbatim,attributes"] +---- +aws eks describe-cluster --name my-cluster --query cluster.version --output text +---- + +[%header,cols="2"] +|=== + +|Kubernetes version +|Platform version + +|`1.27.4` +|`eks.5` + +|`1.26.7` +|`eks.6` + +|=== + +[#cni-network-policy-minimum-vpc] +=== Minimum VPC CNI version + +Version `1.14` or later of the Amazon VPC CNI plugin for Kubernetes on your cluster. You can see which version that you currently have with the following command. + +[source,shell,subs="verbatim,attributes"] +---- +kubectl describe daemonset aws-node --namespace kube-system | grep amazon-k8s-cni: | cut -d : -f 3 +---- + +If your version is earlier than `1.14`, see <> to upgrade to version `1.14` or later. + +[#cni-network-policy-minimum-linux] +=== Minimum Linux kernel version +Your nodes must have Linux kernel version `5.10` or later. You can check your kernel version with `uname -r`. If you're using the latest versions of the Amazon EKS optimized Amazon Linux, Amazon EKS optimized accelerated Amazon Linux AMIs, and Bottlerocket AMIs, they already have the required kernel version. + +The Amazon EKS optimized accelerated Amazon Linux AMI version `v20231116` or later have kernel version `5.10`. + +[#cni-network-policy-configure-policy] +== Step 1: Set up policy enforcement at Pod startup + +The Amazon VPC CNI plugin for Kubernetes configures network policies for pods in parallel with the pod provisioning. Until all of the policies are configured for the new pod, containers in the new pod will start with a _default allow policy_. This is called _standard mode_. A default allow policy means that all ingress and egress traffic is allowed to and from the new pods. For example, the pods will not have any firewall rules enforced (all traffic is allowed) until the new pod is updated with the active policies. + +With the `NETWORK_POLICY_ENFORCING_MODE` variable set to `strict`, pods that use the VPC CNI start with a _default deny policy_, then policies are configured. This is called _strict mode_. In strict mode, you must have a network policy for every endpoint that your pods need to access in your cluster. Note that this requirement applies to the CoreDNS pods. The default deny policy isn't configured for pods with Host networking. + +You can change the default network policy by setting the environment variable `NETWORK_POLICY_ENFORCING_MODE` to `strict` in the `aws-node` container of the VPC CNI `DaemonSet`. + +[source,yaml,subs="verbatim,attributes"] +---- +env: + - name: NETWORK_POLICY_ENFORCING_MODE + value: "strict" +---- + + +[#enable-network-policy-parameter] +== Step 2: Enable the network policy parameter for the add-on + +The network policy feature uses port `8162` on the node for metrics by default. Also, the feature used port `8163` for health probes. If you run another application on the nodes or inside pods that needs to use these ports, the app fails to run. In VPC CNI version `v1.14.1` or later, you can change these ports. + +Use the following procedure to enable the network policy parameter for the add-on. + +[#cni-network-policy-console] +=== {aws-management-console} + +. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. +. In the left navigation pane, select *Clusters*, and then select the name of the cluster that you want to configure the Amazon VPC CNI add-on for. +. Choose the *Add-ons* tab. +. Select the box in the top right of the add-on box and then choose *Edit*. +. On the *Configure `Amazon VPC CNI`* page: ++ +.. Select a `v1.14.0-eksbuild.3` or later version in the *Version* list. +.. Expand the *Optional configuration settings*. +.. Enter the JSON key `"enableNetworkPolicy":` and value `"true"` in *Configuration values*. The resulting text must be a valid JSON object. If this key and value are the only data in the text box, surround the key and value with curly braces `{ }`. ++ +The following example has network policy feature enabled and metrics and health probes are set to the default port numbers: ++ +[source,json,subs="verbatim,attributes"] +---- +{ + "enableNetworkPolicy": "true", + "nodeAgent": { + "healthProbeBindAddr": "8163", + "metricsBindAddr": "8162" + } +} +---- + +[#cni-network-helm] +=== Helm + +If you have installed the Amazon VPC CNI plugin for Kubernetes through `helm`, you can update the configuration to change the ports. + +. Run the following command to change the ports. Set the port number in the value for either key `nodeAgent.metricsBindAddr` or key `nodeAgent.healthProbeBindAddr`, respectively. ++ +[source,shell,subs="verbatim,attributes"] +---- +helm upgrade --set nodeAgent.metricsBindAddr=8162 --set nodeAgent.healthProbeBindAddr=8163 aws-vpc-cni --namespace kube-system eks/aws-vpc-cni +---- + +[#cni-network-policy-kubectl] +=== kubectl +. Open the `aws-node` `DaemonSet` in your editor. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl edit daemonset -n kube-system aws-node +---- +. Replace the port numbers in the following command arguments in the `args:` in the `aws-network-policy-agent` container in the VPC CNI `aws-node` daemonset manifest. ++ +[source,yaml,subs="verbatim,attributes"] +---- + - args: + - --metrics-bind-addr=:8162 + - --health-probe-bind-addr=:8163 +---- + + + +[#cni-network-policy-setup] +== Step 3: Configure your cluster to use Kubernetes network policies + +You can set this for an Amazon EKS add-on or self-managed add-on. + + +[#cni-network-policy-setup-procedure-add-on] +.Amazon EKS add-on +[%collapsible] +==== + +Using the {aws} CLI, you can configure the cluster to use Kubernetes network policies by running the following command. Replace `my-cluster` with the name of your cluster and the IAM role ARN with the role that you are using. +[source,shell,subs="verbatim,attributes"] +---- +aws eks update-addon --cluster-name my-cluster --addon-name vpc-cni --addon-version v1.14.0-eksbuild.3 \ + --service-account-role-arn {arn-aws}iam::123456789012:role/AmazonEKSVPCCNIRole \ + --resolve-conflicts PRESERVE --configuration-values '{"enableNetworkPolicy": "true"}' +---- + +To configure this using the {aws} Management Console, follow the below steps: + +. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. +. In the left navigation pane, select *Clusters*, and then select the name of the cluster that you want to configure the Amazon VPC CNI add-on for. +. Choose the *Add-ons* tab. +. Select the box in the top right of the add-on box and then choose *Edit*. +. On the *Configure `Amazon VPC CNI`* page: ++ +.. Select a `v1.14.0-eksbuild.3` or later version in the *Version* list. +.. Expand the *Optional configuration settings*. +.. Enter the JSON key `"enableNetworkPolicy":` and value `"true"` in *Configuration values*. The resulting text must be a valid JSON object. If this key and value are the only data in the text box, surround the key and value with curly braces `{ }`. The following example shows network policy is enabled: ++ +[source,json,subs="verbatim,attributes"] +---- +{ "enableNetworkPolicy": "true" } +---- ++ +The following screenshot shows an example of this scenario. ++ +image::images/console-cni-config-network-policy.png[{aws-management-console} showing the VPC CNI add-on with network policy in the optional configuration.,scaledwidth=80%] + +==== + +[#cni-network-policy-setup-procedure-self-managed-add-on] +.Self-managed add-on +[%collapsible] +==== + +[#cni-network-policy-helm] +[discrete] +=== Helm + +If you have installed the Amazon VPC CNI plugin for Kubernetes through `helm`, you can update the configuration to enable network policy. + +. Run the following command to enable network policy. ++ +[source,shell,subs="verbatim,attributes"] +---- +helm upgrade --set enableNetworkPolicy=true aws-vpc-cni --namespace kube-system eks/aws-vpc-cni +---- + +[#cni-network-policy-setup-kubectl] +[discrete] +=== kubectl + +. Open the `amazon-vpc-cni` `ConfigMap` in your editor. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl edit configmap -n kube-system amazon-vpc-cni -o yaml +---- +. Add the following line to the `data` in the `ConfigMap`. ++ +[source,bash,subs="verbatim,attributes"] +---- +enable-network-policy-controller: "true" +---- ++ +Once you've added the line, your `ConfigMap` should look like the following example. ++ +[source,yaml,subs="verbatim,attributes"] +---- +apiVersion: v1 + kind: ConfigMap + metadata: + name: amazon-vpc-cni + namespace: kube-system + data: + enable-network-policy-controller: "true" +---- +. Open the `aws-node` `DaemonSet` in your editor. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl edit daemonset -n kube-system aws-node +---- +.. Replace the `false` with `true` in the command argument `--enable-network-policy=false` in the `args:` in the `aws-network-policy-agent` container in the VPC CNI `aws-node` daemonset manifest. ++ +[source,yaml,subs="verbatim,attributes"] +---- + - args: + - --enable-network-policy=true +---- + +==== + +[#cni-network-policy-setup-procedure-confirm] +== Step 4. Next steps + +After you complete the configuration, confirm that the `aws-node` pods are running on your cluster. + +[source,bash,subs="verbatim,attributes"] +---- +kubectl get pods -n kube-system | grep 'aws-node\|amazon' +---- + +An example output is as follows. + +[source,bash,subs="verbatim,attributes"] +---- +aws-node-gmqp7 2/2 Running 1 (24h ago) 24h +aws-node-prnsh 2/2 Running 1 (24h ago) 24h +---- + +There are 2 containers in the `aws-node` pods in versions `1.14` and later. In previous versions and if network policy is disabled, there is only a single container in the `aws-node` pods. + +You can now deploy Kubernetes network policies to your cluster. + +To implement Kubernetes network policies you create Kubernetes `NetworkPolicy` objects and deploy them to your cluster. `NetworkPolicy` objects are scoped to a namespace. You implement policies to allow or deny traffic between Pods based on label selectors, namespaces, and IP address ranges. For more information about creating `NetworkPolicy` objects, see https://kubernetes.io/docs/concepts/services-networking/network-policies/#networkpolicy-resource[Network Policies] in the Kubernetes documentation. + +Enforcement of Kubernetes `NetworkPolicy` objects is implemented using the Extended Berkeley Packet Filter (eBPF). Relative to `iptables` based implementations, it offers lower latency and performance characteristics, including reduced CPU utilization and avoiding sequential lookups. Additionally, eBPF probes provide access to context rich data that helps debug complex kernel level issues and improve observability. Amazon EKS supports an eBPF-based exporter that leverages the probes to log policy results on each node and export the data to external log collectors to aid in debugging. For more information, see the https://ebpf.io/what-is-ebpf/#what-is-ebpf[eBPF documentation]. diff --git a/latest/ug/networking/cni-network-policy.adoc b/latest/ug/networking/cni-network-policy.adoc new file mode 100644 index 000000000..a0abee69e --- /dev/null +++ b/latest/ug/networking/cni-network-policy.adoc @@ -0,0 +1,67 @@ +include::../attributes.txt[] + +[.topic] +[#cni-network-policy] += Limit Pod traffic with Kubernetes network policies +:info_titleabbrev: Kubernetes policies + +[abstract] +-- +Learn how to configure your Amazon EKS cluster to use Kubernetes network policies with the Amazon VPC CNI plugin. Control network traffic to and from pods using network policies for enhanced security. Covers network policy considerations, requirements, setup instructions, and troubleshooting tips. +-- + +By default, there are no restrictions in Kubernetes for IP addresses, ports, or connections between any Pods in your cluster or between your Pods and resources in any other network. You can use Kubernetes _network policy_ to restrict network traffic to and from your Pods. For more information, see https://kubernetes.io/docs/concepts/services-networking/network-policies/[Network Policies] in the Kubernetes documentation. + +If you have version `1.13` or earlier of the Amazon VPC CNI plugin for Kubernetes on your cluster, you need to implement a third party solution to apply Kubernetes network policies to your cluster. Version `1.14` or later of the plugin can implement network policies, so you don't need to use a third party solution. In this topic, you learn how to configure your cluster to use Kubernetes network policy on your cluster without using a third party add-on. + +Network policies in the Amazon VPC CNI plugin for Kubernetes are supported in the following configurations. + +* Version 1.14 or later of the Amazon VPC CNI plugin for Kubernetes on your cluster. +* Cluster configured for `IPv4` or `IPv6` addresses. +* You can use network policies with <>. With network policies, you can control all in-cluster communication. With security groups for Pods, you can control access to {aws} services from applications within a Pod. +* You can use network policies with _custom networking_ and _prefix delegation_. + + +[#cni-network-policy-considerations] +== Considerations + +*Architecture* + +* When applying Amazon VPC CNI plugin for Kubernetes network policies to your cluster with the Amazon VPC CNI plugin for Kubernetes , you can apply the policies to Amazon EC2 Linux nodes only. You can't apply the policies to Fargate or Windows nodes. +* Network policies only apply either `IPv4` or `IPv6` addresses, but not both. In an `IPv4` cluster, the VPC CNI assigns `IPv4` address to pods and applies `IPv4` policies. In an `IPv6` cluster, the VPC CNI assigns `IPv6` address to pods and applies `IPv6` policies. Any `IPv4` network policy rules applied to an `IPv6` cluster are ignored. Any `IPv6` network policy rules applied to an `IPv4` cluster are ignored. + +*Network Policies* + +* Network Policies are only applied to Pods that are part of a Deployment. Standalone Pods that don't have a `metadata.ownerReferences` set can't have network policies applied to them. +* You can apply multiple network policies to the same Pod. When two or more policies that select the same Pod are configured, all policies are applied to the Pod. +* The maximum number of combinations of ports and protocols for a single IP address range (CIDR) is 24 across all of your network policies. Selectors such as `namespaceSelector` resolve to one or more CIDRs. If multiple selectors resolve to a single CIDR or you specify the same direct CIDR multiple times in the same or different network policies, these all count toward this limit. +* For any of your Kubernetes services, the service port must be the same as the container port. If you're using named ports, use the same name in the service spec too. + +*Migration* + +* If your cluster is currently using a third party solution to manage Kubernetes network policies, you can use those same policies with the Amazon VPC CNI plugin for Kubernetes. However you must remove your existing solution so that it isn't managing the same policies. + +[WARNING] +==== +We recommend that after you remove a network policy solution, then you replace all of the nodes that had the network policy solution applied to them. This is because the traffic rules might get left behind by a pod of the solution if it exits suddenly. +==== + +*Installation* + +* The network policy feature creates and requires a `PolicyEndpoint` Custom Resource Definition (CRD) called `policyendpoints.networking.k8s.aws`. `PolicyEndpoint` objects of the Custom Resource are managed by Amazon EKS. You shouldn't modify or delete these resources. +* If you run pods that use the instance role IAM credentials or connect to the EC2 IMDS, be careful to check for network policies that would block access to the EC2 IMDS. You may need to add a network policy to allow access to EC2 IMDS. For more information, see link:AWSEC2/latest/UserGuide/ec2-instance-metadata.html[Instance metadata and user data,type="documentation"] in the Amazon EC2 User Guide. ++ +Pods that use _IAM roles for service accounts_ or _EKS Pod Identity_ don't access EC2 IMDS. +* The Amazon VPC CNI plugin for Kubernetes doesn't apply network policies to additional network interfaces for each pod, only the primary interface for each pod (`eth0`). This affects the following architectures: ++ +** `IPv6` pods with the `ENABLE_V4_EGRESS` variable set to `true`. This variable enables the `IPv4` egress feature to connect the IPv6 pods to `IPv4` endpoints such as those outside the cluster. The `IPv4` egress feature works by creating an additional network interface with a local loopback IPv4 address. +** When using chained network plugins such as Multus. Because these plugins add network interfaces to each pod, network policies aren't applied to the chained network plugins. + +include::cni-network-policy-configure.adoc[leveloffset=+1] + +include::network-policy-disable.adoc[leveloffset=+1] + +include::network-policies-troubleshooting.adoc[leveloffset=+1] + +include::network-policy-stars-demo.adoc[leveloffset=+1] + diff --git a/latest/ug/networking/coredns-add-on-create.adoc b/latest/ug/networking/coredns-add-on-create.adoc new file mode 100644 index 000000000..158a42a61 --- /dev/null +++ b/latest/ug/networking/coredns-add-on-create.adoc @@ -0,0 +1,63 @@ +include::../attributes.txt[] + +[.topic] +[#coredns-add-on-create] += Create the CoreDNS Amazon EKS add-on +:info_titleabbrev: Create + +Create the CoreDNS Amazon EKS add-on. You must have a cluster before you create the add-on. For more information, see <>. + +. See which version of the add-on is installed on your cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl describe deployment coredns --namespace kube-system | grep coredns: | cut -d : -f 3 +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +v1.10.1-eksbuild.13 +---- +. See which type of the add-on is installed on your cluster. Depending on the tool that you created your cluster with, you might not currently have the Amazon EKS add-on type installed on your cluster. Replace [.replaceable]`my-cluster` with the name of your cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks describe-addon --cluster-name my-cluster --addon-name coredns --query addon.addonVersion --output text +---- ++ +If a version number is returned, you have the Amazon EKS type of the add-on installed on your cluster and don't need to complete the remaining steps in this procedure. If an error is returned, you don't have the Amazon EKS type of the add-on installed on your cluster. Complete the remaining steps of this procedure to install it. +. Save the configuration of your currently installed add-on. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get deployment coredns -n kube-system -o yaml > aws-k8s-coredns-old.yaml +---- +. Create the add-on using the {aws} CLI. If you want to use the {aws-management-console} or `eksctl` to create the add-on, see <> and specify `coredns` for the add-on name. Copy the command that follows to your device. Make the following modifications to the command, as needed, and then run the modified command. ++ +** Replace [.replaceable]`my-cluster` with the name of your cluster. +** Replace [.replaceable]`v1.11.3-eksbuild.1` with the latest version listed in the <> for your cluster version. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks create-addon --cluster-name my-cluster --addon-name coredns --addon-version v1.11.3-eksbuild.1 +---- ++ +If you've applied custom settings to your current add-on that conflict with the default settings of the Amazon EKS add-on, creation might fail. If creation fails, you receive an error that can help you resolve the issue. Alternatively, you can add `--resolve-conflicts OVERWRITE` to the previous command. This allows the add-on to overwrite any existing custom settings. Once you've created the add-on, you can update it with your custom settings. +. Confirm that the latest version of the add-on for your cluster's Kubernetes version was added to your cluster. Replace [.replaceable]`my-cluster` with the name of your cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks describe-addon --cluster-name my-cluster --addon-name coredns --query addon.addonVersion --output text +---- ++ +It might take several seconds for add-on creation to complete. ++ +An example output is as follows. ++ +[source,json,subs="verbatim,attributes",role="nocopy"] +---- +v1.11.3-eksbuild.1 +---- +. If you made custom settings to your original add-on, before you created the Amazon EKS add-on, use the configuration that you saved in a previous step to update the Amazon EKS add-on with your custom settings. For instructions to update the add-on, see <>. \ No newline at end of file diff --git a/latest/ug/networking/coredns-add-on-self-managed-update.adoc b/latest/ug/networking/coredns-add-on-self-managed-update.adoc new file mode 100644 index 000000000..bef4c0e49 --- /dev/null +++ b/latest/ug/networking/coredns-add-on-self-managed-update.adoc @@ -0,0 +1,129 @@ +include::../attributes.txt[] + +[.topic] +[#coredns-add-on-self-managed-update] += Update the CoreDNS Amazon EKS self-managed add-on +:info_titleabbrev: Update (self-managed) + +[IMPORTANT] +==== + +We recommend adding the Amazon EKS type of the add-on to your cluster instead of using the self-managed type of the add-on. If you're not familiar with the difference between the types, see <>. For more information about adding an Amazon EKS add-on to your cluster, see <>. If you're unable to use the Amazon EKS add-on, we encourage you to submit an issue about why you can't to the https://github.com/aws/containers-roadmap/issues[Containers roadmap GitHub repository]. + +==== + +Before you begin, review the upgrade considerations. For more information, see <>. + +. Confirm that you have the self-managed type of the add-on installed on your cluster. Replace [.replaceable]`my-cluster` with the name of your cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks describe-addon --cluster-name my-cluster --addon-name coredns --query addon.addonVersion --output text +---- ++ +If an error message is returned, you have the self-managed type of the add-on installed on your cluster. Complete the remaining steps in this procedure. If a version number is returned, you have the Amazon EKS type of the add-on installed on your cluster. To update the Amazon EKS type of the add-on, use the procedure in <>, rather than using this procedure. If you're not familiar with the differences between the add-on types, see <>. +. See which version of the container image is currently installed on your cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl describe deployment coredns -n kube-system | grep Image | cut -d ":" -f 3 +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +v1.8.7-eksbuild.2 +---- +. If your current CoreDNS version is `v1.5.0` or later, but earlier than the version listed in the <> table, then skip this step. If your current version is earlier than `1.5.0`, then you need to modify the `ConfigMap` for CoreDNS to use the forward add-on, rather than the proxy add-on. ++ +.. Open the `ConfigMap` with the following command. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl edit configmap coredns -n kube-system +---- +.. Replace `proxy` in the following line with `forward`. Save the file and exit the editor. ++ +[source,bash,subs="verbatim,attributes"] +---- +proxy . /etc/resolv.conf +---- +. If you originally deployed your cluster on Kubernetes `1.17` or earlier, then you may need to remove a discontinued line from your CoreDNS manifest. ++ +IMPORTANT: You must complete this step before updating to CoreDNS version `1.7.0`, but it's recommended that you complete this step even if you're updating to an earlier version. ++ +.. Check to see if your CoreDNS manifest has the line. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get configmap coredns -n kube-system -o jsonpath='{$.data.Corefile}' | grep upstream +---- ++ +If no output is returned, your manifest doesn't have the line and you can skip to the next step to update CoreDNS. If output is returned, then you need to remove the line. +.. Edit the `ConfigMap` with the following command, removing the line in the file that has the word `upstream` in it. Do not change anything else in the file. Once the line is removed, save the changes. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl edit configmap coredns -n kube-system -o yaml +---- +. Retrieve your current CoreDNS image version: ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl describe deployment coredns -n kube-system | grep Image +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +602401143452.dkr.ecr.region-code.amazonaws.com/eks/coredns:v1.8.7-eksbuild.2 +---- +. If you're updating to CoreDNS `1.8.3` or later, then you need to add the `endpointslices` permission to the `system:coredns` Kubernetes `clusterrole`. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl edit clusterrole system:coredns -n kube-system +---- ++ +Add the following lines under the existing permissions lines in the `rules` section of the file. ++ +[source,yaml,subs="verbatim,attributes"] +---- +[...] +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch +[...] +---- +. Update the CoreDNS add-on by replacing [.replaceable]`602401143452` and [.replaceable]`region-code` with the values from the output returned in a previous step. Replace [.replaceable]`v1.11.3-eksbuild.1` with the CoreDNS version listed in the <> for your Kubernetes version. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl set image deployment.apps/coredns -n kube-system coredns=602401143452.dkr.ecr.region-code.amazonaws.com/eks/coredns:v1.11.3-eksbuild.1 +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +deployment.apps/coredns image updated +---- +. Check the container image version again to confirm that it was updated to the version that you specified in the previous step. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl describe deployment coredns -n kube-system | grep Image | cut -d ":" -f 3 +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +v1.11.3-eksbuild.1 +---- \ No newline at end of file diff --git a/latest/ug/networking/coredns-add-on-update.adoc b/latest/ug/networking/coredns-add-on-update.adoc new file mode 100644 index 000000000..455003ba1 --- /dev/null +++ b/latest/ug/networking/coredns-add-on-update.adoc @@ -0,0 +1,76 @@ +include::../attributes.txt[] + +[.topic] +[#coredns-add-on-update] += Update the CoreDNS Amazon EKS add-on +:info_titleabbrev: Update (EKS add-on) + +Update the Amazon EKS type of the add-on. If you haven't added the Amazon EKS add-on to your cluster, either <> or see <>. + +Before you begin, review the upgrade considerations. For more information, see <>. + +. See which version of the add-on is installed on your cluster. Replace [.replaceable]`my-cluster` with your cluster name. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks describe-addon --cluster-name my-cluster --addon-name coredns --query "addon.addonVersion" --output text +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +v1.10.1-eksbuild.13 +---- ++ +If the version returned is the same as the version for your cluster's Kubernetes version in the <>, then you already have the latest version installed on your cluster and don't need to complete the rest of this procedure. If you receive an error, instead of a version number in your output, then you don't have the Amazon EKS type of the add-on installed on your cluster. You need to <> before you can update it with this procedure. +. Save the configuration of your currently installed add-on. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get deployment coredns -n kube-system -o yaml > aws-k8s-coredns-old.yaml +---- +. Update your add-on using the {aws} CLI. If you want to use the {aws-management-console} or `eksctl` to update the add-on, see <>. Copy the command that follows to your device. Make the following modifications to the command, as needed, and then run the modified command. ++ +** Replace [.replaceable]`my-cluster` with the name of your cluster. +** Replace [.replaceable]`v1.11.3-eksbuild.1` with the latest version listed in the <> for your cluster version. +** The `--resolve-conflicts[.replaceable]``PRESERVE``` option preserves existing configuration values for the add-on. If you've set custom values for add-on settings, and you don't use this option, Amazon EKS overwrites your values with its default values. If you use this option, then we recommend testing any field and value changes on a non-production cluster before updating the add-on on your production cluster. If you change this value to `OVERWRITE`, all settings are changed to Amazon EKS default values. If you've set custom values for any settings, they might be overwritten with Amazon EKS default values. If you change this value to `none`, Amazon EKS doesn't change the value of any settings, but the update might fail. If the update fails, you receive an error message to help you resolve the conflict. +** If you're not updating a configuration setting, remove `--configuration-values '{[.replaceable]``"replicaCount":3``}'` from the command. If you're updating a configuration setting, replace [.replaceable]`"replicaCount":3` with the setting that you want to set. In this example, the number of replicas of CoreDNS is set to `3`. The value that you specify must be valid for the configuration schema. If you don't know the configuration schema, run `aws eks describe-addon-configuration --addon-name coredns --addon-version [.replaceable]``v1.11.3-eksbuild.1```, replacing [.replaceable]`v1.11.3-eksbuild.1` with the version number of the add-on that you want to see the configuration for. The schema is returned in the output. If you have any existing custom configuration, want to remove it all, and set the values for all settings back to Amazon EKS defaults, remove [.replaceable]`"replicaCount":3` from the command, so that you have empty `{}`. For more information about CoreDNS settings, see https://kubernetes.io/docs/tasks/administer-cluster/dns-custom-nameservers/[Customizing DNS Service] in the Kubernetes documentation. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks update-addon --cluster-name my-cluster --addon-name coredns --addon-version v1.11.3-eksbuild.1 \ + --resolve-conflicts PRESERVE --configuration-values '{"replicaCount":3}' +---- ++ +It might take several seconds for the update to complete. +. Confirm that the add-on version was updated. Replace [.replaceable]`my-cluster` with the name of your cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks describe-addon --cluster-name my-cluster --addon-name coredns +---- ++ +It might take several seconds for the update to complete. ++ +An example output is as follows. ++ +[source,json,subs="verbatim,attributes",role="nocopy"] +---- +{ + "addon": { + "addonName": "coredns", + "clusterName": "my-cluster", + "status": "ACTIVE", + "addonVersion": "v1.11.3-eksbuild.1", + "health": { + "issues": [] + }, + "addonArn": "{arn-aws}eks:region:111122223333:addon/my-cluster/coredns/d2c34f06-1111-2222-1eb0-24f64ce37fa4", + "createdAt": "2023-03-01T16:41:32.442000+00:00", + "modifiedAt": "2023-03-01T18:16:54.332000+00:00", + "tags": {}, + "configurationValues": "{\"replicaCount\":3}" + } +} +---- \ No newline at end of file diff --git a/latest/ug/networking/coredns-autoscaling.adoc b/latest/ug/networking/coredns-autoscaling.adoc new file mode 100644 index 000000000..c9cf50cae --- /dev/null +++ b/latest/ug/networking/coredns-autoscaling.adoc @@ -0,0 +1,268 @@ +include::../attributes.txt[] + +[.topic] +[#coredns-autoscaling] += Scale CoreDNS Pods for high DNS traffic +:info_titleabbrev: Scale for high traffic + +[abstract] +-- +Learn how the Amazon EKS add-on for CoreDNS autoscales to handle increased load on DNS pods, improving application availability and cluster scalability. +-- + +When you launch an Amazon EKS cluster with at least one node, a Deployment of two replicas of the CoreDNS image are deployed by default, regardless of the number of nodes deployed in your cluster. The CoreDNS Pods provide name resolution for all Pods in the cluster. Applications use name resolution to connect to pods and services in the cluster as well as connecting to services outside the cluster. As the number of requests for name resolution (queries) from pods increase, the CoreDNS pods can get overwhelmed and slow down, and reject requests that the pods can't handle. + +To handle the increased load on the CoreDNS pods, consider an autoscaling system for CoreDNS. Amazon EKS can manage the autoscaling of the CoreDNS Deployment in the EKS Add-on version of CoreDNS. This CoreDNS autoscaler continuously monitors the cluster state, including the number of nodes and CPU cores. Based on that information, the controller will dynamically adapt the number of replicas of the CoreDNS deployment in an EKS cluster. This feature works for CoreDNS `v1.9` and later. For more information about which versions are compatible with CoreDNS Autoscaling, see the following section. + +The system automatically manages CoreDNS replicas using a dynamic formula based on both the number of nodes and CPU cores in the cluster, calculated as the maximum of (numberOfNodes divided by 16) and (numberOfCPUCores divided by 256). It evaluates demand over 10-minute peak periods, scaling up immediately when needed to handle increased DNS query load, while scaling down gradually by reducing replicas by 33% every 3 minutes to maintain system stability and avoid disruption. + +We recommend using this feature in conjunction with other https://aws.github.io/aws-eks-best-practices/cluster-autoscaling/[EKS Cluster Autoscaling best practices] to improve overall application availability and cluster scalability. + +[#coredns-autoscaling-prereqs] +== Prerequisites + +For Amazon EKS to scale your CoreDNS deployment, there are three prerequisites: + +* You must be using the _EKS Add-on_ version of CoreDNS. +* Your cluster must be running at least the minimum cluster versions and platform versions. +* Your cluster must be running at least the minimum version of the EKS Add-on of CoreDNS. + + +[#coredns-autoscaling-cluster-version] +=== Minimum cluster version + +Autoscaling of CoreDNS is done by a new component in the cluster control plane, managed by Amazon EKS. Because of this, you must upgrade your cluster to an EKS release that supports the minimum platform version that has the new component. + +A new Amazon EKS cluster. To deploy one, see <>. The cluster must be running one of the Kubernetes versions and platform versions listed in the following table or a later version. Note that any Kubernetes and platform versions later than those listed are also supported. You can check your current Kubernetes version by replacing [.replaceable]`my-cluster` in the following command with the name of your cluster and then running the modified command: + +[source,bash,subs="verbatim,attributes"] +---- +aws eks describe-cluster --name my-cluster --query cluster.version --output text +---- + +[%header,cols="2"] +|=== +|Kubernetes version +|Platform version + +| Not Listed +| All Platform Versions + + +|`1.29.3` +|`eks.7` + +|`1.28.8` +|`eks.13` + +|`1.27.12` +|`eks.17` + +|`1.26.15` +|`eks.18` + +|=== + +[NOTE] +==== + +Every platform version of later Kubernetes versions are also supported, for example Kubernetes version `1.30` from `eks.1` and on. + +==== + +[#coredns-autoscaling-coredns-version] +=== Minimum EKS Add-on version + +[%header,cols="3"] +|=== + +|Kubernetes version +|1.29 +|1.28 + +| +|`v1.11.1-eksbuild.9` +|`v1.10.1-eksbuild.11` + +|=== + + +[#coredns-autoscaling-console] +.Configuring CoreDNS autoscaling in the {aws-management-console} +[%collapsible] +==== +. Ensure that your cluster is at or above the minimum cluster version. ++ +Amazon EKS upgrades clusters between platform versions of the same Kubernetes version automatically, and you can't start this process yourself. Instead, you can upgrade your cluster to the next Kubernetes version, and the cluster will be upgraded to that K8s version and the latest platform version. ++ +New Kubernetes versions sometimes introduce significant changes. Therefore, we recommend that you test the behavior of your applications by using a separate cluster of the new Kubernetes version before you update your production clusters. ++ +To upgrade a cluster to a new Kubernetes version, follow the procedure in <>. +. Ensure that you have the EKS Add-on for CoreDNS, not the self-managed CoreDNS Deployment. ++ +Depending on the tool that you created your cluster with, you might not currently have the Amazon EKS add-on type installed on your cluster. To see which type of the add-on is installed on your cluster, you can run the following command. Replace `my-cluster` with the name of your cluster. ++ +[source,shell,subs="verbatim,attributes"] +---- +aws eks describe-addon --cluster-name my-cluster --addon-name coredns --query addon.addonVersion --output text +---- ++ +If a version number is returned, you have the Amazon EKS type of the add-on installed on your cluster and you can continue with the next step. If an error is returned, you don't have the Amazon EKS type of the add-on installed on your cluster. Complete the remaining steps of the procedure <> to replace the self-managed version with the Amazon EKS add-on. +. Ensure that your EKS Add-on for CoreDNS is at a version the same or higher than the minimum EKS Add-on version. ++ +See which version of the add-on is installed on your cluster. You can check in the {aws-management-console} or run the following command: ++ +[source,shell,subs="verbatim,attributes"] +---- +kubectl describe deployment coredns --namespace kube-system | grep coredns: | cut -d : -f 3 +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +v1.10.1-eksbuild.13 +---- ++ +Compare this version with the minimum EKS Add-on version in the previous section. If needed, upgrade the EKS Add-on to a higher version by following the procedure <>. +. Add the autoscaling configuration to the *Optional configuration settings* of the EKS Add-on. ++ +.. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. +.. In the left navigation pane, select *Clusters*, and then select the name of the cluster that you want to configure the add-on for. +.. Choose the *Add-ons* tab. +.. Select the box in the top right of the CoreDNS add-on box and then choose *Edit*. +.. On the *Configure CoreDNS* page: ++ +... Select the *Version* that you'd like to use. We recommend that you keep the same version as the previous step, and update the version and configuration in separate actions. +... Expand the *Optional configuration settings*. +... Enter the JSON key `"autoscaling":` and value of a nested JSON object with a key `"enabled":` and value `true` in *Configuration values*. The resulting text must be a valid JSON object. If this key and value are the only data in the text box, surround the key and value with curly braces `{ }`. The following example shows autoscaling is enabled: ++ +[source,json,subs="verbatim,attributes"] +---- +{ + "autoScaling": { + "enabled": true + } +} +---- +... (Optional) You can provide minimum and maximum values that autoscaling can scale the number of CoreDNS pods to. ++ +The following example shows autoscaling is enabled and all of the optional keys have values. We recommend that the minimum number of CoreDNS pods is always greater than 2 to provide resilience for the DNS service in the cluster. ++ +[source,json,subs="verbatim,attributes"] +---- +{ + "autoScaling": { + "enabled": true, + "minReplicas": 2, + "maxReplicas": 10 + } +} +---- +.. To apply the new configuration by replacing the CoreDNS pods, choose *Save changes*. ++ +Amazon EKS applies changes to the EKS Add-ons by using a _rollout_ of the Kubernetes Deployment for CoreDNS. You can track the status of the rollout in the *Update history* of the add-on in the {aws-management-console} and with `kubectl rollout status deployment/coredns --namespace kube-system`. ++ +`kubectl rollout` has the following commands: ++ +[source,shell,subs="verbatim,attributes"] +---- +kubectl rollout + +history -- View rollout history +pause -- Mark the provided resource as paused +restart -- Restart a resource +resume -- Resume a paused resource +status -- Show the status of the rollout +undo -- Undo a previous rollout +---- ++ +If the rollout takes too long, Amazon EKS will undo the rollout, and a message with the type of *Addon Update* and a status of *Failed* will be added to the *Update history* of the add-on. To investigate any issues, start from the history of the rollout, and run `kubectl logs` on a CoreDNS pod to see the logs of CoreDNS. +. If the new entry in the *Update history* has a status of *Successful*, then the rollout has completed and the add-on is using the new configuration in all of the CoreDNS pods. As you change the number of nodes and CPU cores of nodes in the cluster, Amazon EKS scales the number of replicas of the CoreDNS deployment. + +==== + +[#coredns-autoscaling-cli] +.Configuring CoreDNS autoscaling in the {aws} Command Line Interface +[%collapsible] +==== +. Ensure that your cluster is at or above the minimum cluster version. ++ +Amazon EKS upgrades clusters between platform versions of the same Kubernetes version automatically, and you can't start this process yourself. Instead, you can upgrade your cluster to the next Kubernetes version, and the cluster will be upgraded to that K8s version and the latest platform version. ++ +New Kubernetes versions sometimes introduce significant changes. Therefore, we recommend that you test the behavior of your applications by using a separate cluster of the new Kubernetes version before you update your production clusters. ++ +To upgrade a cluster to a new Kubernetes version, follow the procedure in <>. +. Ensure that you have the EKS Add-on for CoreDNS, not the self-managed CoreDNS Deployment. ++ +Depending on the tool that you created your cluster with, you might not currently have the Amazon EKS add-on type installed on your cluster. To see which type of the add-on is installed on your cluster, you can run the following command. Replace `my-cluster` with the name of your cluster. ++ +[source,shell,subs="verbatim,attributes"] +---- +aws eks describe-addon --cluster-name my-cluster --addon-name coredns --query addon.addonVersion --output text +---- ++ +If a version number is returned, you have the Amazon EKS type of the add-on installed on your cluster. If an error is returned, you don't have the Amazon EKS type of the add-on installed on your cluster. Complete the remaining steps of the procedure <> to replace the self-managed version with the Amazon EKS add-on. +. Ensure that your EKS Add-on for CoreDNS is at a version the same or higher than the minimum EKS Add-on version. ++ +See which version of the add-on is installed on your cluster. You can check in the {aws-management-console} or run the following command: ++ +[source,shell,subs="verbatim,attributes"] +---- +kubectl describe deployment coredns --namespace kube-system | grep coredns: | cut -d : -f 3 +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +v1.10.1-eksbuild.13 +---- ++ +Compare this version with the minimum EKS Add-on version in the previous section. If needed, upgrade the EKS Add-on to a higher version by following the procedure <>. +. Add the autoscaling configuration to the *Optional configuration settings* of the EKS Add-on. ++ +Run the following {aws} CLI command. Replace `my-cluster` with the name of your cluster and the IAM role ARN with the role that you are using. ++ +[source,shell,subs="verbatim,attributes"] +---- +aws eks update-addon --cluster-name my-cluster --addon-name coredns \ + --resolve-conflicts PRESERVE --configuration-values '{"autoScaling":{"enabled":true}}' +---- ++ +Amazon EKS applies changes to the EKS Add-ons by using a _rollout_ of the Kubernetes Deployment for CoreDNS. You can track the status of the rollout in the *Update history* of the add-on in the {aws-management-console} and with `kubectl rollout status deployment/coredns --namespace kube-system`. ++ +`kubectl rollout` has the following commands: ++ +[source,shell,subs="verbatim,attributes"] +---- +kubectl rollout + +history -- View rollout history +pause -- Mark the provided resource as paused +restart -- Restart a resource +resume -- Resume a paused resource +status -- Show the status of the rollout +undo -- Undo a previous rollout +---- ++ +If the rollout takes too long, Amazon EKS will undo the rollout, and a message with the type of *Addon Update* and a status of *Failed* will be added to the *Update history* of the add-on. To investigate any issues, start from the history of the rollout, and run `kubectl logs` on a CoreDNS pod to see the logs of CoreDNS. +. (Optional) You can provide minimum and maximum values that autoscaling can scale the number of CoreDNS pods to. ++ +The following example shows autoscaling is enabled and all of the optional keys have values. We recommend that the minimum number of CoreDNS pods is always greater than 2 to provide resilience for the DNS service in the cluster. ++ +[source,shell,subs="verbatim,attributes"] +---- +aws eks update-addon --cluster-name my-cluster --addon-name coredns \ + --resolve-conflicts PRESERVE --configuration-values '{"autoScaling":{"enabled":true,"minReplicas":2,"maxReplicas":10}}' +---- +. Check the status of the update to the add-on by running the following command: ++ +[source,shell,subs="verbatim,attributes"] +---- +aws eks describe-addon --cluster-name my-cluster --addon-name coredns +---- ++ +If you see this line: `"status": "ACTIVE"`, then the rollout has completed and the add-on is using the new configuration in all of the CoreDNS pods. As you change the number of nodes and CPU cores of nodes in the cluster, Amazon EKS scales the number of replicas of the CoreDNS deployment. + +==== diff --git a/latest/ug/networking/coredns-metrics.adoc b/latest/ug/networking/coredns-metrics.adoc new file mode 100644 index 000000000..c46cd65b4 --- /dev/null +++ b/latest/ug/networking/coredns-metrics.adoc @@ -0,0 +1,15 @@ +include::../attributes.txt[] + +[.topic] +[#coredns-metrics] += Monitor Kubernetes DNS resolution with CoreDNS metrics +:info_titleabbrev: Monitor DNS resolution + +[abstract] +-- +Learn how to collect CoreDNS metrics in Amazon EKS using Prometheus or CloudWatch Agent, enabling monitoring and observability for your Kubernetes DNS resolution. +-- + +CoreDNS as an EKS add-on exposes the metrics from CoreDNS on port `9153` in the Prometheus format in the `kube-dns` service. You can use Prometheus, the Amazon CloudWatch agent, or any other compatible system to scrape (collect) these metrics. + +For an example _scrape configuration_ that is compatible with both Prometheus and the CloudWatch agent, see link:AmazonCloudWatch/latest/monitoring/ContainerInsights-Prometheus-Setup-configure.html[CloudWatch agent configuration for Prometheus,type="documentation"] in the _Amazon CloudWatch User Guide_. \ No newline at end of file diff --git a/latest/ug/networking/creating-a-vpc.adoc b/latest/ug/networking/creating-a-vpc.adoc index a776cd768..350160a9a 100644 --- a/latest/ug/networking/creating-a-vpc.adoc +++ b/latest/ug/networking/creating-a-vpc.adoc @@ -1,14 +1,9 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[creating-a-vpc,creating-a-vpc.title]] +[#creating-a-vpc] = Create an Amazon VPC for your Amazon EKS cluster -:info_doctype: section -:info_title: Create an Amazon VPC for your Amazon EKS cluster :info_titleabbrev: Create a VPC -:info_abstract: Learn how to create an Amazon VPC for your cluster using an Amazon EKS provided {aws} CloudFormation \ - template. - -include::../attributes.txt[] [abstract] -- @@ -17,11 +12,11 @@ Learn how to create an Amazon VPC for your cluster using an Amazon EKS provided You can use Amazon Virtual Private Cloud (Amazon VPC) to launch {aws} resources into a virtual network that you've defined. This virtual network closely resembles a traditional network that you might operate in your own data center. However, it comes with the benefits of using the scalable infrastructure of Amazon Web Services. We recommend that you have a thorough understanding of the Amazon VPC service before deploying production Amazon EKS clusters. For more information, see the link:vpc/latest/userguide/[Amazon VPC User Guide,type="documentation"]. -An Amazon EKS cluster, nodes, and [.noloc]`Kubernetes` resources are deployed to a VPC. If you want to use an existing VPC with Amazon EKS, that VPC must meet the requirements that are described in <>. This topic describes how to create a VPC that meets Amazon EKS requirements using an Amazon EKS provided {aws} CloudFormation template. Once you've deployed a template, you can view the resources created by the template to know exactly what resources it created, and the configuration of those resources. If you are using hybrid nodes, your VPC must have routes in its route table for your on-premises network. For more information about the network requirements for hybrid nodes, see <>. +An Amazon EKS cluster, nodes, and Kubernetes resources are deployed to a VPC. If you want to use an existing VPC with Amazon EKS, that VPC must meet the requirements that are described in <>. This topic describes how to create a VPC that meets Amazon EKS requirements using an Amazon EKS provided {aws} CloudFormation template. Once you've deployed a template, you can view the resources created by the template to know exactly what resources it created, and the configuration of those resources. If you are using hybrid nodes, your VPC must have routes in its route table for your on-premises network. For more information about the network requirements for hybrid nodes, see <>. == Prerequisites -To create a VPC for Amazon EKS, you must have the necessary IAM permissions to create Amazon VPC resources. These resources are VPCs, subnets, security groups, route tables and routes, and internet and NAT gateways. For more information, see link:vpc/latest/userguide/vpc-policy-examples.html#vpc-public-subnet-iam[Create a VPC with a public subnet example policy,type="documentation"] in the Amazon VPC User Guide and the full list of link:service-authorization/latest/reference/list_amazonec2.html#amazonec2-actions-as-permissions[Actions, resources, and condition keys for Amazon EC2,type="documentation"] in the link:service-authorization/latest/reference/reference.html[Service Authorization Reference,type="documentation"]. +To create a VPC for Amazon EKS, you must have the necessary IAM permissions to create Amazon VPC resources. These resources are VPCs, subnets, security groups, route tables and routes, and internet and NAT gateways. For more information, see link:vpc/latest/userguide/vpc-policy-examples.html#vpc-public-subnet-iam[Create a VPC with a public subnet example policy,type="documentation"] in the Amazon VPC User Guide and the full list of link:service-authorization/latest/reference/list_amazonec2.html#amazonec2-actions-as-permissions[Actions, resources, and condition keys for Amazon EC2,type="documentation"] in the link:service-authorization/latest/reference/reference.html[Service Authorization Reference,type="documentation"]. You can create a VPC with public and private subnets, only public subnets, or only private subnets. @@ -30,15 +25,15 @@ You can create a VPC with public and private subnets, only public subnets, or on This VPC has two public and two private subnets. A public subnet's associated route table has a route to an internet gateway. However, the route table of a private subnet doesn't have a route to an internet gateway. One public and one private subnet are deployed to the same Availability Zone. The other public and private subnets are deployed to a second Availability Zone in the same {aws} Region. We recommend this option for most deployments. -With this option, you can deploy your nodes to private subnets. This option allows [.noloc]`Kubernetes` to deploy load balancers to the public subnets that can load balance traffic to [.noloc]`Pods` that run on nodes in the private subnets. Public `IPv4` addresses are automatically assigned to nodes that are deployed to public subnets, but public `IPv4` addresses aren't assigned to nodes deployed to private subnets. +With this option, you can deploy your nodes to private subnets. This option allows Kubernetes to deploy load balancers to the public subnets that can load balance traffic to Pods that run on nodes in the private subnets. Public `IPv4` addresses are automatically assigned to nodes that are deployed to public subnets, but public `IPv4` addresses aren't assigned to nodes deployed to private subnets. -You can also assign `IPv6` addresses to nodes in public and private subnets. The nodes in private subnets can communicate with the cluster and other {aws} services. [.noloc]`Pods` can communicate to the internet through a NAT gateway using `IPv4` addresses or outbound-only Internet gateway using `IPv6` addresses deployed in each Availability Zone. A security group is deployed that has rules that deny all inbound traffic from sources other than the cluster or nodes but allows all outbound traffic. The subnets are tagged so that [.noloc]`Kubernetes` can deploy load balancers to them. +You can also assign `IPv6` addresses to nodes in public and private subnets. The nodes in private subnets can communicate with the cluster and other {aws} services. Pods can communicate to the internet through a NAT gateway using `IPv4` addresses or outbound-only Internet gateway using `IPv6` addresses deployed in each Availability Zone. A security group is deployed that has rules that deny all inbound traffic from sources other than the cluster or nodes but allows all outbound traffic. The subnets are tagged so that Kubernetes can deploy load balancers to them. .. Open the link:cloudformation/[{aws} CloudFormation console,type="console"]. .. From the navigation bar, select an {aws} Region that supports Amazon EKS. -.. Choose *Create stack*, *With new resources (standard)*. -.. Under *Prerequisite - Prepare template*, make sure that *Template is ready* is selected and then under *Specify template*, select *Amazon S3 URL*. -.. You can create a VPC that supports only `IPv4`, or a VPC that supports `IPv4` and `IPv6`. Paste one of the following URLs into the text area under *Amazon S3 URL* and choose *Next*: +.. Choose *Create stack*, *With new resources (standard)*. +.. Under *Prerequisite - Prepare template*, make sure that *Template is ready* is selected and then under *Specify template*, select *Amazon S3 URL*. +.. You can create a VPC that supports only `IPv4`, or a VPC that supports `IPv4` and `IPv6`. Paste one of the following URLs into the text area under *Amazon S3 URL* and choose *Next*: *** `IPv4` @@ -52,16 +47,16 @@ https://s3.us-west-2.amazonaws.com/amazon-eks/cloudformation/2020-10-29/amazon-e ---- https://s3.us-west-2.amazonaws.com/amazon-eks/cloudformation/2020-10-29/amazon-eks-ipv6-vpc-public-private-subnets.yaml ---- -.. On the *Specify stack details* page, enter the parameters, and then choose *Next*. +.. On the *Specify stack details* page, enter the parameters, and then choose *Next*. *** *Stack name*: Choose a stack name for your {aws} CloudFormation stack. For example, you can use the template name you used in the previous step. The name can contain only alphanumeric characters (case-sensitive) and hyphens. It must start with an alphanumeric character and can't be longer than 100 characters. The name must be unique within the {aws} Region and {aws} account that you're creating the cluster in. -*** *VpcBlock*: Choose an `IPv4` CIDR range for your VPC. Each node, [.noloc]`Pod`, and load balancer that you deploy is assigned an `IPv4` address from this block. The default `IPv4` values provide enough IP addresses for most implementations, but if it doesn't, then you can change it. For more information, see link:vpc/latest/userguide/VPC_Subnets.html#VPC_Sizing[VPC and subnet sizing,type="documentation"] in the Amazon VPC User Guide. You can also add additional CIDR blocks to the VPC once it's created. If you're creating an `IPv6` VPC, `IPv6` CIDR ranges are automatically assigned for you from Amazon's Global Unicast Address space. +*** *VpcBlock*: Choose an `IPv4` CIDR range for your VPC. Each node, Pod, and load balancer that you deploy is assigned an `IPv4` address from this block. The default `IPv4` values provide enough IP addresses for most implementations, but if it doesn't, then you can change it. For more information, see link:vpc/latest/userguide/VPC_Subnets.html#VPC_Sizing[VPC and subnet sizing,type="documentation"] in the Amazon VPC User Guide. You can also add additional CIDR blocks to the VPC once it's created. If you're creating an `IPv6` VPC, `IPv6` CIDR ranges are automatically assigned for you from Amazon's Global Unicast Address space. *** *PublicSubnet01Block*: Specify an `IPv4` CIDR block for public subnet 1. The default value provides enough IP addresses for most implementations, but if it doesn't, then you can change it. If you're creating an `IPv6` VPC, this block is specified for you within the template. *** *PublicSubnet02Block*: Specify an `IPv4` CIDR block for public subnet 2. The default value provides enough IP addresses for most implementations, but if it doesn't, then you can change it. If you're creating an `IPv6` VPC, this block is specified for you within the template. *** *PrivateSubnet01Block*: Specify an `IPv4` CIDR block for private subnet 1. The default value provides enough IP addresses for most implementations, but if it doesn't, then you can change it. If you're creating an `IPv6` VPC, this block is specified for you within the template. *** *PrivateSubnet02Block*: Specify an `IPv4` CIDR block for private subnet 2. The default value provides enough IP addresses for most implementations, but if it doesn't, then you can change it. If you're creating an `IPv6` VPC, this block is specified for you within the template. -.. (Optional) On the *Configure stack options* page, tag your stack resources and then choose *Next*. -.. On the *Review* page, choose *Create stack*. +.. (Optional) On the *Configure stack options* page, tag your stack resources and then choose *Next*. +.. On the *Review* page, choose *Create stack*. .. When your stack is created, select it in the console and choose *Outputs*. .. Record the *VpcId* for the VPC that was created. You need this when you create your cluster and nodes. .. Record the *SubnetIds* for the subnets that were created and whether you created them as public or private subnets. You need at least two of these when you create your cluster and nodes. @@ -69,62 +64,62 @@ https://s3.us-west-2.amazonaws.com/amazon-eks/cloudformation/2020-10-29/amazon-e ... Open the Amazon VPC console at https://console.aws.amazon.com/vpc/. ... In the left navigation pane, choose *Subnets* -... Select one of your public subnets (*[.replaceable]`stack-name`/SubnetPublic01* or *[.replaceable]`stack-name`/SubnetPublic02* contains the word *public*) and choose *Actions*, *Edit subnet settings*. -... Choose the *Enable auto-assign `*IPv6*` address* check box and then choose *Save*. +... Select one of your public subnets (*[.replaceable]`stack-name`/SubnetPublic01* or *[.replaceable]`stack-name`/SubnetPublic02* contains the word *public*) and choose *Actions*, *Edit subnet settings*. +... Choose the *Enable auto-assign IPv6 address* check box and then choose *Save*. ... Complete the previous steps again for your other public subnet. == Only public subnets -This VPC has three public subnets that are deployed into different Availability Zones in an {aws} Region. All nodes are automatically assigned public `IPv4` addresses and can send and receive internet traffic through an link:vpc/latest/userguide/VPC_Internet_Gateway.html[internet gateway,type="documentation"]. A link:vpc/latest/userguide/VPC_SecurityGroups.html[security group,type="documentation"] is deployed that denies all inbound traffic and allows all outbound traffic. The subnets are tagged so that [.noloc]`Kubernetes` can deploy load balancers to them. +This VPC has three public subnets that are deployed into different Availability Zones in an {aws} Region. All nodes are automatically assigned public `IPv4` addresses and can send and receive internet traffic through an link:vpc/latest/userguide/VPC_Internet_Gateway.html[internet gateway,type="documentation"]. A link:vpc/latest/userguide/VPC_SecurityGroups.html[security group,type="documentation"] is deployed that denies all inbound traffic and allows all outbound traffic. The subnets are tagged so that Kubernetes can deploy load balancers to them. .. Open the link:cloudformation/[{aws} CloudFormation console,type="console"]. .. From the navigation bar, select an {aws} Region that supports Amazon EKS. -.. Choose *Create stack*, *With new resources (standard)*. -.. Under *Prepare template*, make sure that *Template is ready* is selected and then under *Template source*, select *Amazon S3 URL*. -.. Paste the following URL into the text area under *Amazon S3 URL* and choose *Next*: +.. Choose *Create stack*, *With new resources (standard)*. +.. Under *Prepare template*, make sure that *Template is ready* is selected and then under *Template source*, select *Amazon S3 URL*. +.. Paste the following URL into the text area under *Amazon S3 URL* and choose *Next*: [source,none,subs="verbatim,attributes"] ---- https://s3.us-west-2.amazonaws.com/amazon-eks/cloudformation/2020-10-29/amazon-eks-vpc-sample.yaml ---- -.. On the *Specify Details* page, enter the parameters, and then choose *Next*. +.. On the *Specify Details* page, enter the parameters, and then choose *Next*. *** *Stack name*: Choose a stack name for your {aws} CloudFormation stack. For example, you can call it [.replaceable]`amazon-eks-vpc-sample`. The name can contain only alphanumeric characters (case-sensitive) and hyphens. It must start with an alphanumeric character and can't be longer than 100 characters. The name must be unique within the {aws} Region and {aws} account that you're creating the cluster in. -*** *VpcBlock*: Choose a CIDR block for your VPC. Each node, [.noloc]`Pod`, and load balancer that you deploy is assigned an `IPv4` address from this block. The default `IPv4` values provide enough IP addresses for most implementations, but if it doesn't, then you can change it. For more information, see link:vpc/latest/userguide/VPC_Subnets.html#VPC_Sizing[VPC and subnet sizing,type="documentation"] in the Amazon VPC User Guide. You can also add additional CIDR blocks to the VPC once it's created. +*** *VpcBlock*: Choose a CIDR block for your VPC. Each node, Pod, and load balancer that you deploy is assigned an `IPv4` address from this block. The default `IPv4` values provide enough IP addresses for most implementations, but if it doesn't, then you can change it. For more information, see link:vpc/latest/userguide/VPC_Subnets.html#VPC_Sizing[VPC and subnet sizing,type="documentation"] in the Amazon VPC User Guide. You can also add additional CIDR blocks to the VPC once it's created. *** *Subnet01Block*: Specify a CIDR block for subnet 1. The default value provides enough IP addresses for most implementations, but if it doesn't, then you can change it. *** *Subnet02Block*: Specify a CIDR block for subnet 2. The default value provides enough IP addresses for most implementations, but if it doesn't, then you can change it. *** *Subnet03Block*: Specify a CIDR block for subnet 3. The default value provides enough IP addresses for most implementations, but if it doesn't, then you can change it. -.. (Optional) On the *Options* page, tag your stack resources. Choose *Next*. -.. On the *Review* page, choose *Create*. +.. (Optional) On the *Options* page, tag your stack resources. Choose *Next*. +.. On the *Review* page, choose *Create*. .. When your stack is created, select it in the console and choose *Outputs*. .. Record the *VpcId* for the VPC that was created. You need this when you create your cluster and nodes. .. Record the *SubnetIds* for the subnets that were created. You need at least two of these when you create your cluster and nodes. -.. (Optional) Any cluster that you deploy to this VPC can assign private `IPv4` addresses to your [.noloc]`Pods` and [.noloc]`services`. If you want to deploy clusters to this VPC to assign private `IPv6` addresses to your [.noloc]`Pods` and [.noloc]`services`, make updates to your VPC, subnet, route tables, and security groups. For more information, see link:vpc/latest/userguide/vpc-migrate-ipv6.html[Migrate existing VPCs from IPv4 to IPv6,type="documentation"] in the Amazon VPC User Guide. Amazon EKS requires that your subnets have the `Auto-assign` `IPv6` addresses option enabled. By default, it's disabled. +.. (Optional) Any cluster that you deploy to this VPC can assign private `IPv4` addresses to your Pods and services. If you want to deploy clusters to this VPC to assign private `IPv6` addresses to your Pods and services, make updates to your VPC, subnet, route tables, and security groups. For more information, see link:vpc/latest/userguide/vpc-migrate-ipv6.html[Migrate existing VPCs from IPv4 to IPv6,type="documentation"] in the Amazon VPC User Guide. Amazon EKS requires that your subnets have the `Auto-assign` `IPv6` addresses option enabled. By default, it's disabled. == Only private subnets -This VPC has three private subnets that are deployed into different Availability Zones in the {aws} Region. Resources that are deployed to the subnets can't access the internet, nor can the internet access resources in the subnets. The template creates link:vpc/latest/privatelink/privatelink-access-aws-services.html[VPC endpoints,type="documentation"] using {aws} PrivateLink for several {aws} services that nodes typically need to access. If your nodes need outbound internet access, you can add a public link:vpc/latest/userguide/vpc-nat-gateway.html[NAT gateway,type="documentation"] in the Availability Zone of each subnet after the VPC is created. A link:vpc/latest/userguide/VPC_SecurityGroups.html[security group,type="documentation"] is created that denies all inbound traffic, except from resources deployed into the subnets. A security group also allows all outbound traffic. The subnets are tagged so that [.noloc]`Kubernetes` can deploy internal load balancers to them. If you're creating a VPC with this configuration, see <> for additional requirements and considerations. +This VPC has three private subnets that are deployed into different Availability Zones in the {aws} Region. Resources that are deployed to the subnets can't access the internet, nor can the internet access resources in the subnets. The template creates link:vpc/latest/privatelink/privatelink-access-aws-services.html[VPC endpoints,type="documentation"] using {aws} PrivateLink for several {aws} services that nodes typically need to access. If your nodes need outbound internet access, you can add a public link:vpc/latest/userguide/vpc-nat-gateway.html[NAT gateway,type="documentation"] in the Availability Zone of each subnet after the VPC is created. A link:vpc/latest/userguide/VPC_SecurityGroups.html[security group,type="documentation"] is created that denies all inbound traffic, except from resources deployed into the subnets. A security group also allows all outbound traffic. The subnets are tagged so that Kubernetes can deploy internal load balancers to them. If you're creating a VPC with this configuration, see <> for additional requirements and considerations. .. Open the link:cloudformation/[{aws} CloudFormation console,type="console"]. .. From the navigation bar, select an {aws} Region that supports Amazon EKS. -.. Choose *Create stack*, *With new resources (standard)*. -.. Under *Prepare template*, make sure that *Template is ready* is selected and then under *Template source*, select *Amazon S3 URL*. -.. Paste the following URL into the text area under *Amazon S3 URL* and choose *Next*: +.. Choose *Create stack*, *With new resources (standard)*. +.. Under *Prepare template*, make sure that *Template is ready* is selected and then under *Template source*, select *Amazon S3 URL*. +.. Paste the following URL into the text area under *Amazon S3 URL* and choose *Next*: [source,none,subs="verbatim,attributes"] ---- https://s3.us-west-2.amazonaws.com/amazon-eks/cloudformation/2020-10-29/amazon-eks-fully-private-vpc.yaml ---- -.. On the *Specify Details* page, enter the parameters and then choose *Next*. +.. On the *Specify Details* page, enter the parameters and then choose *Next*. *** *Stack name*: Choose a stack name for your {aws} CloudFormation stack. For example, you can call it [.replaceable]`amazon-eks-fully-private-vpc`. The name can contain only alphanumeric characters (case-sensitive) and hyphens. It must start with an alphanumeric character and can't be longer than 100 characters. The name must be unique within the {aws} Region and {aws} account that you're creating the cluster in. -*** *VpcBlock*: Choose a CIDR block for your VPC. Each node, [.noloc]`Pod`, and load balancer that you deploy is assigned an `IPv4` address from this block. The default `IPv4` values provide enough IP addresses for most implementations, but if it doesn't, then you can change it. For more information, see link:vpc/latest/userguide/VPC_Subnets.html#VPC_Sizing[VPC and subnet sizing,type="documentation"] in the Amazon VPC User Guide. You can also add additional CIDR blocks to the VPC once it's created. +*** *VpcBlock*: Choose a CIDR block for your VPC. Each node, Pod, and load balancer that you deploy is assigned an `IPv4` address from this block. The default `IPv4` values provide enough IP addresses for most implementations, but if it doesn't, then you can change it. For more information, see link:vpc/latest/userguide/VPC_Subnets.html#VPC_Sizing[VPC and subnet sizing,type="documentation"] in the Amazon VPC User Guide. You can also add additional CIDR blocks to the VPC once it's created. *** *PrivateSubnet01Block*: Specify a CIDR block for subnet 1. The default value provides enough IP addresses for most implementations, but if it doesn't, then you can change it. *** *PrivateSubnet02Block*: Specify a CIDR block for subnet 2. The default value provides enough IP addresses for most implementations, but if it doesn't, then you can change it. *** *PrivateSubnet03Block*: Specify a CIDR block for subnet 3. The default value provides enough IP addresses for most implementations, but if it doesn't, then you can change it. -.. (Optional) On the *Options* page, tag your stack resources. Choose *Next*. -.. On the *Review* page, choose *Create*. +.. (Optional) On the *Options* page, tag your stack resources. Choose *Next*. +.. On the *Review* page, choose *Create*. .. When your stack is created, select it in the console and choose *Outputs*. .. Record the *VpcId* for the VPC that was created. You need this when you create your cluster and nodes. .. Record the *SubnetIds* for the subnets that were created. You need at least two of these when you create your cluster and nodes. -.. (Optional) Any cluster that you deploy to this VPC can assign private `IPv4` addresses to your [.noloc]`Pods` and [.noloc]`services`. If you want deploy clusters to this VPC to assign private `IPv6` addresses to your [.noloc]`Pods` and [.noloc]`services`, make updates to your VPC, subnet, route tables, and security groups. For more information, see link:vpc/latest/userguide/vpc-migrate-ipv6.html[Migrate existing VPCs from IPv4 to IPv6,type="documentation"] in the Amazon VPC User Guide. Amazon EKS requires that your subnets have the `Auto-assign IPv6` addresses option enabled (it's disabled by default). +.. (Optional) Any cluster that you deploy to this VPC can assign private `IPv4` addresses to your Pods and services. If you want deploy clusters to this VPC to assign private `IPv6` addresses to your Pods and services, make updates to your VPC, subnet, route tables, and security groups. For more information, see link:vpc/latest/userguide/vpc-migrate-ipv6.html[Migrate existing VPCs from IPv4 to IPv6,type="documentation"] in the Amazon VPC User Guide. Amazon EKS requires that your subnets have the `Auto-assign IPv6` addresses option enabled (it's disabled by default). \ No newline at end of file diff --git a/latest/ug/networking/deploy-ipv6-cluster.adoc b/latest/ug/networking/deploy-ipv6-cluster.adoc new file mode 100644 index 000000000..631aa78df --- /dev/null +++ b/latest/ug/networking/deploy-ipv6-cluster.adoc @@ -0,0 +1,452 @@ +include::../attributes.txt[] + +[.topic] +[#deploy-ipv6-cluster] += Deploying an Amazon EKS `IPv6` cluster and managed Amazon Linux nodes +:info_titleabbrev: Deploy + +In this tutorial, you deploy an `IPv6` Amazon VPC, an Amazon EKS cluster with the `IPv6` family, and a managed node group with Amazon EC2 Amazon Linux nodes. You can't deploy Amazon EC2 Windows nodes in an `IPv6` cluster. You can also deploy Fargate nodes to your cluster, though those instructions aren't provided in this topic for simplicity. + +== Prerequisites + +Complete the following before you start the tutorial: + +Install and configure the following tools and resources that you need to create and manage an Amazon EKS cluster. + +* We recommend that you familiarize yourself with all settings and deploy a cluster with the settings that meet your requirements. For more information, see <>, <>, and the <> for this topic. You can only enable some settings when creating your cluster. +* The `kubectl` command line tool is installed on your device or {aws} CloudShell. The version can be the same as or up to one minor version earlier or later than the Kubernetes version of your cluster. For example, if your cluster version is `1.29`, you can use `kubectl` version `1.28`, `1.29`, or `1.30` with it. To install or upgrade `kubectl`, see <>. +* The IAM security principal that you're using must have permissions to work with Amazon EKS IAM roles, service linked roles, {aws} CloudFormation, a VPC, and related resources. For more information, see link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html[Actions, resources, and condition keys for Amazon Elastic Kubernetes Service,type="documentation"] and link:IAM/latest/UserGuide/using-service-linked-roles.html[Using service-linked roles,type="documentation"] in the IAM User Guide. +* If you use the eksctl, install version `{eksctl-min-version}` or later on your computer. To install or update to it, see https://eksctl.io/installation[Installation] in the `eksctl` documentation. +* Version `2.12.3` or later or version `1.27.160` or later of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device or {aws} CloudShell. To check your current version, use `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or Homebrew for macOS are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the _{aws} Command Line Interface User Guide_. The {aws} CLI version that is installed in {aws} CloudShell might also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the _{aws} CloudShell User Guide_. If you use the {aws} CloudShell, you may need to link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[install version 2.12.3 or later or 1.27.160 or later of the {aws} CLI,type="documentation"], because the default {aws} CLI version installed in the {aws} CloudShell may be an earlier version. + + +//[#deploy-ipv6-cluster-procedure] +//====== Procedure + +You can use the eksctl or CLI to deploy an `IPv6` cluster. + + +== Deploy an IPv6 cluster with eksctl + +.. Create the `ipv6-cluster.yaml` file. Copy the command that follows to your device. Make the following modifications to the command as needed and then run the modified command: ++ +*** Replace [.replaceable]`my-cluster` with a name for your cluster. The name can contain only alphanumeric characters (case-sensitive) and hyphens. It must start with an alphanumeric character and can't be longer than 100 characters. The name must be unique within the {aws} Region and {aws} account that you're creating the cluster in. +*** Replace [.replaceable]`region-code` with any {aws} Region that is supported by Amazon EKS. For a list of {aws} Regions, see link:general/latest/gr/eks.html[Amazon EKS endpoints and quotas,type="documentation"] in the {aws} General Reference guide. +*** The value for `version` with the version of your cluster. For more information, see link:eks/latest/userguide/kubernetes-versions.html[Amazon EKS supported versions,type="documentation"]. +*** Replace [.replaceable]`my-nodegroup` with a name for your node group. The node group name can't be longer than 63 characters. It must start with letter or digit, but can also include hyphens and underscores for the remaining characters. +*** Replace [.replaceable]`t3.medium` with any link:AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances[{aws} Nitro System instance type,type="documentation"]. ++ +[source,yaml,subs="verbatim,attributes"] +---- +cat >ipv6-cluster.yaml < +aws-node-t74jh 1/1 Running 0 5m32s 2600:1f13:b66:8203:4516:2080:8ced:1ca9 ip-192-168-253-70.region-code.compute.internal +coredns-85d5b4454c-cw7w2 1/1 Running 0 56m 2600:1f13:b66:8203:34e5:: ip-192-168-253-70.region-code.compute.internal +coredns-85d5b4454c-tx6n8 1/1 Running 0 56m 2600:1f13:b66:8203:34e5::1 ip-192-168-253-70.region-code.compute.internal +kube-proxy-btpbk 1/1 Running 0 5m36s 2600:1f13:b66:8200:11a5:ade0:c590:6ac8 ip-192-168-34-75.region-code.compute.internal +kube-proxy-jjk2g 1/1 Running 0 5m33s 2600:1f13:b66:8203:4516:2080:8ced:1ca9 ip-192-168-253-70.region-code.compute.internal +---- +.. Confirm that default services are assigned `IPv6` addresses. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get services -n kube-system -o wide +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR +kube-dns ClusterIP fd30:3087:b6c2::a 53/UDP,53/TCP 57m k8s-app=kube-dns +---- +.. (Optional) <> or deploy the <> and a sample application to load balance HTTP applications with <> or network traffic with <> to `IPv6` Pods. +.. After you've finished with the cluster and nodes that you created for this tutorial, you should clean up the resources that you created with the following command. ++ +[source,bash,subs="verbatim,attributes"] +---- +eksctl delete cluster my-cluster +---- + + +== Deploy an IPv6 cluster with {aws} CLI + +[IMPORTANT] +==== +** You must complete all steps in this procedure as the same user. To check the current user, run the following command: ++ +[source,bash,subs="verbatim,attributes"] +---- +aws sts get-caller-identity +---- +** You must complete all steps in this procedure in the same shell. Several steps use variables set in previous steps. Steps that use variables won't function properly if the variable values are set in a different shell. If you use the link:cloudshell/latest/userguide/welcome.html[{aws} CloudShell,type="documentation"] to complete the following procedure, remember that if you don't interact with it using your keyboard or pointer for approximately 20–30 minutes, your shell session ends. Running processes do not count as interactions. +** The instructions are written for the Bash shell, and may need adjusting in other shells. +==== + + +Replace all example values in the steps of this procedure with your own values. + +.. Run the following commands to set some variables used in later steps. Replace [.replaceable]`region-code` with the {aws} Region that you want to deploy your resources in. The value can be any {aws} Region that is supported by Amazon EKS. For a list of {aws} Regions, see link:general/latest/gr/eks.html[Amazon EKS endpoints and quotas,type="documentation"] in the {aws} General Reference guide. Replace [.replaceable]`my-cluster` with a name for your cluster. The name can contain only alphanumeric characters (case-sensitive) and hyphens. It must start with an alphanumeric character and can't be longer than 100 characters. The name must be unique within the {aws} Region and {aws} account that you're creating the cluster in. Replace [.replaceable]`my-nodegroup` with a name for your node group. The node group name can't be longer than 63 characters. It must start with letter or digit, but can also include hyphens and underscores for the remaining characters. Replace [.replaceable]`111122223333` with your account ID. ++ +[source,bash,subs="verbatim,attributes"] +---- +export region_code=region-code +export cluster_name=my-cluster +export nodegroup_name=my-nodegroup +export account_id=111122223333 +---- +.. Create an Amazon VPC with public and private subnets that meets Amazon EKS and `IPv6` requirements. ++ +... Run the following command to set a variable for your {aws} CloudFormation stack name. You can replace [.replaceable]`my-eks-ipv6-vpc` with any name you choose. ++ +[source,bash,subs="verbatim,attributes"] +---- +export vpc_stack_name=my-eks-ipv6-vpc +---- +... Create an `IPv6` VPC using an {aws} CloudFormation template. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws cloudformation create-stack --region $region_code --stack-name $vpc_stack_name \ + --template-url https://s3.us-west-2.amazonaws.com/amazon-eks/cloudformation/2020-10-29/amazon-eks-ipv6-vpc-public-private-subnets.yaml +---- ++ +The stack takes a few minutes to create. Run the following command. Don't continue to the next step until the output of the command is `CREATE_COMPLETE`. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws cloudformation describe-stacks --region $region_code --stack-name $vpc_stack_name --query Stacks[].StackStatus --output text +---- +... Retrieve the IDs of the public subnets that were created. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws cloudformation describe-stacks --region $region_code --stack-name $vpc_stack_name \ + --query='Stacks[].Outputs[?OutputKey==`SubnetsPublic`].OutputValue' --output text +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +subnet-0a1a56c486EXAMPLE,subnet-099e6ca77aEXAMPLE +---- +... Enable the auto-assign `IPv6` address option for the public subnets that were created. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws ec2 modify-subnet-attribute --region $region_code --subnet-id subnet-0a1a56c486EXAMPLE --assign-ipv6-address-on-creation +aws ec2 modify-subnet-attribute --region $region_code --subnet-id subnet-099e6ca77aEXAMPLE --assign-ipv6-address-on-creation +---- +... Retrieve the names of the subnets and security groups created by the template from the deployed {aws} CloudFormation stack and store them in variables for use in a later step. ++ +[source,bash,subs="verbatim,attributes"] +---- +security_groups=$(aws cloudformation describe-stacks --region $region_code --stack-name $vpc_stack_name \ + --query='Stacks[].Outputs[?OutputKey==`SecurityGroups`].OutputValue' --output text) + +public_subnets=$(aws cloudformation describe-stacks --region $region_code --stack-name $vpc_stack_name \ + --query='Stacks[].Outputs[?OutputKey==`SubnetsPublic`].OutputValue' --output text) + +private_subnets=$(aws cloudformation describe-stacks --region $region_code --stack-name $vpc_stack_name \ + --query='Stacks[].Outputs[?OutputKey==`SubnetsPrivate`].OutputValue' --output text) + +subnets=${public_subnets},${private_subnets} +---- +.. Create a cluster IAM role and attach the required Amazon EKS IAM managed policy to it. Kubernetes clusters managed by Amazon EKS make calls to other {aws} services on your behalf to manage the resources that you use with the service. ++ +... Run the following command to create the `eks-cluster-role-trust-policy.json` file. ++ +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:943f588d-abe3-4381-a1be-fb7126177ba7[] +---- +... Run the following command to set a variable for your role name. You can replace [.replaceable]`myAmazonEKSClusterRole` with any name you choose. ++ +[source,bash,subs="verbatim,attributes"] +---- +export cluster_role_name=myAmazonEKSClusterRole +---- +... Create the role. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam create-role --role-name $cluster_role_name --assume-role-policy-document file://"eks-cluster-role-trust-policy.json" +---- +... Retrieve the ARN of the IAM role and store it in a variable for a later step. ++ +[source,bash,subs="verbatim,attributes"] +---- +CLUSTER_IAM_ROLE=$(aws iam get-role --role-name $cluster_role_name --query="Role.Arn" --output text) +---- +... Attach the required Amazon EKS managed IAM policy to the role. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam attach-role-policy --policy-arn {arn-aws}iam::aws:policy/AmazonEKSClusterPolicy --role-name $cluster_role_name +---- +.. Create your cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks create-cluster --region $region_code --name $cluster_name --kubernetes-version 1.XX \ + --role-arn $CLUSTER_IAM_ROLE --resources-vpc-config subnetIds=$subnets,securityGroupIds=$security_groups \ + --kubernetes-network-config ipFamily=ipv6 +---- ++ +... NOTE: You might receive an error that one of the Availability Zones in your request doesn't have sufficient capacity to create an Amazon EKS cluster. If this happens, the error output contains the Availability Zones that can support a new cluster. Retry creating your cluster with at least two subnets that are located in the supported Availability Zones for your account. For more information, see <>. ++ +The cluster takes several minutes to create. Run the following command. Don't continue to the next step until the output from the command is `ACTIVE`. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks describe-cluster --region $region_code --name $cluster_name --query cluster.status +---- +.. Create or update a `kubeconfig` file for your cluster so that you can communicate with your cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks update-kubeconfig --region $region_code --name $cluster_name +---- ++ +By default, the `config` file is created in `~/.kube` or the new cluster's configuration is added to an existing `config` file in `~/.kube`. +.. Create a node IAM role. ++ +... Run the following command to create the `vpc-cni-ipv6-policy.json` file. ++ +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:eed00e6f-0f5b-4778-a47a-4bbbb4c8ec4e[] +---- +... Create the IAM policy. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam create-policy --policy-name AmazonEKS_CNI_IPv6_Policy --policy-document file://vpc-cni-ipv6-policy.json +---- +... Run the following command to create the `node-role-trust-relationship.json` file. ++ +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:8e5d2dad-b475-4def-8cfb-8e38da1fa6b0[] +---- +... Run the following command to set a variable for your role name. You can replace [.replaceable]`AmazonEKSNodeRole` with any name you choose. ++ +[source,bash,subs="verbatim,attributes"] +---- +export node_role_name=AmazonEKSNodeRole +---- +... Create the IAM role. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam create-role --role-name $node_role_name --assume-role-policy-document file://"node-role-trust-relationship.json" +---- +... Attach the IAM policy to the IAM role. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam attach-role-policy --policy-arn {arn-aws}iam::$account_id:policy/AmazonEKS_CNI_IPv6_Policy \ + --role-name $node_role_name +---- ++ +IMPORTANT: For simplicity in this tutorial, the policy is attached to this IAM role. In a production cluster however, we recommend attaching the policy to a separate IAM role. For more information, see <>. +... Attach two required IAM managed policies to the IAM role. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam attach-role-policy --policy-arn {arn-aws}iam::aws:policy/AmazonEKSWorkerNodePolicy \ + --role-name $node_role_name +aws iam attach-role-policy --policy-arn {arn-aws}iam::aws:policy/AmazonEC2ContainerRegistryReadOnly \ + --role-name $node_role_name +---- +... Retrieve the ARN of the IAM role and store it in a variable for a later step. ++ +[source,bash,subs="verbatim,attributes"] +---- +node_iam_role=$(aws iam get-role --role-name $node_role_name --query="Role.Arn" --output text) +---- +.. Create a managed node group. ++ +... View the IDs of the subnets that you created in a previous step. ++ +[source,bash,subs="verbatim,attributes"] +---- +echo $subnets +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +subnet-0a1a56c486EXAMPLE,subnet-099e6ca77aEXAMPLE,subnet-0377963d69EXAMPLE,subnet-0c05f819d5EXAMPLE +---- +... Create the node group. Replace [.replaceable]`0a1a56c486EXAMPLE`, [.replaceable]`099e6ca77aEXAMPLE`, [.replaceable]`0377963d69EXAMPLE`, and [.replaceable]`0c05f819d5EXAMPLE` with the values returned in the output of the previous step. Be sure to remove the commas between subnet IDs from the previous output in the following command. You can replace [.replaceable]`t3.medium` with any link:AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances[{aws} Nitro System instance type,type="documentation"]. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks create-nodegroup --region $region_code --cluster-name $cluster_name --nodegroup-name $nodegroup_name \ + --subnets subnet-0a1a56c486EXAMPLE subnet-099e6ca77aEXAMPLE subnet-0377963d69EXAMPLE subnet-0c05f819d5EXAMPLE \ + --instance-types t3.medium --node-role $node_iam_role +---- ++ +The node group takes a few minutes to create. Run the following command. Don't proceed to the next step until the output returned is `ACTIVE`. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks describe-nodegroup --region $region_code --cluster-name $cluster_name --nodegroup-name $nodegroup_name \ + --query nodegroup.status --output text +---- +.. Confirm that the default Pods are assigned `IPv6` addresses in the `IP` column. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get pods -n kube-system -o wide +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES +aws-node-rslts 1/1 Running 1 5m36s 2600:1f13:b66:8200:11a5:ade0:c590:6ac8 ip-192-168-34-75.region-code.compute.internal +aws-node-t74jh 1/1 Running 0 5m32s 2600:1f13:b66:8203:4516:2080:8ced:1ca9 ip-192-168-253-70.region-code.compute.internal +coredns-85d5b4454c-cw7w2 1/1 Running 0 56m 2600:1f13:b66:8203:34e5:: ip-192-168-253-70.region-code.compute.internal +coredns-85d5b4454c-tx6n8 1/1 Running 0 56m 2600:1f13:b66:8203:34e5::1 ip-192-168-253-70.region-code.compute.internal +kube-proxy-btpbk 1/1 Running 0 5m36s 2600:1f13:b66:8200:11a5:ade0:c590:6ac8 ip-192-168-34-75.region-code.compute.internal +kube-proxy-jjk2g 1/1 Running 0 5m33s 2600:1f13:b66:8203:4516:2080:8ced:1ca9 ip-192-168-253-70.region-code.compute.internal +---- +.. Confirm that the default services are assigned `IPv6` addresses in the `IP` column. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get services -n kube-system -o wide +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR +kube-dns ClusterIP fd30:3087:b6c2::a 53/UDP,53/TCP 57m k8s-app=kube-dns +---- +.. (Optional) <> or deploy the <> and a sample application to load balance HTTP applications with <> or network traffic with <> to `IPv6` Pods. +.. After you've finished with the cluster and nodes that you created for this tutorial, you should clean up the resources that you created with the following commands. Make sure that you're not using any of the resources outside of this tutorial before deleting them. ++ +... If you're completing this step in a different shell than you completed the previous steps in, set the values of all the variables used in previous steps, replacing the example values with the values you specified when you completed the previous steps. If you're completing this step in the same shell that you completed the previous steps in, skip to the next step. ++ +[source,bash,subs="verbatim,attributes"] +---- +export region_code=region-code +export vpc_stack_name=my-eks-ipv6-vpc +export cluster_name=my-cluster +export nodegroup_name=my-nodegroup +export account_id=111122223333 +export node_role_name=AmazonEKSNodeRole +export cluster_role_name=myAmazonEKSClusterRole +---- +... Delete your node group. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks delete-nodegroup --region $region_code --cluster-name $cluster_name --nodegroup-name $nodegroup_name +---- ++ +Deletion takes a few minutes. Run the following command. Don't proceed to the next step if any output is returned. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks list-nodegroups --region $region_code --cluster-name $cluster_name --query nodegroups --output text +---- +... Delete the cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks delete-cluster --region $region_code --name $cluster_name +---- ++ +The cluster takes a few minutes to delete. Before continuing make sure that the cluster is deleted with the following command. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks describe-cluster --region $region_code --name $cluster_name +---- ++ +Don't proceed to the next step until your output is similar to the following output. ++ +[source,bash,subs="verbatim,attributes"] +---- +An error occurred (ResourceNotFoundException) when calling the DescribeCluster operation: No cluster found for name: my-cluster. +---- +... Delete the IAM resources that you created. Replace [.replaceable]`AmazonEKS_CNI_IPv6_Policy` with the name you chose, if you chose a different name than the one used in previous steps. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam detach-role-policy --role-name $cluster_role_name --policy-arn {arn-aws}iam::aws:policy/AmazonEKSClusterPolicy +aws iam detach-role-policy --role-name $node_role_name --policy-arn {arn-aws}iam::aws:policy/AmazonEKSWorkerNodePolicy +aws iam detach-role-policy --role-name $node_role_name --policy-arn {arn-aws}iam::aws:policy/AmazonEC2ContainerRegistryReadOnly +aws iam detach-role-policy --role-name $node_role_name --policy-arn {arn-aws}iam::$account_id:policy/AmazonEKS_CNI_IPv6_Policy +aws iam delete-policy --policy-arn {arn-aws}iam::$account_id:policy/AmazonEKS_CNI_IPv6_Policy +aws iam delete-role --role-name $cluster_role_name +aws iam delete-role --role-name $node_role_name +---- +... Delete the {aws} CloudFormation stack that created the VPC. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws cloudformation delete-stack --region $region_code --stack-name $vpc_stack_name +---- \ No newline at end of file diff --git a/latest/ug/networking/eks-networking-add-ons.adoc b/latest/ug/networking/eks-networking-add-ons.adoc index 1bdbfbdde..94d7e156e 100644 --- a/latest/ug/networking/eks-networking-add-ons.adoc +++ b/latest/ug/networking/eks-networking-add-ons.adoc @@ -1,4640 +1,65 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[eks-networking-add-ons,eks-networking-add-ons.title]] +[#eks-networking-add-ons] = Manage networking add-ons for Amazon EKS clusters -:info_doctype: section -:info_title: Manage networking add-ons for Amazon EKS \ - clusters :info_titleabbrev: Manage networking add-ons -:info_abstract: Learn how to manage networking add-ons for your Amazon EKS cluster, including built-in \ - components like Amazon VPC CNI plugin for Kubernetes, CoreDNS, and kube-proxy, as well as optional \ - {aws} add-ons for load balancing and service mesh. - - -include::../attributes.txt[] [abstract] -- -Learn how to manage networking add-ons for your Amazon EKS cluster, including built-in components like [.noloc]`Amazon VPC CNI plugin for Kubernetes`, [.noloc]`CoreDNS`, and `kube-proxy`, as well as optional {aws} add-ons for load balancing and service mesh. +Learn how to manage networking add-ons for your Amazon EKS cluster, including built-in components like Amazon VPC CNI plugin for Kubernetes, CoreDNS, and `kube-proxy`, as well as optional {aws} add-ons for load balancing and service mesh. -- Several networking add-ons are available for your Amazon EKS cluster. -[[eks-networking-add-ons-built-in,eks-networking-add-ons-built-in.title]] +[#eks-networking-add-ons-built-in] == Built-in add-ons [NOTE] ==== -If you create clusters in any way except by using the console, each cluster comes with the self-managed versions of the built-in add-ons. The self-managed versions can't be managed from the {aws-management-console}, {aws} Command Line Interface, or SDKs. You manage the configuration and upgrades of self-managed add-ons. +**When you create an EKS cluster:** + +* **Using the {aws} Console**: The built-in add-ons (like CoreDNS, kube-proxy, etc.) are automatically installed as Amazon EKS Add-ons. These can be easily configured and updated through the {aws} Console, CLI, or SDKs. -We recommend adding the Amazon EKS type of the add-on to your cluster instead of using the self-managed type of the add-on. If you create clusters in the console, the Amazon EKS type of these add-ons is installed. +* **Using other methods** (CLI, SDKs, etc.): The same built-in add-ons are installed as self-managed versions that run as regular Kubernetes deployments. These require manual configuration and updates since they can't be managed through {aws} tools. + +We recommend using Amazon EKS Add-ons rather than self-managed versions to simplify add-on management and enable centralized configuration and updates through {aws} services. ==== -*[.noloc]`Amazon VPC CNI plugin for Kubernetes`*:: -This CNI add-on creates elastic network interfaces and attaches them to your Amazon EC2 nodes. The add-on also assigns a private `IPv4` or `IPv6` address from your VPC to each [.noloc]`Pod` and service. This add-on is installed, by default, on your cluster. For more information, see <>. If you are using hybrid nodes, the VPC CNI is still installed by default but it is prevented from running on your hybrid nodes with an anti-affinity rule. For more information about your CNI options for hybrid nodes, see <>. +*Amazon VPC CNI plugin for Kubernetes*:: +This CNI add-on creates elastic network interfaces and attaches them to your Amazon EC2 nodes. The add-on also assigns a private `IPv4` or `IPv6` address from your VPC to each Pod and service. This add-on is installed, by default, on your cluster. For more information, see <>. If you are using hybrid nodes, the VPC CNI is still installed by default but it is prevented from running on your hybrid nodes with an anti-affinity rule. For more information about your CNI options for hybrid nodes, see <>. -*[.noloc]`CoreDNS`*:: -[.noloc]`CoreDNS` is a flexible, extensible DNS server that can serve as the [.noloc]`Kubernetes` cluster DNS. [.noloc]`CoreDNS` provides name resolution for all [.noloc]`Pods` in the cluster. This add-on is installed, by default, on your cluster. For more information, see <>. +*CoreDNS*:: +CoreDNS is a flexible, extensible DNS server that can serve as the Kubernetes cluster DNS. CoreDNS provides name resolution for all Pods in the cluster. This add-on is installed, by default, on your cluster. For more information, see <>. *`kube-proxy`*:: -This add-on maintains network rules on your Amazon EC2 nodes and enables network communication to your [.noloc]`Pods`. This add-on is installed, by default, on your cluster. For more information, see <>. +This add-on maintains network rules on your Amazon EC2 nodes and enables network communication to your Pods. This add-on is installed, by default, on your cluster. For more information, see <>. -[[eks-networking-add-ons-optional,eks-networking-add-ons-optional.title]] +[#eks-networking-add-ons-optional] == Optional {aws} networking add-ons -*[.noloc]`{aws} Load Balancer Controller`*:: -When you deploy [.noloc]`Kubernetes` service objects of type `loadbalancer`, the controller creates {aws} Network Load Balancers . When you create [.noloc]`Kubernetes` ingress objects, the controller creates {aws} Application Load Balancers. We recommend using this controller to provision Network Load Balancers, rather than using the https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/service/annotations/#legacy-cloud-provider[legacy Cloud Provider] controller built-in to [.noloc]`Kubernetes`. For more information, see the https://kubernetes-sigs.github.io/aws-load-balancer-controller[{aws} Load Balancer Controller] documentation. +*{aws} Load Balancer Controller*:: +When you deploy Kubernetes service objects of type `loadbalancer`, the controller creates {aws} Network Load Balancers . When you create Kubernetes ingress objects, the controller creates {aws} Application Load Balancers. We recommend using this controller to provision Network Load Balancers, rather than using the https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/guide/service/annotations/#legacy-cloud-provider[legacy Cloud Provider] controller built-in to Kubernetes. For more information, see the https://kubernetes-sigs.github.io/aws-load-balancer-controller[{aws} Load Balancer Controller] documentation. *{aws} Gateway API Controller*:: -This controller lets you connect services across multiple [.noloc]`Kubernetes` clusters using the https://gateway-api.sigs.k8s.io/[Kubernetes gateway API]. The controller connects [.noloc]`Kubernetes` services running on Amazon EC2 instances, containers, and serverless functions by using the link:vpc-lattice/latest/ug/what-is-vpc-service-network.html[Amazon VPC Lattice,type="documentation"] service. For more information, see the https://www.gateway-api-controller.eks.aws.dev/[{aws} Gateway API Controller] documentation. +This controller lets you connect services across multiple Kubernetes clusters using the https://gateway-api.sigs.k8s.io/[Kubernetes gateway API]. The controller connects Kubernetes services running on Amazon EC2 instances, containers, and serverless functions by using the link:vpc-lattice/latest/ug/what-is-vpc-service-network.html[Amazon VPC Lattice,type="documentation"] service. For more information, see the https://www.gateway-api-controller.eks.aws.dev/[{aws} Gateway API Controller] documentation. For more information about add-ons, see <>. +include::managing-vpc-cni.adoc[leveloffset=+1] -[.topic] -[[managing-vpc-cni,managing-vpc-cni.title]] -== Amazon VPC CNI -:info_title: Assign IPs to [.noloc]`Pods` with the Amazon VPC CNI -:info_titleabbrev: Amazon VPC CNI -:info_abstract: Discover how the [.noloc]`Amazon VPC CNI plugin for Kubernetes` add-on works to assign private IP addresses and create network interfaces for Pods and services in your Amazon EKS cluster. - -[abstract] --- -Discover how the [.noloc]`Amazon VPC CNI plugin for Kubernetes` add-on works to assign private IP addresses and create network interfaces for [.noloc]`Pods` and services in your Amazon EKS cluster. --- - -[TIP] -==== -With Amazon EKS Auto Mode, you don't need to install or upgrade networking add-ons. Auto Mode includes pod networking and load balancing capabilities. - -For more information, see <>. -==== - -The [.noloc]`Amazon VPC CNI plugin for Kubernetes` add-on is deployed on each Amazon EC2 node in your Amazon EKS cluster. The add-on creates link:AWSEC2/latest/UserGuide/using-eni.html[elastic network interfaces,type="documentation"] and attaches them to your Amazon EC2 nodes. The add-on also assigns a private `IPv4` or `IPv6` address from your VPC to each [.noloc]`Pod`. - -A version of the add-on is deployed with each Fargate node in your cluster, but you don't update it on Fargate nodes. Other compatible CNI plugins are available for use on Amazon EKS clusters, but this is the only CNI plugin supported by Amazon EKS for nodes that run on {aws} infrastructure. For more information about the other compatible CNI plugins, see <>. The VPC CNI isn't supported for use with hybrid nodes. For more information about your CNI options for hybrid nodes, see <>. - -The following table lists the latest available version of the Amazon EKS add-on type for each [.noloc]`Kubernetes` version. - -[[vpc-cni-latest-available-version,vpc-cni-latest-available-version.title]] -=== [.noloc]`Amazon VPC CNI` versions - -[options="header"] -|=== -| Kubernetes version | Amazon EKS type of VPC CNI version -| 1.31 | v1.19.0-eksbuild.1 -| 1.30 | v1.19.0-eksbuild.1 -| 1.29 | v1.19.0-eksbuild.1 -| 1.28 | v1.19.0-eksbuild.1 -| 1.27 | v1.19.0-eksbuild.1 -| 1.26 | v1.19.0-eksbuild.1 -| 1.25 | v1.19.0-eksbuild.1 -| 1.24 | v1.19.0-eksbuild.1 -| 1.23 | v1.18.5-eksbuild.1 -|=== - -[IMPORTANT] -==== - -If you're self-managing this add-on, the versions in the table might not be the same as the available self-managed versions. For more information about updating the self-managed type of this add-on, see <>. - -==== - -[IMPORTANT] -==== - -To upgrade to VPC CNI v1.12.0 or later, you must upgrade to VPC CNI v1.7.0 first. We recommend that you update one minor version at a time. - -==== - - -[[manage-vpc-cni-add-on-on-considerations,manage-vpc-cni-add-on-on-considerations.title]] -=== Considerations - -The following are considerations for using the feature. - - - -* Versions are specified as `major-version.minor-version.patch-version-eksbuild.build-number`. -* Check version compatibility for each feature. Some features of each release of the [.noloc]`Amazon VPC CNI plugin for Kubernetes` require certain [.noloc]`Kubernetes` versions. When using different Amazon EKS features, if a specific version of the add-on is required, then it's noted in the feature documentation. Unless you have a specific reason for running an earlier version, we recommend running the latest version. - - -[.topic] -[[vpc-add-on-create,vpc-add-on-create.title]] -=== Creating the Amazon VPC CNI (Amazon EKS add-on) - -Use the following steps to create the [.noloc]`Amazon VPC CNI plugin for Kubernetes` Amazon EKS add-on. - -Before you begin, review the considerations. For more information, see <>. - - -[[vpc-add-on-create-prerequisites,vpc-add-on-create-prerequisites.title]] -==== Prerequisites - -The following are prerequisites for the [.noloc]`Amazon VPC CNI plugin for Kubernetes` Amazon EKS add-on. - - - -* An existing Amazon EKS cluster. To deploy one, see <>. -* An existing {aws} Identity and Access Management (IAM) [.noloc]`OpenID Connect` ([.noloc]`OIDC`) provider for your cluster. To determine whether you already have one, or to create one, see <>. -* An IAM role with the link:aws-managed-policy/latest/reference/AmazonEKS_CNI_Policy.html[AmazonEKS_CNI_Policy,type="documentation"] IAM policy (if your cluster uses the `IPv4` family) or an IPv6 policy (if your cluster uses the `IPv6` family) attached to it. For more information about the VPC CNI role, see <>. For information about the IPv6 policy, see <>. -* If you're using version `1.7.0` or later of the [.noloc]`Amazon VPC CNI plugin for Kubernetes` and you use custom [.noloc]`Pod` security policies, see <> and <>. - -[IMPORTANT] -==== - -[.noloc]`Amazon VPC CNI plugin for Kubernetes` versions `v1.16.0` to `v1.16.1` removed compatibility with [.noloc]`Kubernetes` versions `1.23` and earlier. VPC CNI version `v1.16.2` restores compatibility with [.noloc]`Kubernetes` versions `1.23` and earlier and CNI spec `v0.4.0`. - -[.noloc]`Amazon VPC CNI plugin for Kubernetes` versions `v1.16.0` to `v1.16.1` implement CNI specification version `v1.0.0`. CNI spec `v1.0.0` is supported on EKS clusters that run the [.noloc]`Kubernetes` versions `v1.24` or later. VPC CNI version `v1.16.0` to `v1.16.1` and CNI spec `v1.0.0` aren't supported on [.noloc]`Kubernetes` version `v1.23` or earlier. For more information about `v1.0.0` of the CNI spec, see https://github.com/containernetworking/cni/blob/spec-v1.0.0/SPEC.md[Container Network Interface (CNI) Specification] on [.noloc]`GitHub`. - -==== - - -[[vpc-add-on-create-procedure,vpc-add-on-create-procedure.title]] -==== Procedure - -After you complete the prerequisites, use the following steps to create the add-on. - -. See which version of the add-on is installed on your cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl describe daemonset aws-node --namespace kube-system | grep amazon-k8s-cni: | cut -d : -f 3 ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -v1.16.4-eksbuild.2 ----- -. See which type of the add-on is installed on your cluster. Depending on the tool that you created your cluster with, you might not currently have the Amazon EKS add-on type installed on your cluster. Replace [.replaceable]`my-cluster` with the name of your cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks describe-addon --cluster-name my-cluster --addon-name vpc-cni --query addon.addonVersion --output text ----- -+ -If a version number is returned, you have the Amazon EKS type of the add-on installed on your cluster and don't need to complete the remaining steps in this procedure. If an error is returned, you don't have the Amazon EKS type of the add-on installed on your cluster. Complete the remaining steps of this procedure to install it. -. Save the configuration of your currently installed add-on. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl get daemonset aws-node -n kube-system -o yaml > aws-k8s-cni-old.yaml ----- -. Create the add-on using the {aws} CLI. If you want to use the {aws-management-console} or `eksctl` to create the add-on, see <> and specify `vpc-cni` for the add-on name. Copy the command that follows to your device. Make the following modifications to the command, as needed, and then run the modified command. -+ -** Replace [.replaceable]`my-cluster` with the name of your cluster. -** Replace [.replaceable]`v1.19.0-eksbuild.1` with the latest version listed in the latest version table for your cluster version. For the latest version table, see <>. -** Replace [.replaceable]`111122223333` with your account ID and [.replaceable]`AmazonEKSVPCCNIRole` with the name of an <> that you've created. Specifying a role requires that you have an IAM [.noloc]`OpenID Connect` ([.noloc]`OIDC`) provider for your cluster. To determine whether you have one for your cluster, or to create one, see <>. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks create-addon --cluster-name my-cluster --addon-name vpc-cni --addon-version v1.19.0-eksbuild.1 \ - --service-account-role-arn {arn-aws}iam::111122223333:role/AmazonEKSVPCCNIRole ----- -+ -If you've applied custom settings to your current add-on that conflict with the default settings of the Amazon EKS add-on, creation might fail. If creation fails, you receive an error that can help you resolve the issue. Alternatively, you can add `--resolve-conflicts OVERWRITE` to the previous command. This allows the add-on to overwrite any existing custom settings. Once you've created the add-on, you can update it with your custom settings. -. Confirm that the latest version of the add-on for your cluster's [.noloc]`Kubernetes` version was added to your cluster. Replace [.replaceable]`my-cluster` with the name of your cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks describe-addon --cluster-name my-cluster --addon-name vpc-cni --query addon.addonVersion --output text ----- -+ -It might take several seconds for add-on creation to complete. -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -v1.19.0-eksbuild.1 ----- -. If you made custom settings to your original add-on, before you created the Amazon EKS add-on, use the configuration that you saved in a previous step to update the EKS add-on with your custom settings. Follow the steps in <>. -. (Optional) Install the `cni-metrics-helper` to your cluster. It scrapes elastic network interface and IP address information, aggregates it at a cluster level, and publishes the metrics to Amazon CloudWatch. For more information, see https://github.com/aws/amazon-vpc-cni-k8s/blob/master/cmd/cni-metrics-helper/README.md[cni-metrics-helper] on GitHub. - - -[.topic] -[[vpc-add-on-update,vpc-add-on-update.title]] -=== Updating the Amazon VPC CNI (Amazon EKS add-on) - -Update the Amazon EKS type of the [.noloc]`Amazon VPC CNI plugin for Kubernetes` add-on. If you haven't added the Amazon EKS type of the add-on to your cluster, you can install it by following <>. Or, update the other type of VPC CNI installation by following <>. - -. See which version of the add-on is installed on your cluster. Replace [.replaceable]`my-cluster` with your cluster name. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks describe-addon --cluster-name my-cluster --addon-name vpc-cni --query "addon.addonVersion" --output text ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -v1.16.4-eksbuild.2 ----- -+ -Compare the version with the table of latest versions at <>. If the version returned is the same as the version for your cluster's [.noloc]`Kubernetes` version in the latest version table, then you already have the latest version installed on your cluster and don't need to complete the rest of this procedure. If you receive an error, instead of a version number in your output, then you don't have the Amazon EKS type of the add-on installed on your cluster. You need to create the add-on before you can update it with this procedure. To create the Amazon EKS type of the VPC CNI add-on, you can follow <>. -. Save the configuration of your currently installed add-on. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl get daemonset aws-node -n kube-system -o yaml > aws-k8s-cni-old.yaml ----- -. Update your add-on using the {aws} CLI. If you want to use the {aws-management-console} or `eksctl` to update the add-on, see <>. Copy the command that follows to your device. Make the following modifications to the command, as needed, and then run the modified command. -+ -** Replace [.replaceable]`my-cluster` with the name of your cluster. -** Replace [.replaceable]`v1.19.0-eksbuild.1` with the latest version listed in the latest version table for your cluster version. -** Replace [.replaceable]`111122223333` with your account ID and [.replaceable]`AmazonEKSVPCCNIRole` with the name of an existing IAM role that you've created. To create an IAM role for the VPC CNI, see <>. Specifying a role requires that you have an IAM [.noloc]`OpenID Connect` ([.noloc]`OIDC`) provider for your cluster. To determine whether you have one for your cluster, or to create one, see <>. -** The `--resolve-conflicts PRESERVE` option preserves existing configuration values for the add-on. If you've set custom values for add-on settings, and you don't use this option, Amazon EKS overwrites your values with its default values. If you use this option, then we recommend testing any field and value changes on a non-production cluster before updating the add-on on your production cluster. If you change this value to `OVERWRITE`, all settings are changed to Amazon EKS default values. If you've set custom values for any settings, they might be overwritten with Amazon EKS default values. If you change this value to `none`, Amazon EKS doesn't change the value of any settings, but the update might fail. If the update fails, you receive an error message to help you resolve the conflict. -** If you're not updating a configuration setting, remove `--configuration-values '{[.replaceable]``"env":{"AWS_VPC_K8S_CNI_EXTERNALSNAT":"true"}``}'` from the command. If you're updating a configuration setting, replace [.replaceable]`"env":{"AWS_VPC_K8S_CNI_EXTERNALSNAT":"true"}` with the setting that you want to set. In this example, the `AWS_VPC_K8S_CNI_EXTERNALSNAT` environment variable is set to `true`. The value that you specify must be valid for the configuration schema. If you don't know the configuration schema, run `aws eks describe-addon-configuration --addon-name vpc-cni --addon-version [.replaceable]``v1.19.0-eksbuild.1```, replacing [.replaceable]`v1.19.0-eksbuild.1` with the version number of the add-on that you want to see the configuration for. The schema is returned in the output. If you have any existing custom configuration, want to remove it all, and set the values for all settings back to Amazon EKS defaults, remove [.replaceable]`"env":{"AWS_VPC_K8S_CNI_EXTERNALSNAT":"true"}` from the command, so that you have empty `{}`. For an explanation of each setting, see https://github.com/aws/amazon-vpc-cni-k8s#cni-configuration-variables[CNI Configuration Variables] on GitHub. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks update-addon --cluster-name my-cluster --addon-name vpc-cni --addon-version v1.19.0-eksbuild.1 \ - --service-account-role-arn {arn-aws}iam::111122223333:role/AmazonEKSVPCCNIRole \ - --resolve-conflicts PRESERVE --configuration-values '{"env":{"AWS_VPC_K8S_CNI_EXTERNALSNAT":"true"}}' ----- -+ -It might take several seconds for the update to complete. -. Confirm that the add-on version was updated. Replace [.replaceable]`my-cluster` with the name of your cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks describe-addon --cluster-name my-cluster --addon-name vpc-cni ----- -+ -It might take several seconds for the update to complete. -+ -An example output is as follows. -+ -[source,json,subs="verbatim,attributes"] ----- -{ - "addon": { - "addonName": "vpc-cni", - "clusterName": "my-cluster", - "status": "ACTIVE", - "addonVersion": "v1.19.0-eksbuild.1", - "health": { - "issues": [] - }, - "addonArn": "{arn-aws}eks:region:111122223333:addon/my-cluster/vpc-cni/74c33d2f-b4dc-8718-56e7-9fdfa65d14a9", - "createdAt": "2023-04-12T18:25:19.319000+00:00", - "modifiedAt": "2023-04-12T18:40:28.683000+00:00", - "serviceAccountRoleArn": "{arn-aws}iam::111122223333:role/AmazonEKSVPCCNIRole", - "tags": {}, - "configurationValues": "{\"env\":{\"AWS_VPC_K8S_CNI_EXTERNALSNAT\":\"true\"}}" - } -} ----- - - -[.topic] -[[vpc-add-on-self-managed-update,vpc-add-on-self-managed-update.title]] -=== Updating the Amazon VPC CNI (self-managed add-on) - -[IMPORTANT] -==== - -We recommend adding the Amazon EKS type of the add-on to your cluster instead of using the self-managed type of the add-on. If you're not familiar with the difference between the types, see <>. For more information about adding an Amazon EKS add-on to your cluster, see <>. If you're unable to use the Amazon EKS add-on, we encourage you to submit an issue about why you can't to the https://github.com/aws/containers-roadmap/issues[Containers roadmap GitHub repository]. - -==== -. Confirm that you don't have the Amazon EKS type of the add-on installed on your cluster. Replace [.replaceable]`my-cluster` with the name of your cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks describe-addon --cluster-name my-cluster --addon-name vpc-cni --query addon.addonVersion --output text ----- -+ -If an error message is returned, you don't have the Amazon EKS type of the add-on installed on your cluster. To self-manage the add-on, complete the remaining steps in this procedure to update the add-on. If a version number is returned, you have the Amazon EKS type of the add-on installed on your cluster. To update it, use the procedure in <>, rather than using this procedure. If you're not familiar with the differences between the add-on types, see <>. -. See which version of the container image is currently installed on your cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl describe daemonset aws-node --namespace kube-system | grep amazon-k8s-cni: | cut -d : -f 3 ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -v1.16.4-eksbuild.2 ----- -+ -Your output might not include the build number. -. Backup your current settings so you can configure the same settings once you've updated your version. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl get daemonset aws-node -n kube-system -o yaml > aws-k8s-cni-old.yaml ----- -To review the available versions and familiarize yourself with the changes in the version that you want to update to, see https://github.com/aws/amazon-vpc-cni-k8s/releases[releases] on [.noloc]`GitHub`. Note that we recommend updating to the same `major`.``minor``.``patch`` version listed in the latest available versions table, even if later versions are available on GitHub. For the latest available version table, see <>. The build versions listed in the table aren't specified in the self-managed versions listed on GitHub. Update your version by completing the tasks in one of the following options: -+ -** If you don't have any custom settings for the add-on, then run the command under the `To apply this release:` heading on GitHub for the https://github.com/aws/amazon-vpc-cni-k8s/releases[release] that you're updating to. -** If you have custom settings, download the manifest file with the following command. Change [.replaceable]`https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/v1.19.0/config/master/aws-k8s-cni.yaml` to the URL for the release on GitHub that you're updating to. -+ -[source,bash,subs="verbatim,attributes"] ----- -curl -O https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/v1.19.0/config/master/aws-k8s-cni.yaml ----- -+ -If necessary, modify the manifest with the custom settings from the backup you made in a previous step and then apply the modified manifest to your cluster. If your nodes don't have access to the private Amazon EKS Amazon ECR repositories that the images are pulled from (see the lines that start with `image:` in the manifest), then you'll have to download the images, copy them to your own repository, and modify the manifest to pull the images from your repository. For more information, see <>. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl apply -f aws-k8s-cni.yaml ----- -. Confirm that the new version is now installed on your cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl describe daemonset aws-node --namespace kube-system | grep amazon-k8s-cni: | cut -d : -f 3 ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -v1.19.0 ----- -. (Optional) Install the `cni-metrics-helper` to your cluster. It scrapes elastic network interface and IP address information, aggregates it at a cluster level, and publishes the metrics to Amazon CloudWatch. For more information, see https://github.com/aws/amazon-vpc-cni-k8s/blob/master/cmd/cni-metrics-helper/README.md[cni-metrics-helper] on GitHub. - - -[.topic] -[[cni-iam-role,cni-iam-role.title]] -=== Configure Amazon VPC CNI plugin to use IRSA -:info_doctype: section -:info_title: Configure Amazon VPC CNI plugin to use IRSA -:info_titleabbrev: Configure VPC CNI for IRSA -:info_abstract: Learn how to configure the [.noloc]`Amazon VPC CNI plugin for Kubernetes` to use IAM roles for service accounts (IRSA) for [.noloc]`Pod` networking in Amazon EKS clusters. - -[abstract] --- -Learn how to configure the [.noloc]`Amazon VPC CNI plugin for Kubernetes` to use IAM roles for service accounts (IRSA) for [.noloc]`Pod` networking in Amazon EKS clusters. --- - -The https://github.com/aws/amazon-vpc-cni-k8s[Amazon VPC CNI plugin for Kubernetes] is the networking plugin for [.noloc]`Pod` networking in Amazon EKS clusters. The plugin is responsible for allocating VPC IP addresses to [.noloc]`Kubernetes` nodes and configuring the necessary networking for [.noloc]`Pods` on each node. The plugin: - - -* Requires {aws} Identity and Access Management (IAM) permissions. If your cluster uses the `IPv4` family, the permissions are specified in the ` link:aws-managed-policy/latest/reference/AmazonEKS_CNI_Policy.html[AmazonEKS_CNI_Policy,type="documentation"]` {aws} managed policy.If your cluster uses the `IPv6` family, then the permissions must be added to an IAM policy that you create; for instructions, see <>. You can attach the policy to the Amazon EKS node IAM role, or to a separate IAM role. For instructions to attach the policy to the Amazon EKS node IAM role, see <>. We recommend that you assign it to a separate role, as detailed in this topic. -* Creates and is configured to use a [.noloc]`Kubernetes` service account named `aws-node` when it's deployed. The service account is bound to a [.noloc]`Kubernetes` `clusterrole` named `aws-node`, which is assigned the required [.noloc]`Kubernetes` permissions. - - -[NOTE] -==== - -The [.noloc]`Pods` for the [.noloc]`Amazon VPC CNI plugin for Kubernetes` have access to the permissions assigned to the <>, unless you block access to IMDS. For more information, see https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node[Restrict access to the instance profile assigned to the worker node]. - -==== - -* An existing Amazon EKS cluster. To deploy one, see <>. -* An existing {aws} Identity and Access Management (IAM) [.noloc]`OpenID Connect` ([.noloc]`OIDC`) provider for your cluster. To determine whether you already have one, or to create one, see <>. - - -[[cni-iam-role-create-role,cni-iam-role-create-role.title]] -==== Step 1: Create the [.noloc]`Amazon VPC CNI plugin for Kubernetes` IAM role -. Determine the IP family of your cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks describe-cluster --name my-cluster | grep ipFamily ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -"ipFamily": "ipv4" ----- -+ -The output may return `ipv6` instead. -. Create the IAM role. You can use `eksctl` or `kubectl` and the {aws} CLI to create your IAM role. -+ -eksctl::: -** Create an IAM role and attach the IAM policy to the role with the command that matches the IP family of your cluster. The command creates and deploys an {aws} CloudFormation stack that creates an IAM role, attaches the policy that you specify to it, and annotates the existing `aws-node` [.noloc]`Kubernetes` service account with the ARN of the IAM role that is created. -+ -*** `IPv4` -+ -Replace [.replaceable]`my-cluster` with your own value. -+ -[source,bash,subs="verbatim,attributes"] ----- -eksctl create iamserviceaccount \ - --name aws-node \ - --namespace kube-system \ - --cluster my-cluster \ - --role-name AmazonEKSVPCCNIRole \ - --attach-policy-arn {arn-aws}iam::aws:policy/AmazonEKS_CNI_Policy \ - --override-existing-serviceaccounts \ - --approve ----- -*** `IPv6` -+ -Replace [.replaceable]`my-cluster` with your own value. Replace [.replaceable]`111122223333` with your account ID and replace [.replaceable]`AmazonEKS_CNI_IPv6_Policy` with the name of your `IPv6` policy. If you don't have an `IPv6` policy, see <> to create one. To use `IPv6` with your cluster, it must meet several requirements. For more information, see <>. -+ -[source,bash,subs="verbatim,attributes"] ----- -eksctl create iamserviceaccount \ - --name aws-node \ - --namespace kube-system \ - --cluster my-cluster \ - --role-name AmazonEKSVPCCNIRole \ - --attach-policy-arn {arn-aws}iam::111122223333:policy/AmazonEKS_CNI_IPv6_Policy \ - --override-existing-serviceaccounts \ - --approve ----- - - -kubectl and the {aws} CLI::: -... View your cluster's OIDC provider URL. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks describe-cluster --name my-cluster --query "cluster.identity.oidc.issuer" --output text ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -https://oidc.eks.region-code.amazonaws.com/id/EXAMPLED539D4633E53DE1B71EXAMPLE ----- -+ -If no output is returned, then you must <>. -... Copy the following contents to a file named [.replaceable]`vpc-cni-trust-policy.json`. Replace [.replaceable]`111122223333` with your account ID and [.replaceable]`EXAMPLED539D4633E53DE1B71EXAMPLE` with the output returned in the previous step. Replace [.replaceable]`region-code` with the {aws} Region that your cluster is in. -+ -[source,json,subs="verbatim,attributes"] ----- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "{arn-aws}iam::111122223333:oidc-provider/oidc.eks.region-code.amazonaws.com/id/EXAMPLED539D4633E53DE1B71EXAMPLE" - }, - "Action": "sts:AssumeRoleWithWebIdentity", - "Condition": { - "StringEquals": { - "oidc.eks.region-code.amazonaws.com/id/EXAMPLED539D4633E53DE1B71EXAMPLE:aud": "sts.amazonaws.com", - "oidc.eks.region-code.amazonaws.com/id/EXAMPLED539D4633E53DE1B71EXAMPLE:sub": "system:serviceaccount:kube-system:aws-node" - } - } - } - ] -} ----- -... Create the role. You can replace [.replaceable]`AmazonEKSVPCCNIRole` with any name that you choose. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws iam create-role \ - --role-name AmazonEKSVPCCNIRole \ - --assume-role-policy-document file://"vpc-cni-trust-policy.json" ----- -... Attach the required IAM policy to the role. Run the command that matches the IP family of your cluster. -+ -**** `IPv4` -+ -[source,bash,subs="verbatim,attributes"] ----- -aws iam attach-role-policy \ - --policy-arn {arn-aws}iam::aws:policy/AmazonEKS_CNI_Policy \ - --role-name AmazonEKSVPCCNIRole ----- -**** `IPv6` -+ -Replace [.replaceable]`111122223333` with your account ID and [.replaceable]`AmazonEKS_CNI_IPv6_Policy` with the name of your `IPv6` policy. If you don't have an `IPv6` policy, see <> to create one. To use `IPv6` with your cluster, it must meet several requirements. For more information, see <>. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws iam attach-role-policy \ - --policy-arn {arn-aws}iam::111122223333:policy/AmazonEKS_CNI_IPv6_Policy \ - --role-name AmazonEKSVPCCNIRole ----- -... Run the following command to annotate the `aws-node` service account with the ARN of the IAM role that you created previously. Replace the [.replaceable]`example values` with your own values. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl annotate serviceaccount \ - -n kube-system aws-node \ - eks.amazonaws.com/role-arn={arn-aws}iam::111122223333:role/AmazonEKSVPCCNIRole ----- -. (Optional) Configure the {aws} Security Token Service endpoint type used by your [.noloc]`Kubernetes` service account. For more information, see <>. - - -[[cni-iam-role-redeploy-pods,cni-iam-role-redeploy-pods.title]] -==== Step 2: Re-deploy [.noloc]`Amazon VPC CNI plugin for Kubernetes` [.noloc]`Pods` -. Delete and re-create any existing [.noloc]`Pods` that are associated with the service account to apply the credential environment variables. The annotation is not applied to [.noloc]`Pods` that are currently running without the annotation. The following command deletes the existing `aws-node` [.noloc]`DaemonSet` [.noloc]`Pods` and deploys them with the service account annotation. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl delete Pods -n kube-system -l k8s-app=aws-node ----- -. Confirm that the [.noloc]`Pods` all restarted. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl get pods -n kube-system -l k8s-app=aws-node ----- -. Describe one of the [.noloc]`Pods` and verify that the `AWS_WEB_IDENTITY_TOKEN_FILE` and `AWS_ROLE_ARN` environment variables exist. Replace [.replaceable]`cpjw7` with the name of one of your [.noloc]`Pods` returned in the output of the previous step. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl describe pod -n kube-system aws-node-cpjw7 | grep 'AWS_ROLE_ARN:\|AWS_WEB_IDENTITY_TOKEN_FILE:' ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -AWS_ROLE_ARN: {arn-aws}iam::111122223333:role/AmazonEKSVPCCNIRole - AWS_WEB_IDENTITY_TOKEN_FILE: /var/run/secrets/eks.amazonaws.com/serviceaccount/token - AWS_ROLE_ARN: {arn-aws}iam::111122223333:role/AmazonEKSVPCCNIRole - AWS_WEB_IDENTITY_TOKEN_FILE: /var/run/secrets/eks.amazonaws.com/serviceaccount/token ----- -+ -Two sets of duplicate results are returned because the [.noloc]`Pod` contains two containers. Both containers have the same values. -+ -If your [.noloc]`Pod` is using the {aws} Regional endpoint, then the following line is also returned in the previous output. -+ -[source,bash,subs="verbatim,attributes"] ----- -AWS_STS_REGIONAL_ENDPOINTS=regional ----- - - -[[remove-cni-policy-node-iam-role,remove-cni-policy-node-iam-role.title]] -==== Step 3: Remove the CNI policy from the node IAM role - -If your <> currently has the `AmazonEKS_CNI_Policy` IAM (`IPv4`) policyor an <>attached to it, and you've created a separate IAM role, attached the policy to it instead, and assigned it to the `aws-node` [.noloc]`Kubernetes` service account, then we recommend that you remove the policy from your node role with the {aws} CLI command that matches the IP family of your cluster. Replace [.replaceable]`AmazonEKSNodeRole` with the name of your node role. - - - -* `IPv4` -+ -[source,bash,subs="verbatim,attributes"] ----- -aws iam detach-role-policy --role-name AmazonEKSNodeRole --policy-arn {arn-aws}iam::aws:policy/AmazonEKS_CNI_Policy ----- -* `IPv6` -+ -Replace [.replaceable]`111122223333` with your account ID and [.replaceable]`AmazonEKS_CNI_IPv6_Policy` with the name of your `IPv6` policy. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws iam detach-role-policy --role-name AmazonEKSNodeRole --policy-arn {arn-aws}iam::111122223333:policy/AmazonEKS_CNI_IPv6_Policy ----- - - -[[cni-iam-role-create-ipv6-policy,cni-iam-role-create-ipv6-policy.title]] -==== Create IAM policy for clusters that use the `IPv6` family - -If you created a cluster that uses the `IPv6` family and the cluster has version `1.10.1` or later of the [.noloc]`Amazon VPC CNI plugin for Kubernetes` add-on configured, then you need to create an IAM policy that you can assign to an IAM role. If you have an existing cluster that you didn't configure with the `IPv6` family when you created it, then to use `IPv6`, you must create a new cluster. For more information about using `IPv6` with your cluster, see <>. - -. Copy the following text and save it to a file named `vpc-cni-ipv6-policy.json`. -+ -[source,json,subs="verbatim,attributes"] ----- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "ec2:AssignIpv6Addresses", - "ec2:DescribeInstances", - "ec2:DescribeTags", - "ec2:DescribeNetworkInterfaces", - "ec2:DescribeInstanceTypes" - ], - "Resource": "*" - }, - { - "Effect": "Allow", - "Action": [ - "ec2:CreateTags" - ], - "Resource": [ - "{arn-aws}ec2:*:*:network-interface/*" - ] - } - ] -} ----- -. Create the IAM policy. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws iam create-policy --policy-name AmazonEKS_CNI_IPv6_Policy --policy-document file://vpc-cni-ipv6-policy.json ----- - - -[.topic] -[[pod-networking-use-cases,pod-networking-use-cases.title]] -=== Learn about VPC CNI modes and configuration - -[abstract] --- -Discover how [.noloc]`Amazon VPC CNI plugin for Kubernetes` provides pod networking capabilities and settings for different Amazon EKS node types and use cases, including security groups, [.noloc]`Kubernetes` network policies, custom networking, IPv4, and IPv6 support. --- - -The [.noloc]`Amazon VPC CNI plugin for Kubernetes` provides networking for [.noloc]`Pods`. Use the following table to learn more about the available networking features. - -[cols="1,1", options="header"] -|=== -|Networking feature -|Learn more - - -|Configure your cluster to assign IPv6 addresses to clusters, [.noloc]`Pods`, and services -|<> - -|Use IPv4 Source Network Address Translation for [.noloc]`Pods` -|<> - -|Restrict network traffic to and from your [.noloc]`Pods` -|<> - -|Customize the secondary network interface in nodes -|<> - -|Increase IP addresses for your node -|<> - -|Use security groups for [.noloc]`Pod` network traffic -|<> - -|Use multiple network interfaces for [.noloc]`Pods` -|<> -|=== - -[.topic] -[[cni-ipv6,cni-ipv6.title]] -==== Learn about IPv6 addresses to clusters, [.noloc]`pods`, and services - -[abstract] --- -Learn how to deploy an `IPv6` cluster and nodes with Amazon EKS for assigning `IPv6` addresses to [.noloc]`Pods` and [.noloc]`services` instead of `IPv4`, leveraging IP prefix delegation and the latest [.noloc]`Amazon VPC CNI` plugin. --- - -*Applies to*: [.noloc]`Pods` with Amazon EC2 instances and Fargate [.noloc]`Pods` - -By default, [.noloc]`Kubernetes` assigns `IPv4` addresses to your [.noloc]`Pods` and [.noloc]`services`. Instead of assigning `IPv4` addresses to your [.noloc]`Pods` and [.noloc]`services`, you can configure your cluster to assign `IPv6` addresses to them. Amazon EKS doesn't support dual-stacked [.noloc]`Pods` or [.noloc]`services`, even though [.noloc]`Kubernetes` does in version `1.23` and later. As a result, you can't assign both `IPv4` and `IPv6` addresses to your [.noloc]`Pods` and [.noloc]`services`. - -You select which IP family you want to use for your cluster when you create it. You can't change the family after you create the cluster. - -For a tutorial to deploy an Amazon EKS `IPv6` cluster, see <>. - -//[[ipv6-considerations,ipv6-considerations.title]] -//===== Considerations - -The following are considerations for using the feature: - -===== `IPv6` Feature support - -* *No [.noloc]`Windows` support*: [.noloc]`Windows` [.noloc]`Pods` and [.noloc]`services` aren't supported. -* *Nitro-based EC2 nodes required*: You can only use `IPv6` with {aws} Nitro-based Amazon EC2 or Fargate nodes. -* *EC2 and Fargate nodes supported*: You can use `IPv6` with <> with Amazon EC2 nodes and Fargate nodes. -* *Outposts not supported*: You can't use `IPv6` with <>. -* *FSx for Lustre is not supported*: The <> is not supported. -* *Instance Metadata Service not supported*: Use of the Amazon EC2 link:AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html[Instance Metadata Service,type="documentation"] `IPv6` endpoint is not supported with Amazon EKS. -* *Custom networking not supported*: If you previously used <> to help alleviate IP address exhaustion, you can use `IPv6` instead. You can't use custom networking with `IPv6`. If you use custom networking for network isolation, then you might need to continue to use custom networking and the `IPv4` family for your clusters. - - -===== IP address assignments - -* *Kubernetes services*: Kubernetes services are only assigned an `IPv6` addresses. They aren't assigned IPv4 addresses. -* *Pods*: Pods are assigned an IPv6 address and a host-local IPv4 address. The host-local IPv4 address is assigned by using a host-local CNI plugin chained with VPC CNI and the address is not reported to the Kubernetes control plane. It is only used when a pod needs to communicate with an external IPv4 resources in another Amazon VPC or the internet. The host-local IPv4 address gets SNATed (by VPC CNI) to the primary IPv4 address of the primary ENI of the worker node. -* *Pods and services*: [.noloc]`Pods` and [.noloc]`services` are only assigned an `IPv6` address. They aren't assigned an `IPv4` address. Because [.noloc]`Pods` are able to communicate to `IPv4` endpoints through NAT on the instance itself, link:vpc/latest/userguide/vpc-nat-gateway.html#nat-gateway-nat64-dns64[DNS64 and NAT64,type="documentation"] aren't needed. If the traffic needs a public IP address, the traffic is then source network address translated to a public IP. -* *Routing addresses*: The source `IPv6` address of a [.noloc]`Pod` isn't source network address translated to the `IPv6` address of the node when communicating outside of the VPC. It is routed using an internet gateway or egress-only internet gateway. -* *Nodes*: All nodes are assigned an `IPv4` and `IPv6` address. -* *Fargate [.noloc]`Pods`*: Each Fargate [.noloc]`Pod` receives an `IPv6` address from the CIDR that's specified for the subnet that it's deployed in. The underlying hardware unit that runs Fargate [.noloc]`Pods` gets a unique `IPv4` and `IPv6` address from the CIDRs that are assigned to the subnet that the hardware unit is deployed in. - - -===== How to use `IPv6` with EKS - -* *Create new cluster*: You must create a new cluster and specify that you want to use the `IPv6` family for that cluster. You can't enable the `IPv6` family for a cluster that you updated from a previous version. For instructions on how to create a new cluster, see Considerations . -* *Use recent VPC CNI*: Deploy Amazon VPC CNI version `1.10.1` or later. This version or later is deployed by default. After you deploy the add-on, you can't downgrade your Amazon VPC CNI add-on to a version lower than `1.10.1` without first removing all nodes in all node groups in your cluster. -* *Configure VPC CNI for `IPv6`*: If you use Amazon EC2 nodes, you must configure the Amazon VPC CNI add-on with IP prefix delegation and `IPv6`. If you choose the `IPv6` family when creating your cluster, the `1.10.1` version of the add-on defaults to this configuration. This is the case for both a self-managed or Amazon EKS add-on. For more information about IP prefix delegation, see <>. -* *Configure `IPv4` and `IPv6` addresses*: When you create a cluster, the VPC and subnets that you specify must have an `IPv6` CIDR block that's assigned to the VPC and subnets that you specify. They must also have an `IPv4` CIDR block assigned to them. This is because, even if you only want to use `IPv6`, a VPC still requires an `IPv4` CIDR block to function. For more information, see link:vpc/latest/userguide/working-with-vpcs.html#vpc-associate-ipv6-cidr[Associate an IPv6 CIDR block with your VPC,type="documentation"] in the Amazon VPC User Guide. -* *Auto-assign IPv6 addresses to nodes:* When you create your nodes, you must specify subnets that are configured to auto-assign `IPv6` addresses. Otherwise, you can't deploy your nodes. By default, this configuration is disabled. For more information, see link:vpc/latest/userguide/vpc-ip-addressing.html#subnet-ipv6[Modify the IPv6 addressing attribute for your subnet,type="documentation"] in the Amazon VPC User Guide. -* *Set route tables to use `IPv6`*: The route tables that are assigned to your subnets must have routes for `IPv6` addresses. For more information, see link:vpc/latest/userguide/vpc-migrate-ipv6.html[Migrate to IPv6,type="documentation"] in the Amazon VPC User Guide. -* *Set security groups for `IPv6`*: Your security groups must allow `IPv6` addresses. For more information, see link:vpc/latest/userguide/vpc-migrate-ipv6.html[Migrate to IPv6,type="documentation"] in the Amazon VPC User Guide. -* *Set up load balancer*: Use version `2.3.1` or later of the {aws} Load Balancer Controller to load balance HTTP applications using the <> or network traffic using the <> to `IPv6` [.noloc]`Pods` with either load balancer in IP mode, but not instance mode. For more information, see <>. -* *Add `IPv6` IAM policy*: You must attach an `IPv6` IAM policy to your node IAM or CNI IAM role. Between the two, we recommend that you attach it to a CNI IAM role. For more information, see <> and <>. -* *Evaluate all components*: Perform a thorough evaluation of your applications, Amazon EKS add-ons, and {aws} services that you integrate with before deploying `IPv6` clusters. This is to ensure that everything works as expected with `IPv6`. -* *Add `BootstrapArguments` self-managed node groups*: When creating a self-managed node group in a cluster that uses the `IPv6` family, user-data must include the following `BootstrapArguments` for the https://github.com/awslabs/amazon-eks-ami/blob/main/templates/al2/runtime/bootstrap.sh[bootstrap.sh] file that runs at node start up. Replace [.replaceable]`your-cidr` with the `IPv6` [.noloc]`CIDR` range of your cluster's VPC. -+ -[source,bash,subs="verbatim,attributes"] ----- ---ip-family ipv6 --service-ipv6-cidr your-cidr ----- -+ -If you don't know the `IPv6` `CIDR` range for your cluster, you can see it with the following command (requires the {aws} CLI version `2.4.9` or later). -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks describe-cluster --name my-cluster --query cluster.kubernetesNetworkConfig.serviceIpv6Cidr --output text ----- - - -[.topic] -[[deploy-ipv6-cluster,deploy-ipv6-cluster.title]] -===== Deploying an Amazon EKS `IPv6` cluster and managed Amazon Linux nodes - -In this tutorial, you deploy an `IPv6` Amazon VPC, an Amazon EKS cluster with the `IPv6` family, and a managed node group with Amazon EC2 Amazon Linux nodes. You can't deploy Amazon EC2 [.noloc]`Windows` nodes in an `IPv6` cluster. You can also deploy Fargate nodes to your cluster, though those instructions aren't provided in this topic for simplicity. - -====== Prerequisites - -Complete the following before you start the tutorial: - -Install and configure the following tools and resources that you need to create and manage an Amazon EKS cluster. - -* We recommend that you familiarize yourself with all settings and deploy a cluster with the settings that meet your requirements. For more information, see <>, <>, and the <> for this topic. You can only enable some settings when creating your cluster. -* The `kubectl` command line tool is installed on your device or {aws} CloudShell. The version can be the same as or up to one minor version earlier or later than the [.noloc]`Kubernetes` version of your cluster. For example, if your cluster version is `1.29`, you can use `kubectl` version `1.28`, `1.29`, or `1.30` with it. To install or upgrade `kubectl`, see <>. -* The IAM security principal that you're using must have permissions to work with Amazon EKS IAM roles, service linked roles, {aws} CloudFormation, a VPC, and related resources. For more information, see link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html[Actions, resources, and condition keys for Amazon Elastic Kubernetes Service,type="documentation"] and link:IAM/latest/UserGuide/using-service-linked-roles.html[Using service-linked roles,type="documentation"] in the IAM User Guide. -* If you use the [.noloc]`eksctl`, install version `{eksctl-min-version}` or later on your computer. To install or update to it, see https://eksctl.io/installation[Installation] in the `eksctl` documentation. -* Version `2.12.3` or later or version `1.27.160` or later of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device or {aws} CloudShell. To check your current version, use `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or [.noloc]`Homebrew` for [.noloc]`macOS` are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the _{aws} Command Line Interface User Guide_. The {aws} CLI version that is installed in {aws} CloudShell might also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the _{aws} CloudShell User Guide_. If you use the {aws} CloudShell, you may need to link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[install version 2.12.3 or later or 1.27.160 or later of the {aws} CLI,type="documentation"], because the default {aws} CLI version installed in the {aws} CloudShell may be an earlier version. - - -//[[deploy-ipv6-cluster-procedure,deploy-ipv6-cluster-procedure.title]] -//====== Procedure - -You can use the [.noloc]`eksctl` or CLI to deploy an `IPv6` cluster. - - -====== Deploy an IPv6 cluster with [.noloc]`eksctl` - -.. Create the `ipv6-cluster.yaml` file. Copy the command that follows to your device. Make the following modifications to the command as needed and then run the modified command: -+ -*** Replace [.replaceable]`my-cluster` with a name for your cluster. The name can contain only alphanumeric characters (case-sensitive) and hyphens. It must start with an alphanumeric character and can't be longer than 100 characters. The name must be unique within the {aws} Region and {aws} account that you're creating the cluster in. -*** Replace [.replaceable]`region-code` with any {aws} Region that is supported by Amazon EKS. For a list of {aws} Regions, see link:general/latest/gr/eks.html[Amazon EKS endpoints and quotas,type="documentation"] in the {aws} General Reference guide. -*** The value for `version` with the version of your cluster. For more information, see <>. -*** Replace [.replaceable]`my-nodegroup` with a name for your node group. The node group name can't be longer than 63 characters. It must start with letter or digit, but can also include hyphens and underscores for the remaining characters. -*** Replace [.replaceable]`t3.medium` with any link:AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances[{aws} Nitro System instance type,type="documentation"]. -+ -[source,yaml,subs="verbatim,attributes"] ----- -cat >ipv6-cluster.yaml < -aws-node-t74jh 1/1 Running 0 5m32s 2600:1f13:b66:8203:4516:2080:8ced:1ca9 ip-192-168-253-70.region-code.compute.internal -coredns-85d5b4454c-cw7w2 1/1 Running 0 56m 2600:1f13:b66:8203:34e5:: ip-192-168-253-70.region-code.compute.internal -coredns-85d5b4454c-tx6n8 1/1 Running 0 56m 2600:1f13:b66:8203:34e5::1 ip-192-168-253-70.region-code.compute.internal -kube-proxy-btpbk 1/1 Running 0 5m36s 2600:1f13:b66:8200:11a5:ade0:c590:6ac8 ip-192-168-34-75.region-code.compute.internal -kube-proxy-jjk2g 1/1 Running 0 5m33s 2600:1f13:b66:8203:4516:2080:8ced:1ca9 ip-192-168-253-70.region-code.compute.internal ----- -.. Confirm that default services are assigned `IPv6` addresses. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl get services -n kube-system -o wide ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR -kube-dns ClusterIP fd30:3087:b6c2::a 53/UDP,53/TCP 57m k8s-app=kube-dns ----- -.. (Optional) <> or deploy the <> and a sample application to load balance HTTP applications with <> or network traffic with <> to `IPv6` [.noloc]`Pods`. -.. After you've finished with the cluster and nodes that you created for this tutorial, you should clean up the resources that you created with the following command. -+ -[source,bash,subs="verbatim,attributes"] ----- -eksctl delete cluster my-cluster ----- - - -====== Deploy an IPv6 cluster with {aws} CLI - -[IMPORTANT] -==== -** You must complete all steps in this procedure as the same user. To check the current user, run the following command: -+ -[source,bash,subs="verbatim,attributes"] ----- -aws sts get-caller-identity ----- -** You must complete all steps in this procedure in the same shell. Several steps use variables set in previous steps. Steps that use variables won't function properly if the variable values are set in a different shell. If you use the link:cloudshell/latest/userguide/welcome.html[{aws} CloudShell,type="documentation"] to complete the following procedure, remember that if you don't interact with it using your keyboard or pointer for approximately 20–30 minutes, your shell session ends. Running processes do not count as interactions. -** The instructions are written for the Bash shell, and may need adjusting in other shells. -==== - - -Replace all [.replaceable]`example values` in the steps of this procedure with your own values. - -.. Run the following commands to set some variables used in later steps. Replace [.replaceable]`region-code` with the {aws} Region that you want to deploy your resources in. The value can be any {aws} Region that is supported by Amazon EKS. For a list of {aws} Regions, see link:general/latest/gr/eks.html[Amazon EKS endpoints and quotas,type="documentation"] in the {aws} General Reference guide. Replace [.replaceable]`my-cluster` with a name for your cluster. The name can contain only alphanumeric characters (case-sensitive) and hyphens. It must start with an alphanumeric character and can't be longer than 100 characters. The name must be unique within the {aws} Region and {aws} account that you're creating the cluster in. Replace [.replaceable]`my-nodegroup` with a name for your node group. The node group name can't be longer than 63 characters. It must start with letter or digit, but can also include hyphens and underscores for the remaining characters. Replace [.replaceable]`111122223333` with your account ID. -+ -[source,bash,subs="verbatim,attributes"] ----- -export region_code=region-code -export cluster_name=my-cluster -export nodegroup_name=my-nodegroup -export account_id=111122223333 ----- -.. Create an Amazon VPC with public and private subnets that meets Amazon EKS and `IPv6` requirements. -+ -... Run the following command to set a variable for your {aws} CloudFormation stack name. You can replace [.replaceable]`my-eks-ipv6-vpc` with any name you choose. -+ -[source,bash,subs="verbatim,attributes"] ----- -export vpc_stack_name=my-eks-ipv6-vpc ----- -... Create an `IPv6` VPC using an {aws} CloudFormation template. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws cloudformation create-stack --region $region_code --stack-name $vpc_stack_name \ - --template-url https://s3.us-west-2.amazonaws.com/amazon-eks/cloudformation/2020-10-29/amazon-eks-ipv6-vpc-public-private-subnets.yaml ----- -+ -The stack takes a few minutes to create. Run the following command. Don't continue to the next step until the output of the command is `CREATE_COMPLETE`. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws cloudformation describe-stacks --region $region_code --stack-name $vpc_stack_name --query Stacks[].StackStatus --output text ----- -... Retrieve the IDs of the public subnets that were created. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws cloudformation describe-stacks --region $region_code --stack-name $vpc_stack_name \ - --query='Stacks[].Outputs[?OutputKey==`SubnetsPublic`].OutputValue' --output text ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -subnet-0a1a56c486EXAMPLE,subnet-099e6ca77aEXAMPLE ----- -... Enable the auto-assign `IPv6` address option for the public subnets that were created. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws ec2 modify-subnet-attribute --region $region_code --subnet-id subnet-0a1a56c486EXAMPLE --assign-ipv6-address-on-creation -aws ec2 modify-subnet-attribute --region $region_code --subnet-id subnet-099e6ca77aEXAMPLE --assign-ipv6-address-on-creation ----- -... Retrieve the names of the subnets and security groups created by the template from the deployed {aws} CloudFormation stack and store them in variables for use in a later step. -+ -[source,bash,subs="verbatim,attributes"] ----- -security_groups=$(aws cloudformation describe-stacks --region $region_code --stack-name $vpc_stack_name \ - --query='Stacks[].Outputs[?OutputKey==`SecurityGroups`].OutputValue' --output text) - -public_subnets=$(aws cloudformation describe-stacks --region $region_code --stack-name $vpc_stack_name \ - --query='Stacks[].Outputs[?OutputKey==`SubnetsPublic`].OutputValue' --output text) - -private_subnets=$(aws cloudformation describe-stacks --region $region_code --stack-name $vpc_stack_name \ - --query='Stacks[].Outputs[?OutputKey==`SubnetsPrivate`].OutputValue' --output text) - -subnets=${public_subnets},${private_subnets} ----- -.. Create a cluster IAM role and attach the required Amazon EKS IAM managed policy to it. [.noloc]`Kubernetes` clusters managed by Amazon EKS make calls to other {aws} services on your behalf to manage the resources that you use with the service. -+ -... Run the following command to create the `eks-cluster-role-trust-policy.json` file. -+ -[source,json,subs="verbatim,attributes"] ----- -cat >eks-cluster-role-trust-policy.json <>. -+ -The cluster takes several minutes to create. Run the following command. Don't continue to the next step until the output from the command is `ACTIVE`. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks describe-cluster --region $region_code --name $cluster_name --query cluster.status ----- -.. Create or update a `kubeconfig` file for your cluster so that you can communicate with your cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks update-kubeconfig --region $region_code --name $cluster_name ----- -+ -By default, the `config` file is created in `~/.kube` or the new cluster's configuration is added to an existing `config` file in `~/.kube`. -.. Create a node IAM role. -+ -... Run the following command to create the `vpc-cni-ipv6-policy.json` file. -+ -[source,json,subs="verbatim,attributes"] ----- -cat >vpc-cni-ipv6-policy <node-role-trust-relationship.json <>. -... Attach two required IAM managed policies to the IAM role. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws iam attach-role-policy --policy-arn {arn-aws}iam::aws:policy/AmazonEKSWorkerNodePolicy \ - --role-name $node_role_name -aws iam attach-role-policy --policy-arn {arn-aws}iam::aws:policy/AmazonEC2ContainerRegistryReadOnly \ - --role-name $node_role_name ----- -... Retrieve the ARN of the IAM role and store it in a variable for a later step. -+ -[source,bash,subs="verbatim,attributes"] ----- -node_iam_role=$(aws iam get-role --role-name $node_role_name --query="Role.Arn" --output text) ----- -.. Create a managed node group. -+ -... View the IDs of the subnets that you created in a previous step. -+ -[source,bash,subs="verbatim,attributes"] ----- -echo $subnets ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -subnet-0a1a56c486EXAMPLE,subnet-099e6ca77aEXAMPLE,subnet-0377963d69EXAMPLE,subnet-0c05f819d5EXAMPLE ----- -... Create the node group. Replace [.replaceable]`0a1a56c486EXAMPLE`, [.replaceable]`099e6ca77aEXAMPLE`, [.replaceable]`0377963d69EXAMPLE`, and [.replaceable]`0c05f819d5EXAMPLE` with the values returned in the output of the previous step. Be sure to remove the commas between subnet IDs from the previous output in the following command. You can replace [.replaceable]`t3.medium` with any link:AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances[{aws} Nitro System instance type,type="documentation"]. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks create-nodegroup --region $region_code --cluster-name $cluster_name --nodegroup-name $nodegroup_name \ - --subnets subnet-0a1a56c486EXAMPLE subnet-099e6ca77aEXAMPLE subnet-0377963d69EXAMPLE subnet-0c05f819d5EXAMPLE \ - --instance-types t3.medium --node-role $node_iam_role ----- -+ -The node group takes a few minutes to create. Run the following command. Don't proceed to the next step until the output returned is `ACTIVE`. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks describe-nodegroup --region $region_code --cluster-name $cluster_name --nodegroup-name $nodegroup_name \ - --query nodegroup.status --output text ----- -.. Confirm that the default [.noloc]`Pods` are assigned `IPv6` addresses in the `IP` column. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl get pods -n kube-system -o wide ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES -aws-node-rslts 1/1 Running 1 5m36s 2600:1f13:b66:8200:11a5:ade0:c590:6ac8 ip-192-168-34-75.region-code.compute.internal -aws-node-t74jh 1/1 Running 0 5m32s 2600:1f13:b66:8203:4516:2080:8ced:1ca9 ip-192-168-253-70.region-code.compute.internal -coredns-85d5b4454c-cw7w2 1/1 Running 0 56m 2600:1f13:b66:8203:34e5:: ip-192-168-253-70.region-code.compute.internal -coredns-85d5b4454c-tx6n8 1/1 Running 0 56m 2600:1f13:b66:8203:34e5::1 ip-192-168-253-70.region-code.compute.internal -kube-proxy-btpbk 1/1 Running 0 5m36s 2600:1f13:b66:8200:11a5:ade0:c590:6ac8 ip-192-168-34-75.region-code.compute.internal -kube-proxy-jjk2g 1/1 Running 0 5m33s 2600:1f13:b66:8203:4516:2080:8ced:1ca9 ip-192-168-253-70.region-code.compute.internal ----- -.. Confirm that the default services are assigned `IPv6` addresses in the `IP` column. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl get services -n kube-system -o wide ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR -kube-dns ClusterIP fd30:3087:b6c2::a 53/UDP,53/TCP 57m k8s-app=kube-dns ----- -.. (Optional) <> or deploy the <> and a sample application to load balance HTTP applications with <> or network traffic with <> to `IPv6` [.noloc]`Pods`. -.. After you've finished with the cluster and nodes that you created for this tutorial, you should clean up the resources that you created with the following commands. Make sure that you're not using any of the resources outside of this tutorial before deleting them. -+ -... If you're completing this step in a different shell than you completed the previous steps in, set the values of all the variables used in previous steps, replacing the [.replaceable]`example values` with the values you specified when you completed the previous steps. If you're completing this step in the same shell that you completed the previous steps in, skip to the next step. -+ -[source,bash,subs="verbatim,attributes"] ----- -export region_code=region-code -export vpc_stack_name=my-eks-ipv6-vpc -export cluster_name=my-cluster -export nodegroup_name=my-nodegroup -export account_id=111122223333 -export node_role_name=AmazonEKSNodeRole -export cluster_role_name=myAmazonEKSClusterRole ----- -... Delete your node group. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks delete-nodegroup --region $region_code --cluster-name $cluster_name --nodegroup-name $nodegroup_name ----- -+ -Deletion takes a few minutes. Run the following command. Don't proceed to the next step if any output is returned. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks list-nodegroups --region $region_code --cluster-name $cluster_name --query nodegroups --output text ----- -... Delete the cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks delete-cluster --region $region_code --name $cluster_name ----- -+ -The cluster takes a few minutes to delete. Before continuing make sure that the cluster is deleted with the following command. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks describe-cluster --region $region_code --name $cluster_name ----- -+ -Don't proceed to the next step until your output is similar to the following output. -+ -[source,bash,subs="verbatim,attributes"] ----- -An error occurred (ResourceNotFoundException) when calling the DescribeCluster operation: No cluster found for name: my-cluster. ----- -... Delete the IAM resources that you created. Replace [.replaceable]`AmazonEKS_CNI_IPv6_Policy` with the name you chose, if you chose a different name than the one used in previous steps. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws iam detach-role-policy --role-name $cluster_role_name --policy-arn {arn-aws}iam::aws:policy/AmazonEKSClusterPolicy -aws iam detach-role-policy --role-name $node_role_name --policy-arn {arn-aws}iam::aws:policy/AmazonEKSWorkerNodePolicy -aws iam detach-role-policy --role-name $node_role_name --policy-arn {arn-aws}iam::aws:policy/AmazonEC2ContainerRegistryReadOnly -aws iam detach-role-policy --role-name $node_role_name --policy-arn {arn-aws}iam::$account_id:policy/AmazonEKS_CNI_IPv6_Policy -aws iam delete-policy --policy-arn {arn-aws}iam::$account_id:policy/AmazonEKS_CNI_IPv6_Policy -aws iam delete-role --role-name $cluster_role_name -aws iam delete-role --role-name $node_role_name ----- -... Delete the {aws} CloudFormation stack that created the VPC. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws cloudformation delete-stack --region $region_code --stack-name $vpc_stack_name ----- - - -[.topic] -[[external-snat,external-snat.title]] -==== Enable outbound internet access for [.noloc]`pods` - -[abstract] --- -Learn how Amazon EKS manages external communication for [.noloc]`Pods` using Source Network Address Translation (SNAT), allowing Pods to access internet resources or networks connected via VPC peering, Transit Gateway, or {aws} Direct Connect. --- - -*Applies to*: [.noloc]`Linux` `IPv4` Fargate nodes, [.noloc]`Linux` nodes with Amazon EC2 instances - -If you deployed your cluster using the `IPv6` family, then the information in this topic isn't applicable to your cluster, because `IPv6` addresses are not network translated. For more information about using `IPv6` with your cluster, see <>. - -By default, each [.noloc]`Pod` in your cluster is assigned a link:AWSEC2/latest/UserGuide/using-instance-addressing.html#concepts-private-addresses[private,type="documentation"]``IPv4`` address from a classless inter-domain routing (CIDR) block that is associated with the VPC that the [.noloc]`Pod` is deployed in. [.noloc]`Pods` in the same VPC communicate with each other using these private IP addresses as end points. When a [.noloc]`Pod` communicates to any `IPv4` address that isn't within a CIDR block that's associated to your VPC, the Amazon VPC CNI plugin (for both https://github.com/aws/amazon-vpc-cni-k8s#amazon-vpc-cni-k8s[Linux] or https://github.com/aws/amazon-vpc-cni-plugins/tree/master/plugins/vpc-bridge[Windows]) translates the [.noloc]`Pod's` `IPv4` address to the primary private `IPv4` address of the primary link:AWSEC2/latest/UserGuide/using-eni.html#eni-basics[elastic network interface,type="documentation"] of the node that the [.noloc]`Pod` is running on, by default ^^<>^^. - -[NOTE] -==== - -For [.noloc]`Windows` nodes, there are additional details to consider. By default, the https://github.com/aws/amazon-vpc-cni-plugins/tree/master/plugins/vpc-bridge[VPC CNI plugin for Windows] is defined with a networking configuration in which the traffic to a destination within the same VPC is excluded for SNAT. This means that internal VPC communication has SNAT disabled and the IP address allocated to a [.noloc]`Pod` is routable inside the VPC. But traffic to a destination outside of the VPC has the source [.noloc]`Pod` IP SNAT'ed to the instance ENI's primary IP address. This default configuration for [.noloc]`Windows` ensures that the pod can access networks outside of your VPC in the same way as the host instance. - -==== - -Due to this behavior: - - - -* Your [.noloc]`Pods` can communicate with internet resources only if the node that they're running on has a link:AWSEC2/latest/UserGuide/using-instance-addressing.html#concepts-public-addresses[public,type="documentation"] or link:vpc/latest/userguide/vpc-eips.html[elastic,type="documentation"] IP address assigned to it and is in a link:vpc/latest/userguide/configure-subnets.html#subnet-basics[public subnet,type="documentation"]. A public subnet's associated link:vpc/latest/userguide/VPC_Route_Tables.html[route table,type="documentation"] has a route to an internet gateway. We recommend deploying nodes to private subnets, whenever possible. -* For versions of the plugin earlier than `1.8.0`, resources that are in networks or VPCs that are connected to your cluster VPC using link:vpc/latest/peering/what-is-vpc-peering.html[VPC peering,type="documentation"], a link:whitepapers/latest/aws-vpc-connectivity-options/transit-vpc-option.html[transit VPC,type="documentation"], or link:directconnect/latest/UserGuide/Welcome.html[{aws} Direct Connect,type="documentation"] can't initiate communication to your [.noloc]`Pods` behind secondary elastic network interfaces. Your [.noloc]`Pods` can initiate communication to those resources and receive responses from them, though. - -If either of the following statements are true in your environment, then change the default configuration with the command that follows. - - - -* You have resources in networks or VPCs that are connected to your cluster VPC using link:vpc/latest/peering/what-is-vpc-peering.html[VPC peering,type="documentation"], a link:whitepapers/latest/aws-vpc-connectivity-options/transit-vpc-option.html[transit VPC,type="documentation"], or link:directconnect/latest/UserGuide/Welcome.html[{aws} Direct Connect,type="documentation"] that need to initiate communication with your [.noloc]`Pods` using an `IPv4` address and your plugin version is earlier than `1.8.0`. -* Your [.noloc]`Pods` are in a link:vpc/latest/userguide/configure-subnets.html#subnet-basics[private subnet,type="documentation"] and need to communicate outbound to the internet. The subnet has a route to a link:vpc/latest/userguide/vpc-nat-gateway.html[NAT gateway,type="documentation"]. - - -[source,bash,subs="verbatim,attributes"] ----- -kubectl set env daemonset -n kube-system aws-node AWS_VPC_K8S_CNI_EXTERNALSNAT=true ----- - -[NOTE] -==== - -The `AWS_VPC_K8S_CNI_EXTERNALSNAT` and `AWS_VPC_K8S_CNI_EXCLUDE_SNAT_CIDRS` CNI configuration variables aren't applicable to [.noloc]`Windows` nodes. Disabling SNAT isn't supported for [.noloc]`Windows`. As for excluding a list of `IPv4` CIDRs from SNAT, you can define this by specifying the `ExcludedSnatCIDRs` parameter in the [.noloc]`Windows` bootstrap script. For more information on using this parameter, see <>. - -==== - -[[snat-exception,snat-exception.title]] -===== Host networking - -^^*^^If a [.noloc]`Pod's` spec contains `hostNetwork=true` (default is `false`), then its IP address isn't translated to a different address. This is the case for the `kube-proxy` and [.noloc]`Amazon VPC CNI plugin for Kubernetes` [.noloc]`Pods` that run on your cluster, by default. For these [.noloc]`Pods`, the IP address is the same as the node's primary IP address, so the [.noloc]`Pod's` IP address isn't translated. For more information about a [.noloc]`Pod's` `hostNetwork` setting, see https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#podspec-v1-core[PodSpec v1 core] in the [.noloc]`Kubernetes` API reference. - -[.topic] -[[cni-network-policy,cni-network-policy.title]] -==== Limit [.noloc]`pod` traffic with [.noloc]`Kubernetes` network policies - -[abstract] --- -Learn how to configure your Amazon EKS cluster to use [.noloc]`Kubernetes` network policies with the [.noloc]`Amazon VPC CNI` plugin. Control network traffic to and from pods using network policies for enhanced security. Covers network policy considerations, requirements, setup instructions, and troubleshooting tips. --- - -By default, there are no restrictions in [.noloc]`Kubernetes` for IP addresses, ports, or connections between any [.noloc]`Pods` in your cluster or between your [.noloc]`Pods` and resources in any other network. You can use [.noloc]`Kubernetes` _network policy_ to restrict network traffic to and from your [.noloc]`Pods`. For more information, see https://kubernetes.io/docs/concepts/services-networking/network-policies/[Network Policies] in the [.noloc]`Kubernetes` documentation. - -If you have version `1.13` or earlier of the [.noloc]`Amazon VPC CNI plugin for Kubernetes` on your cluster, you need to implement a third party solution to apply [.noloc]`Kubernetes` network policies to your cluster. Version `1.14` or later of the plugin can implement network policies, so you don't need to use a third party solution. In this topic, you learn how to configure your cluster to use [.noloc]`Kubernetes` network policy on your cluster without using a third party add-on. - -Network policies in the [.noloc]`Amazon VPC CNI plugin for Kubernetes` are supported in the following configurations. - - - -* Amazon EKS clusters of version `1.25` and later. -* Version 1.14 or later of the [.noloc]`Amazon VPC CNI plugin for Kubernetes` on your cluster. -* Cluster configured for `IPv4` or `IPv6` addresses. -* You can use network policies with <>. With network policies, you can control all in-cluster communication. With security groups for [.noloc]`Pods`, you can control access to {aws} services from applications within a [.noloc]`Pod`. -* You can use network policies with _custom networking_ and _prefix delegation_. - - -[[cni-network-policy-considerations,cni-network-policy-considerations.title]] -===== Considerations - -*Architecture* - -* When applying [.noloc]`Amazon VPC CNI plugin for Kubernetes` network policies to your cluster with the [.noloc]`Amazon VPC CNI plugin for Kubernetes` , you can apply the policies to Amazon EC2 Linux nodes only. You can't apply the policies to Fargate or Windows nodes. -* Network policies only apply either `IPv4` or `IPv6` addresses, but not both. In an `IPv4` cluster, the VPC CNI assigns `IPv4` address to pods and applies `IPv4` policies. In an `IPv6` cluster, the VPC CNI assigns `IPv6` address to pods and applies `IPv6` policies. Any `IPv4` network policy rules applied to an `IPv6` cluster are ignored. Any `IPv6` network policy rules applied to an `IPv4` cluster are ignored. - -*Network Policies* - -* Network Policies are only applied to [.noloc]`Pods` that are part of a [.noloc]`Deployment`. Standalone [.noloc]`Pods` that don't have a `metadata.ownerReferences` set can't have network policies applied to them. -* You can apply multiple network policies to the same [.noloc]`Pod`. When two or more policies that select the same [.noloc]`Pod` are configured, all policies are applied to the [.noloc]`Pod`. -* The maximum number of unique combinations of ports for each protocol in each `ingress:` or `egress:` selector in a network policy is 24. -* For any of your [.noloc]`Kubernetes` services, the service port must be the same as the container port. If you're using named ports, use the same name in the service spec too. - -*Migration* - -* If your cluster is currently using a third party solution to manage [.noloc]`Kubernetes` network policies, you can use those same policies with the [.noloc]`Amazon VPC CNI plugin for Kubernetes`. However you must remove your existing solution so that it isn't managing the same policies. - -*Installation* - -* The network policy feature creates and requires a `PolicyEndpoint` Custom Resource Definition (CRD) called `policyendpoints.networking.k8s.aws`. `PolicyEndpoint` objects of the Custom Resource are managed by Amazon EKS. You shouldn't modify or delete these resources. -* If you run pods that use the instance role IAM credentials or connect to the EC2 IMDS, be careful to check for network policies that would block access to the EC2 IMDS. You may need to add a network policy to allow access to EC2 IMDS. For more information, see link:AWSEC2/latest/UserGuide/ec2-instance-metadata.html[Instance metadata and user data,type="documentation"] in the Amazon EC2 User Guide. -+ -Pods that use _IAM roles for service accounts_ or _EKS Pod Identity_ don't access EC2 IMDS. -* The [.noloc]`Amazon VPC CNI plugin for Kubernetes` doesn't apply network policies to additional network interfaces for each pod, only the primary interface for each pod (`eth0`). This affects the following architectures: -+ -** `IPv6` pods with the `ENABLE_V4_EGRESS` variable set to `true`. This variable enables the `IPv4` egress feature to connect the IPv6 pods to `IPv4` endpoints such as those outside the cluster. The `IPv4` egress feature works by creating an additional network interface with a local loopback IPv4 address. -** When using chained network plugins such as [.noloc]`Multus`. Because these plugins add network interfaces to each pod, network policies aren't applied to the chained network plugins. - - -[.topic] -[[cni-network-policy-configure,cni-network-policy-configure.title]] -===== Restrict Pod network traffic with [.noloc]`Kubernetes` network policies - -[abstract] --- -Learn how to deploy [.noloc]`Kubernetes` network policies on your Amazon EKS cluster. --- - -You can use a [.noloc]`Kubernetes` network policy to restrict network traffic to and from your [.noloc]`Pods`. For more information, see https://kubernetes.io/docs/concepts/services-networking/network-policies/[Network Policies] in the [.noloc]`Kubernetes` documentation. - -You must configure the following in order to use this feature: - -. Set up policy enforcement at [.noloc]`Pod` startup. You do this in the `aws-node` container of the VPC CNI `DaemonSet`. -. Enable the network policy parameter for the add-on. -. Configure your cluster to use the [.noloc]`Kubernetes` network policy - -Before you begin, review the considerations. For more information, see <>. - -[[cni-network-policy-prereqs,cni-network-policy-prereqs.title]] -====== Prerequisites - -The following are prerequisites for the feature: - - - -* -.Minimum cluster version -An existing Amazon EKS cluster. To deploy one, see <>. The cluster must be [.noloc]`Kubernetes` version `1.25` or later. The cluster must be running one of the [.noloc]`Kubernetes` versions and platform versions listed in the following table. Note that any [.noloc]`Kubernetes` and platform versions later than those listed are also supported. You can check your current [.noloc]`Kubernetes` version by replacing [.replaceable]`my-cluster` in the following command with the name of your cluster and then running the modified command: -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks describe-cluster - --name my-cluster --query cluster.version --output - text ----- -+ -[cols="1,1", options="header"] -|=== -|Kubernetes version -|Platform version - - -|`1.27.4` -|`eks.5` - -|`1.26.7` -|`eks.6` - -|`1.25.12` -|`eks.7` -|=== -* -.Minimum VPC CNI version -Version `1.14` or later of the [.noloc]`Amazon VPC CNI plugin for Kubernetes` on your cluster. You can see which version that you currently have with the following command. -+ -[source,shell,subs="verbatim,attributes"] ----- -kubectl describe daemonset aws-node --namespace kube-system | grep amazon-k8s-cni: | cut -d : -f 3 ----- -+ -If your version is earlier than `1.14`, see <> to upgrade to version `1.14` or later. -* -.Minimum Linux kernel version -Your nodes must have Linux kernel version `5.10` or later. You can check your kernel version with `uname -r`. If you're using the latest versions of the Amazon EKS optimized Amazon Linux, Amazon EKS optimized accelerated Amazon Linux AMIs, and Bottlerocket AMIs, they already have the required kernel version. -+ -The Amazon EKS optimized accelerated Amazon Linux AMI version `v20231116` or later have kernel version `5.10`. - - -[[cni-network-policy-configure-policy,cni-network-policy-configure-policy.title]] -====== Step 1: Set up policy enforcement at [.noloc]`Pod` startup - - -The [.noloc]`Amazon VPC CNI plugin for Kubernetes` configures network policies for pods in parallel with the pod provisioning. Until all of the policies are configured for the new pod, containers in the new pod will start with a _default allow policy_. This is called _standard mode_. A default allow policy means that all ingress and egress traffic is allowed to and from the new pods. For example, the pods will not have any firewall rules enforced (all traffic is allowed) until the new pod is updated with the active policies. - -With the `NETWORK_POLICY_ENFORCING_MODE` variable set to `strict`, pods that use the VPC CNI start with a _default deny policy_, then policies are configured. This is called _strict mode_. In strict mode, you must have a network policy for every endpoint that your pods need to access in your cluster. Note that this requirement applies to the [.noloc]`CoreDNS` pods. The default deny policy isn't configured for pods with Host networking. - -You can change the default network policy by setting the environment variable `NETWORK_POLICY_ENFORCING_MODE` to `strict` in the `aws-node` container of the VPC CNI `DaemonSet`. - -[source,yaml,subs="verbatim,attributes"] ----- -env: - - name: NETWORK_POLICY_ENFORCING_MODE - value: "strict" ----- - - -[[enable-network-policy-parameter,enable-network-policy-parameter.title]] -====== Step 2: Enable the network policy parameter for the add-on - -The network policy feature uses port `8162` on the node for metrics by default. Also, the feature used port `8163` for health probes. If you run another application on the nodes or inside pods that needs to use these ports, the app fails to run. In VPC CNI version `v1.14.1` or later, you can change these ports. - -Use the following procedure to enable the network policy parameter for the add-on. - - - -{aws-management-console}:: -.. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. -.. In the left navigation pane, select *Clusters*, and then select the name of the cluster that you want to configure the Amazon VPC CNI add-on for. -.. Choose the *Add-ons* tab. -.. Select the box in the top right of the add-on box and then choose *Edit*. -.. On the *Configure [.replaceable]`name of add-on`* page: -+ -... Select a `v1.14.0-eksbuild.3` or later version in the *Version* list. -... Expand the *Optional configuration settings*. -... Enter the JSON key `"enableNetworkPolicy":` and value `"true"` in *Configuration values*. The resulting text must be a valid JSON object. If this key and value are the only data in the text box, surround the key and value with curly braces `{ }`. -+ -The following example has network policy feature enabled and metrics and health probes are set to the default port numbers: -+ -[source,json,subs="verbatim,attributes"] ----- -{ - "enableNetworkPolicy": "true", - "nodeAgent": { - "healthProbeBindAddr": "8163", - "metricsBindAddr": "8162" - } -} ----- - - -Helm:: - -If you have installed the [.noloc]`Amazon VPC CNI plugin for Kubernetes` through `helm`, you can update the configuration to change the ports. - -.. Run the following command to change the ports. Set the port number in the value for either key `nodeAgent.metricsBindAddr` or key `nodeAgent.healthProbeBindAddr`, respectively. -+ -[source,shell,subs="verbatim,attributes"] ----- -helm upgrade --set nodeAgent.metricsBindAddr=8162 --set nodeAgent.healthProbeBindAddr=8163 aws-vpc-cni --namespace kube-system eks/aws-vpc-cni ----- - - -[.noloc]`kubectl`:: -.. Open the `aws-node` `DaemonSet` in your editor. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl edit daemonset -n kube-system aws-node ----- -.. Replace the port numbers in the following command arguments in the `args:` in the `aws-network-policy-agent` container in the VPC CNI `aws-node` daemonset manifest. -+ -[source,yaml,subs="verbatim,attributes"] ----- - - args: - - --metrics-bind-addr=:8162 - - --health-probe-bind-addr=:8163 ----- - - -[[cni-mount-bpf,cni-mount-bpf.title]] -====== Step 3: Mount the Berkeley Packet Filter (BPF) file system on your nodes - -You must mount the Berkeley Packet Filter (BPF) file system on each of your nodes. - -[NOTE] -==== - -If your cluster is version `1.27` or later, you can skip this step as all Amazon EKS optimized Amazon Linux and Bottlerocket AMIs for `1.27` or later have this feature already. - -For all other cluster versions, if you upgrade the Amazon EKS optimized Amazon Linux to version `v20230703` or later or you upgrade the Bottlerocket AMI to version `v1.0.2` or later, you can skip this step. - -==== -. Mount the Berkeley Packet Filter (BPF) file system on each of your nodes. -+ -[source,shell,subs="verbatim,attributes"] ----- -sudo mount -t bpf bpffs /sys/fs/bpf ----- -. Then, add the same command to your user data in your launch template for your Amazon EC2 Auto Scaling Groups. - - -[[cni-network-policy-setup,cni-network-policy-setup.title]] -====== Step 4: Configure your cluster to use [.noloc]`Kubernetes` network policies - -Configure the cluster to use [.noloc]`Kubernetes` network policies. You can set this for an Amazon EKS add-on or self-managed add-on. - - -[[cni-network-policy-setup-procedure-add-on,cni-network-policy-setup-procedure-add-on.title]] -.Amazon EKS add-on -[%collapsible] -==== - -{aws-management-console}:: -.. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. -.. In the left navigation pane, select *Clusters*, and then select the name of the cluster that you want to configure the Amazon VPC CNI add-on for. -.. Choose the *Add-ons* tab. -.. Select the box in the top right of the add-on box and then choose *Edit*. -.. On the *Configure [.replaceable]`name of addon`* page: -+ -... Select a `v1.14.0-eksbuild.3` or later version in the *Version* list. -... Expand the *Optional configuration settings*. -... Enter the JSON key `"enableNetworkPolicy":` and value `"true"` in *Configuration values*. The resulting text must be a valid JSON object. If this key and value are the only data in the text box, surround the key and value with curly braces `{ }`. The following example shows network policy is enabled: -+ -[source,json,subs="verbatim,attributes"] ----- -{ "enableNetworkPolicy": "true" } ----- -+ -The following screenshot shows an example of this scenario. -+ -image::images/console-cni-config-network-policy.png[{aws-management-console} showing the VPC CNI add-on with network policy in the optional configuration.,scaledwidth=80%] - - -{aws} CLI:: -.. Run the following {aws} CLI command. Replace `my-cluster` with the name of your cluster and the IAM role ARN with the role that you are using. -+ -[source,shell,subs="verbatim,attributes"] ----- -aws eks update-addon --cluster-name my-cluster --addon-name vpc-cni --addon-version v1.14.0-eksbuild.3 \ - --service-account-role-arn {arn-aws}iam::123456789012:role/AmazonEKSVPCCNIRole \ - --resolve-conflicts PRESERVE --configuration-values '{"enableNetworkPolicy": "true"}' ----- - -==== - -[[cni-network-policy-setup-procedure-self-managed-add-on,cni-network-policy-setup-procedure-self-managed-add-on.title]] -.Self-managed add-on -[%collapsible] -==== - -Helm:: - -If you have installed the [.noloc]`Amazon VPC CNI plugin for Kubernetes` through `helm`, you can update the configuration to enable network policy. - -.. Run the following command to enable network policy. -+ -[source,shell,subs="verbatim,attributes"] ----- -helm upgrade --set enableNetworkPolicy=true aws-vpc-cni --namespace kube-system eks/aws-vpc-cni ----- - - -[.noloc]`kubectl`:: -.. Open the `amazon-vpc-cni` `ConfigMap` in your editor. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl edit configmap -n kube-system amazon-vpc-cni -o yaml ----- -.. Add the following line to the `data` in the `ConfigMap`. -+ -[source,bash,subs="verbatim,attributes"] ----- -enable-network-policy-controller: "true" ----- -+ -Once you've added the line, your `ConfigMap` should look like the following example. -+ -[source,yaml,subs="verbatim,attributes"] ----- -apiVersion: v1 - kind: ConfigMap - metadata: - name: amazon-vpc-cni - namespace: kube-system - data: - enable-network-policy-controller: "true" ----- -.. Open the `aws-node` `DaemonSet` in your editor. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl edit daemonset -n kube-system aws-node ----- -.. Replace the `false` with `true` in the command argument `--enable-network-policy=false` in the `args:` in the `aws-network-policy-agent` container in the VPC CNI `aws-node` daemonset manifest. -+ -[source,yaml,subs="verbatim,attributes"] ----- - - args: - - --enable-network-policy=true ----- - -==== - -[[cni-network-policy-setup-procedure-confirm,cni-network-policy-setup-procedure-confirm.title]] -====== Step 5. Next steps - -After you complete the configuration, confirm that the `aws-node` pods are running on your cluster. - -[source,bash,subs="verbatim,attributes"] ----- -kubectl get pods -n kube-system | grep 'aws-node\|amazon' ----- - -An example output is as follows. - -[source,bash,subs="verbatim,attributes"] ----- -aws-node-gmqp7 2/2 Running 1 (24h ago) 24h -aws-node-prnsh 2/2 Running 1 (24h ago) 24h ----- - -There are 2 containers in the `aws-node` pods in versions `1.14` and later. In previous versions and if network policy is disabled, there is only a single container in the `aws-node` pods. - -You can now deploy [.noloc]`Kubernetes` network policies to your cluster. - -To implement [.noloc]`Kubernetes` network policies you create [.noloc]`Kubernetes` `NetworkPolicy` objects and deploy them to your cluster. `NetworkPolicy` objects are scoped to a namespace. You implement policies to allow or deny traffic between [.noloc]`Pods` based on label selectors, namespaces, and IP address ranges. For more information about creating `NetworkPolicy` objects, see https://kubernetes.io/docs/concepts/services-networking/network-policies/#networkpolicy-resource[Network Policies] in the [.noloc]`Kubernetes` documentation. - -Enforcement of [.noloc]`Kubernetes` `NetworkPolicy` objects is implemented using the [.noloc]`Extended Berkeley Packet Filter` ([.noloc]`eBPF`). Relative to `iptables` based implementations, it offers lower latency and performance characteristics, including reduced CPU utilization and avoiding sequential lookups. Additionally, [.noloc]`eBPF` probes provide access to context rich data that helps debug complex kernel level issues and improve observability. Amazon EKS supports an [.noloc]`eBPF`-based exporter that leverages the probes to log policy results on each node and export the data to external log collectors to aid in debugging. For more information, see the https://ebpf.io/what-is-ebpf/#what-is-ebpf[eBPF documentation]. - -[.topic] -[[network-policy-disable,network-policy-disable.title]] -===== Disable [.noloc]`Kubernetes` network policies for Amazon EKS Pod network traffic - -[abstract] --- -Learn how to disable [.noloc]`Kubernetes` network policies for Amazon EKS Pod network traffic. --- - -Disable [.noloc]`Kubernetes` network policies to stop restricting Amazon EKS Pod network traffic - -. List all Kubernetes network policies. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl get netpol -A ----- -. Delete each Kubernetes network policy. You must delete all network policies before disabling network policies. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl delete netpol ----- -. Open the aws-node DaemonSet in your editor. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl edit daemonset -n kube-system aws-node ----- -. Replace the `true` with `false` in the command argument `--enable-network-policy=true` in the `args:` in the `aws-network-policy-agent` container in the VPC CNI `aws-node` daemonset manifest. -+ -[source,yaml,subs="verbatim,attributes"] ----- - - args: - - --enable-network-policy=true ----- - - -include::network-policies-troubleshooting.adoc[leveloffset=+1] - - -include::network-policy-stars-demo.adoc[leveloffset=+1] - - -[.topic] -[[cni-custom-network,cni-custom-network.title]] -=== Deploy [.noloc]`pods` in alternate subnets with custom networking - -[abstract] --- -Learn how to enable custom networking for Amazon EKS [.noloc]`Pods` to deploy them in different subnets or use different security groups than the node's primary network interface, increasing IP address availability and network isolation. --- - -*Applies to*: [.noloc]`Linux` `IPv4` Fargate nodes, [.noloc]`Linux` nodes with Amazon EC2 instances - -By default, when the [.noloc]`Amazon VPC CNI plugin for Kubernetes` creates secondary link:AWSEC2/latest/UserGuide/using-eni.html[elastic network interfaces,type="documentation"] (network interfaces) for your Amazon EC2 node, it creates them in the same subnet as the node's primary network interface. It also associates the same security groups to the secondary network interface that are associated to the primary network interface. For one or more of the following reasons, you might want the plugin to create secondary network interfaces in a different subnet or want to associate different security groups to the secondary network interfaces, or both: - - - -* There's a limited number of `IPv4` addresses that are available in the subnet that the primary network interface is in. This might limit the number of [.noloc]`Pods` that you can create in the subnet. By using a different subnet for secondary network interfaces, you can increase the number of available `IPv4` addresses available for [.noloc]`Pods`. -* For security reasons, your [.noloc]`Pods` might need to use a different subnet or security groups than the node's primary network interface. -* The nodes are configured in public subnets, and you want to place the [.noloc]`Pods` in private subnets. The route table associated to a public subnet includes a route to an internet gateway. The route table associated to a private subnet doesn't include a route to an internet gateway. - - -[[cni-custom-network-considerations,cni-custom-network-considerations.title]] -==== Considerations - -The following are considerations for using the feature. - - - -* With custom networking enabled, no IP addresses assigned to the primary network interface are assigned to [.noloc]`Pods`. Only IP addresses from secondary network interfaces are assigned to [.noloc]`Pods`. -* If your cluster uses the `IPv6` family, you can't use custom networking. -* If you plan to use custom networking only to help alleviate `IPv4` address exhaustion, you can create a cluster using the `IPv6` family instead. For more information, see <>. -* Even though [.noloc]`Pods` deployed to subnets specified for secondary network interfaces can use different subnet and security groups than the node's primary network interface, the subnets and security groups must be in the same VPC as the node. -* For Fargate, subnets are controlled through the Fargate profile. For more information, see <>. - - -[.topic] -[[cni-custom-network-tutorial,cni-custom-network-tutorial.title]] -==== Customizing the secondary network interface in Amazon EKS nodes - -[abstract] --- -Learn how your [.noloc]`Pods` can use different security groups and subnets than the primary elastic network interface of the Amazon EC2 node that they run on. --- - -Complete the following before you start the tutorial: - - - -* Review the considerations -* Familiarity with how the [.noloc]`Amazon VPC CNI plugin for Kubernetes` creates secondary network interfaces and assigns IP addresses to [.noloc]`Pods`. For more information, see https://github.com/aws/amazon-vpc-cni-k8s#eni-allocation[ENI Allocation] on [.noloc]`GitHub`. -* Version `2.12.3` or later or version `1.27.160` or later of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device or {aws} CloudShell. To check your current version, use `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or [.noloc]`Homebrew` for [.noloc]`macOS` are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the _{aws} Command Line Interface User Guide_. The {aws} CLI version that is installed in {aws} CloudShell might also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the _{aws} CloudShell User Guide_. -* The `kubectl` command line tool is installed on your device or {aws} CloudShell. The version can be the same as or up to one minor version earlier or later than the [.noloc]`Kubernetes` version of your cluster. For example, if your cluster version is `1.29`, you can use `kubectl` version `1.28`, `1.29`, or `1.30` with it. To install or upgrade `kubectl`, see <>. -* We recommend that you complete the steps in this topic in a Bash shell. If you aren't using a Bash shell, some script commands such as line continuation characters and the way variables are set and used require adjustment for your shell. Additionally, the quoting and escaping rules for your shell might be different. For more information, see link:cli/latest/userguide/cli-usage-parameters-quoting-strings.html[Using quotation marks with strings in the {aws} CLI,type="documentation"] in the {aws} Command Line Interface User Guide. - -For this tutorial, we recommend using the [.replaceable]`example values`, except where it's noted to replace them. You can replace any [.replaceable]`example value` when completing the steps for a production cluster. We recommend completing all steps in the same terminal. This is because variables are set and used throughout the steps and won't exist in different terminals. - -The commands in this topic are formatted using the conventions listed in link:cli/latest/userguide/welcome-examples.html[Using the {aws} CLI examples,type="documentation"]. If you're running commands from the command line against resources that are in a different {aws} Region than the default {aws} Region defined in the {aws} CLI link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-profiles[profile,type="documentation"] that you're using, then you need to add `--region [.replaceable]``region-code``` to the commands. - -When you want to deploy custom networking to your production cluster, skip to <>. - -[[custom-networking-create-cluster,custom-networking-create-cluster.title]] -===== Step 1: Create a test VPC and cluster - -The following procedures help you create a test VPC and cluster and configure custom networking for that cluster. We don't recommend using the test cluster for production workloads because several unrelated features that you might use on your production cluster aren't covered in this topic. For more information, see <>. - -. Define the `cluster_name` and `account_id` variables.. -+ -[source,bash,subs="verbatim,attributes"] ----- -export cluster_name=my-custom-networking-cluster -account_id=$(aws sts get-caller-identity --query Account --output text) ----- -. Create a VPC. -+ -.. If you are deploying to a test system, create a VPC using an Amazon EKS {aws} CloudFormation template. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws cloudformation create-stack --stack-name my-eks-custom-networking-vpc \ - --template-url https://s3.us-west-2.amazonaws.com/amazon-eks/cloudformation/2020-10-29/amazon-eks-vpc-private-subnets.yaml \ - --parameters ParameterKey=VpcBlock,ParameterValue=192.168.0.0/24 \ - ParameterKey=PrivateSubnet01Block,ParameterValue=192.168.0.64/27 \ - ParameterKey=PrivateSubnet02Block,ParameterValue=192.168.0.96/27 \ - ParameterKey=PublicSubnet01Block,ParameterValue=192.168.0.0/27 \ - ParameterKey=PublicSubnet02Block,ParameterValue=192.168.0.32/27 ----- -+ -The {aws} CloudFormation stack takes a few minutes to create. To check on the stack's deployment status, run the following command. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws cloudformation describe-stacks --stack-name my-eks-custom-networking-vpc --query Stacks\[\].StackStatus --output text ----- -+ -Don't continue to the next step until the output of the command is `CREATE_COMPLETE`. -.. Define variables with the values of the private subnet IDs created by the template. -+ -[source,bash,subs="verbatim,attributes"] ----- -subnet_id_1=$(aws cloudformation describe-stack-resources --stack-name my-eks-custom-networking-vpc \ - --query "StackResources[?LogicalResourceId=='PrivateSubnet01'].PhysicalResourceId" --output text) -subnet_id_2=$(aws cloudformation describe-stack-resources --stack-name my-eks-custom-networking-vpc \ - --query "StackResources[?LogicalResourceId=='PrivateSubnet02'].PhysicalResourceId" --output text) ----- -.. Define variables with the Availability Zones of the subnets retrieved in the previous step. -+ -[source,bash,subs="verbatim,attributes"] ----- -az_1=$(aws ec2 describe-subnets --subnet-ids $subnet_id_1 --query 'Subnets[*].AvailabilityZone' --output text) -az_2=$(aws ec2 describe-subnets --subnet-ids $subnet_id_2 --query 'Subnets[*].AvailabilityZone' --output text) ----- -. Create a cluster IAM role. -+ -.. Run the following command to create an IAM trust policy JSON file. -+ -[source,json,subs="verbatim,attributes"] ----- -cat >eks-cluster-role-trust-policy.json <>. -.. The cluster takes several minutes to create. To check on the cluster's deployment status, run the following command. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks describe-cluster --name my-custom-networking-cluster --query cluster.status ----- -+ -Don't continue to the next step until the output of the command is `"ACTIVE"`. -.. Configure `kubectl` to communicate with your cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks update-kubeconfig --name my-custom-networking-cluster ----- - - -[[custom-networking-configure-vpc,custom-networking-configure-vpc.title]] -===== Step 2: Configure your VPC - -This tutorial requires the VPC created in <>. For a production cluster, adjust the steps accordingly for your VPC by replacing all of the [.replaceable]`example values` with your own. - -. Confirm that your currently-installed [.noloc]`Amazon VPC CNI plugin for Kubernetes` is the latest version. To determine the latest version for the Amazon EKS add-on type and update your version to it, see <>. To determine the latest version for the self-managed add-on type and update your version to it, see <>. -. Retrieve the ID of your cluster VPC and store it in a variable for use in later steps. For a production cluster, replace [.replaceable]`my-custom-networking-cluster` with the name of your cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -vpc_id=$(aws eks describe-cluster --name my-custom-networking-cluster --query "cluster.resourcesVpcConfig.vpcId" --output text) ----- -. Associate an additional Classless Inter-Domain Routing (CIDR) block with your cluster's VPC. The CIDR block can't overlap with any existing associated CIDR blocks. -+ -.. View the current CIDR blocks associated to your VPC. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws ec2 describe-vpcs --vpc-ids $vpc_id \ - --query 'Vpcs[*].CidrBlockAssociationSet[*].{CIDRBlock: CidrBlock, State: CidrBlockState.State}' --out table ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- ----------------------------------- -| DescribeVpcs | -+-----------------+--------------+ -| CIDRBlock | State | -+-----------------+--------------+ -| 192.168.0.0/24 | associated | -+-----------------+--------------+ ----- -.. Associate an additional CIDR block to your VPC. For more information, see link:vpc/latest/userguide/modify-vpcs.html#add-ipv4-cidr[Associate additional IPv4 CIDR blocks with your VPC,type="documentation"] in the Amazon VPC User Guide. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws ec2 associate-vpc-cidr-block --vpc-id $vpc_id --cidr-block 192.168.1.0/24 ----- -.. Confirm that the new block is associated. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws ec2 describe-vpcs --vpc-ids $vpc_id --query 'Vpcs[*].CidrBlockAssociationSet[*].{CIDRBlock: CidrBlock, State: CidrBlockState.State}' --out table ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- ----------------------------------- -| DescribeVpcs | -+-----------------+--------------+ -| CIDRBlock | State | -+-----------------+--------------+ -| 192.168.0.0/24 | associated | -| 192.168.1.0/24 | associated | -+-----------------+--------------+ ----- - -+ -Don't proceed to the next step until your new CIDR block's `State` is `associated`. -. Create as many subnets as you want to use in each Availability Zone that your existing subnets are in. Specify a CIDR block that's within the CIDR block that you associated with your VPC in a previous step. -+ -.. Create new subnets. The subnets must be created in a different VPC CIDR block than your existing subnets are in, but in the same Availability Zones as your existing subnets. In this example, one subnet is created in the new CIDR block in each Availability Zone that the current private subnets exist in. The IDs of the subnets created are stored in variables for use in later steps. The `Name` values match the values assigned to the subnets created using the Amazon EKS VPC template in a previous step. Names aren't required. You can use different names. -+ -[source,bash,subs="verbatim,attributes"] ----- -new_subnet_id_1=$(aws ec2 create-subnet --vpc-id $vpc_id --availability-zone $az_1 --cidr-block 192.168.1.0/27 \ - --tag-specifications 'ResourceType=subnet,Tags=[{Key=Name,Value=my-eks-custom-networking-vpc-PrivateSubnet01},{Key=kubernetes.io/role/internal-elb,Value=1}]' \ - --query Subnet.SubnetId --output text) -new_subnet_id_2=$(aws ec2 create-subnet --vpc-id $vpc_id --availability-zone $az_2 --cidr-block 192.168.1.32/27 \ - --tag-specifications 'ResourceType=subnet,Tags=[{Key=Name,Value=my-eks-custom-networking-vpc-PrivateSubnet02},{Key=kubernetes.io/role/internal-elb,Value=1}]' \ - --query Subnet.SubnetId --output text) ----- -+ -IMPORTANT: By default, your new subnets are implicitly associated with your VPC's link:vpc/latest/userguide/VPC_Route_Tables.html#RouteTables[main route table,type="documentation"]. This route table allows communication between all the resources that are deployed in the VPC. However, it doesn't allow communication with resources that have IP addresses that are outside the CIDR blocks that are associated with your VPC. You can associate your own route table to your subnets to change this behavior. For more information, see link:vpc/latest/userguide/VPC_Route_Tables.html#subnet-route-tables[Subnet route tables,type="documentation"] in the Amazon VPC User Guide. -.. View the current subnets in your VPC. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws ec2 describe-subnets --filters "Name=vpc-id,Values=$vpc_id" \ - --query 'Subnets[*].{SubnetId: SubnetId,AvailabilityZone: AvailabilityZone,CidrBlock: CidrBlock}' \ - --output table ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- ----------------------------------------------------------------------- -| DescribeSubnets | -+------------------+--------------------+----------------------------+ -| AvailabilityZone | CidrBlock | SubnetId | -+------------------+--------------------+----------------------------+ -| us-west-2d | 192.168.0.0/27 | subnet-example1 | -| us-west-2a | 192.168.0.32/27 | subnet-example2 | -| us-west-2a | 192.168.0.64/27 | subnet-example3 | -| us-west-2d | 192.168.0.96/27 | subnet-example4 | -| us-west-2a | 192.168.1.0/27 | subnet-example5 | -| us-west-2d | 192.168.1.32/27 | subnet-example6 | -+------------------+--------------------+----------------------------+ ----- -+ -You can see the subnets in the `192.168.1.0` CIDR block that you created are in the same Availability Zones as the subnets in the `192.168.0.0` CIDR block. - - -[[custom-networking-configure-kubernetes,custom-networking-configure-kubernetes.title]] -===== Step 3: Configure [.noloc]`Kubernetes` resources -. Set the `AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG` environment variable to `true` in the `aws-node` [.noloc]`DaemonSet`. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl set env daemonset aws-node -n kube-system AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG=true ----- -. Retrieve the ID of your <> and store it in a variable for use in the next step. Amazon EKS automatically creates this security group when you create your cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -cluster_security_group_id=$(aws eks describe-cluster --name $cluster_name --query cluster.resourcesVpcConfig.clusterSecurityGroupId --output text) ----- -. [[custom-networking-create-eniconfig]]Create an `ENIConfig` custom resource for each subnet that you want to deploy [.noloc]`Pods` in. -+ -.. Create a unique file for each network interface configuration. -+ -+ -The following commands create separate `ENIConfig` files for the two subnets that were created in a previous step. The value for `name` must be unique. The name is the same as the Availability Zone that the subnet is in. The cluster security group is assigned to the `ENIConfig`. -+ -[source,yaml,subs="verbatim,attributes"] ----- -cat >$az_1.yaml <$az_2.yaml <> later in this tutorial. -+ -NOTE: If you don't specify a valid security group for use with a production cluster and you're using: - -*** version `1.8.0` or later of the [.noloc]`Amazon VPC CNI plugin for Kubernetes`, then the security groups associated with the node's primary elastic network interface are used. -*** a version of the [.noloc]`Amazon VPC CNI plugin for Kubernetes` that's earlier than `1.8.0`, then the default security group for the VPC is assigned to secondary network interfaces. - -+ -IMPORTANT: -*** `AWS_VPC_K8S_CNI_EXTERNALSNAT=false` is a default setting in the configuration for the Amazon VPC CNI plugin for [.noloc]`Kubernetes`. If you're using the default setting, then traffic that is destined for IP addresses that aren't within one of the CIDR blocks associated with your VPC use the security groups and subnets of your node's primary network interface. The subnets and security groups defined in your `ENIConfigs` that are used to create secondary network interfaces aren't used for this traffic. For more information about this setting, see <>. -*** If you also use security groups for [.noloc]`Pods`, the security group that's specified in a `SecurityGroupPolicy` is used instead of the security group that's specified in the `ENIConfigs`. For more information, see <>. - -.. Apply each custom resource file that you created to your cluster with the following commands. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl apply -f $az_1.yaml -kubectl apply -f $az_2.yaml ----- -. Confirm that your `ENIConfigs` were created. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl get ENIConfigs ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -NAME AGE -us-west-2a 117s -us-west-2d 105s ----- -. If you're enabling custom networking on a production cluster and named your `ENIConfigs` something other than the Availability Zone that you're using them for, then skip to the <> to deploy Amazon EC2 nodes. -+ -Enable [.noloc]`Kubernetes` to automatically apply the `ENIConfig` for an Availability Zone to any new Amazon EC2 nodes created in your cluster. -+ -.. For the test cluster in this tutorial, skip to the <>. -+ -For a production cluster, check to see if an [.noloc]`annotation` with the key `k8s.amazonaws.com/eniConfig` for the `https://github.com/aws/amazon-vpc-cni-k8s#eni_config_annotation_def[ENI_CONFIG_ANNOTATION_DEF]` environment variable exists in the container spec for the `aws-node` [.noloc]`DaemonSet`. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl describe daemonset aws-node -n kube-system | grep ENI_CONFIG_ANNOTATION_DEF ----- -+ -If output is returned, the annotation exists. If no output is returned, then the variable is not set. For a production cluster, you can use either this setting or the setting in the following step. If you use this setting, it overrides the setting in the following step. In this tutorial, the setting in the next step is used. -.. [[custom-networking-automatically-apply-eniconfig]]Update your `aws-node` [.noloc]`DaemonSet` to automatically apply the `ENIConfig` for an Availability Zone to any new Amazon EC2 nodes created in your cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl set env daemonset aws-node -n kube-system ENI_CONFIG_LABEL_DEF=topology.kubernetes.io/zone ----- - - -[[custom-networking-deploy-nodes,custom-networking-deploy-nodes.title]] -===== Step 4: Deploy Amazon EC2 nodes -. Create a node IAM role. -+ -.. Run the following command to create an IAM trust policy JSON file. -+ -[source,json,subs="verbatim,attributes"] ----- -cat >node-role-trust-relationship.json <>. -. Create one of the following types of node groups. To determine the instance type that you want to deploy, see <>. For this tutorial, complete the *Managed*, *Without a launch template or with a launch template without an AMI ID specified* option. If you're going to use the node group for production workloads, then we recommend that you familiarize yourself with all of the managed node group <> and self-managed node group <> options before deploying the node group. -+ -** *Managed* – Deploy your node group using one of the following options: -+ -*** *Without a launch template or with a launch template without an AMI ID specified* – Run the following command. For this tutorial, use the [.replaceable]`example values`. For a production node group, replace all [.replaceable]`example values` with your own. The node group name can't be longer than 63 characters. It must start with letter or digit, but can also include hyphens and underscores for the remaining characters. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks create-nodegroup --cluster-name $cluster_name --nodegroup-name my-nodegroup \ - --subnets $subnet_id_1 $subnet_id_2 --instance-types t3.medium --node-role $node_role_arn ----- -*** *With a launch template with a specified AMI ID*:: - -+ -.... Determine the Amazon EKS recommended number of maximum [.noloc]`Pods `for your nodes. Follow the instructions in <>, adding `--cni-custom-networking-enabled` to step 3 in that topic. Note the output for use in the next step. -.... In your launch template, specify an Amazon EKS optimized AMI ID, or a custom AMI built off the Amazon EKS optimized AMI, then <> and provide the following user data in the launch template. This user data passes arguments into the `bootstrap.sh` file. For more information about the bootstrap file, see https://github.com/awslabs/amazon-eks-ami/blob/main/templates/al2/runtime/bootstrap.sh[bootstrap.sh] on [.noloc]`GitHub`. You can replace [.replaceable]`20` with either the value from the previous step (recommended) or your own value. -+ -[source,bash,subs="verbatim,attributes"] ----- -/etc/eks/bootstrap.sh my-cluster --use-max-pods false --kubelet-extra-args '--max-pods=20' ----- -+ -If you've created a custom AMI that is not built off the Amazon EKS optimized AMI, then you need to custom create the configuration yourself. -** *Self-managed*:: - -+ -... Determine the Amazon EKS recommended number of maximum [.noloc]`Pods` for your nodes. Follow the instructions in <>, adding `--cni-custom-networking-enabled` to step 3 in that topic. Note the output for use in the next step. -... Deploy the node group using the instructions in <>. Specify the following text for the *BootstrapArguments* parameter. You can replace [.replaceable]`20` with either the value from the previous step (recommended) or your own value. -+ -[source,bash,subs="verbatim,attributes"] ----- ---use-max-pods false --kubelet-extra-args '--max-pods=20' ----- -+ -NOTE: If you want nodes in a production cluster to support a significantly higher number of [.noloc]`Pods`, run the script in <> again. Also, add the `--cni-prefix-delegation-enabled` option to the command. For example, [.replaceable]`110` is returned for an `m5.large` instance type. For instructions on how to enable this capability, see <>. You can use this capability with custom networking. -+ -Node group creation takes several minutes. You can check the status of the creation of a managed node group with the following command. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks describe-nodegroup --cluster-name $cluster_name --nodegroup-name my-nodegroup --query nodegroup.status --output text ----- -+ -Don't continue to the next step until the output returned is `ACTIVE`. -. [[custom-networking-annotate-eniconfig]]For the tutorial, you can skip this step. -+ -For a production cluster, if you didn't name your `ENIConfigs` the same as the Availability Zone that you're using them for, then you must annotate your nodes with the `ENIConfig` name that should be used with the node. This step isn't necessary if you only have one subnet in each Availability Zone and you named your `ENIConfigs` with the same names as your Availability Zones. This is because the [.noloc]`Amazon VPC CNI plugin for Kubernetes` automatically associates the correct `ENIConfig` with the node for you when you enabled it to do so in a <>. -+ -.. Get the list of nodes in your cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl get nodes ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -NAME STATUS ROLES AGE VERSION -ip-192-168-0-126.us-west-2.compute.internal Ready 8m49s v1.22.9-eks-810597c -ip-192-168-0-92.us-west-2.compute.internal Ready 8m34s v1.22.9-eks-810597c ----- -.. Determine which Availability Zone each node is in. Run the following command for each node that was returned in the previous step. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws ec2 describe-instances --filters Name=network-interface.private-dns-name,Values=ip-192-168-0-126.us-west-2.compute.internal \ ---query 'Reservations[].Instances[].{AvailabilityZone: Placement.AvailabilityZone, SubnetId: SubnetId}' ----- -+ -An example output is as follows. -+ -[source,json,subs="verbatim,attributes"] ----- -[ - { - "AvailabilityZone": "us-west-2d", - "SubnetId": "subnet-Example5" - } -] ----- -.. Annotate each node with the `ENIConfig` that you created for the subnet ID and Availability Zone. You can only annotate a node with one `ENIConfig`, though multiple nodes can be annotated with the same `ENIConfig`. Replace the [.replaceable]`example values` with your own. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl annotate node ip-192-168-0-126.us-west-2.compute.internal k8s.amazonaws.com/eniConfig=EniConfigName1 -kubectl annotate node ip-192-168-0-92.us-west-2.compute.internal k8s.amazonaws.com/eniConfig=EniConfigName2 ----- -. [[custom-networking-terminate-existing-nodes]]If you had nodes in a production cluster with running [.noloc]`Pods` before you switched to using the custom networking feature, complete the following tasks: -+ -.. Make sure that you have available nodes that are using the custom networking feature. -.. Cordon and drain the nodes to gracefully shut down the [.noloc]`Pods`. For more information, see https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/[Safely Drain a Node] in the [.noloc]`Kubernetes` documentation. -.. Terminate the nodes. If the nodes are in an existing managed node group, you can delete the node group. Copy the command that follows to your device. Make the following modifications to the command as needed and then run the modified command: -+ -*** Replace [.replaceable]`my-cluster` with the name for your cluster. -*** Replace [.replaceable]`my-nodegroup` with the name for your node group. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks delete-nodegroup --cluster-name my-cluster --nodegroup-name my-nodegroup ----- - -+ -Only new nodes that are registered with the `k8s.amazonaws.com/eniConfig` label use the custom networking feature. -. Confirm that [.noloc]`Pods` are assigned an IP address from a CIDR block that's associated to one of the subnets that you created in a previous step. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl get pods -A -o wide ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES -kube-system aws-node-2rkn4 1/1 Running 0 7m19s 192.168.0.92 ip-192-168-0-92.us-west-2.compute.internal -kube-system aws-node-k96wp 1/1 Running 0 7m15s 192.168.0.126 ip-192-168-0-126.us-west-2.compute.internal -kube-system coredns-657694c6f4-smcgr 1/1 Running 0 56m 192.168.1.23 ip-192-168-0-92.us-west-2.compute.internal -kube-system coredns-657694c6f4-stwv9 1/1 Running 0 56m 192.168.1.28 ip-192-168-0-92.us-west-2.compute.internal -kube-system kube-proxy-jgshq 1/1 Running 0 7m19s 192.168.0.92 ip-192-168-0-92.us-west-2.compute.internal -kube-system kube-proxy-wx9vk 1/1 Running 0 7m15s 192.168.0.126 ip-192-168-0-126.us-west-2.compute.internal ----- -+ -You can see that the coredns [.noloc]`Pods` are assigned IP addresses from the `192.168.1.0` CIDR block that you added to your VPC. Without custom networking, they would have been assigned addresses from the `192.168.0.0` CIDR block, because it was the only CIDR block originally associated with the VPC. -+ -If a [.noloc]`Pod's` `spec` contains `hostNetwork=true`, it's assigned the primary IP address of the node. It isn't assigned an address from the subnets that you added. By default, this value is set to `false`. This value is set to `true` for the `kube-proxy` and [.noloc]`Amazon VPC CNI plugin for Kubernetes` (`aws-node`) [.noloc]`Pods` that run on your cluster. This is why the `kube-proxy` and the plugin's `aws-node` [.noloc]`Pods` aren't assigned `192.168.1.[.replaceable]``x``` addresses in the previous output. For more information about a [.noloc]`Pod's` `hostNetwork` setting, see https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#podspec-v1-core[PodSpec v1 core] in the [.noloc]`Kubernetes` API reference. - - -[[custom-network-delete-resources,custom-network-delete-resources.title]] -===== Step 5: Delete tutorial resources - -After you complete the tutorial, we recommend that you delete the resources that you created. You can then adjust the steps to enable custom networking for a production cluster. - -. If the node group that you created was just for testing, then delete it. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks delete-nodegroup --cluster-name $cluster_name --nodegroup-name my-nodegroup ----- -+ -Even after the {aws} CLI output says that the cluster is deleted, the delete process might not actually be complete. The delete process takes a few minutes. Confirm that it's complete by running the following command. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks describe-nodegroup --cluster-name $cluster_name --nodegroup-name my-nodegroup --query nodegroup.status --output text ----- -+ -Don't continue until the returned output is similar to the following output. -+ -[source,bash,subs="verbatim,attributes"] ----- -An error occurred (ResourceNotFoundException) when calling the DescribeNodegroup operation: No node group found for name: my-nodegroup. ----- -. If the node group that you created was just for testing, then delete the node IAM role. -+ -.. Detach the policies from the role. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws iam detach-role-policy --role-name myCustomNetworkingNodeRole --policy-arn {arn-aws}iam::aws:policy/AmazonEKSWorkerNodePolicy -aws iam detach-role-policy --role-name myCustomNetworkingNodeRole --policy-arn {arn-aws}iam::aws:policy/AmazonEC2ContainerRegistryReadOnly -aws iam detach-role-policy --role-name myCustomNetworkingNodeRole --policy-arn {arn-aws}iam::aws:policy/AmazonEKS_CNI_Policy ----- -.. Delete the role. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws iam delete-role --role-name myCustomNetworkingNodeRole ----- -. Delete the cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks delete-cluster --name $cluster_name ----- -+ -Confirm the cluster is deleted with the following command. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks describe-cluster --name $cluster_name --query cluster.status --output text ----- -+ -When output similar to the following is returned, the cluster is successfully deleted. -+ -[source,bash,subs="verbatim,attributes"] ----- -An error occurred (ResourceNotFoundException) when calling the DescribeCluster operation: No cluster found for name: my-cluster. ----- -. Delete the cluster IAM role. -+ -.. Detach the policies from the role. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws iam detach-role-policy --role-name myCustomNetworkingAmazonEKSClusterRole --policy-arn {arn-aws}iam::aws:policy/AmazonEKSClusterPolicy ----- -.. Delete the role. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws iam delete-role --role-name myCustomNetworkingAmazonEKSClusterRole ----- -. Delete the subnets that you created in a previous step. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws ec2 delete-subnet --subnet-id $new_subnet_id_1 -aws ec2 delete-subnet --subnet-id $new_subnet_id_2 ----- -. Delete the VPC that you created. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws cloudformation delete-stack --stack-name my-eks-custom-networking-vpc ----- - - -[.topic] -[[cni-increase-ip-addresses,cni-increase-ip-addresses.title]] -=== Assign more IP addresses to Amazon EKS nodes with prefixes - -[abstract] --- -Learn how to significantly increase the number of IP addresses that you can assign to [.noloc]`Pods` by assigning IP prefixes with Amazon EKS, improving scalability and reducing launch delays for large and spiky workloads. --- - -*Applies to*: Linux and Windows nodes with Amazon EC2 instances - -*Applies to*: Public and private subnets - -Each Amazon EC2 instance supports a maximum number of elastic network interfaces and a maximum number of IP addresses that can be assigned to each network interface. Each node requires one IP address for each network interface. All other available IP addresses can be assigned to `Pods`. Each `Pod` requires its own IP address. As a result, you might have nodes that have available compute and memory resources, but can't accommodate additional `Pods` because the node has run out of IP addresses to assign to `Pods`. - -You can increase the number of IP addresses that nodes can assign to `Pods` by assigning IP prefixes, rather than assigning individual secondary IP addresses to your nodes. Each prefix includes several IP addresses. If you don't configure your cluster for IP prefix assignment, your cluster must make more Amazon EC2 application programming interface (API) calls to configure network interfaces and IP addresses necessary for [.noloc]`Pod` connectivity. As clusters grow to larger sizes, the frequency of these API calls can lead to longer [.noloc]`Pod` and instance launch times. This results in scaling delays to meet the demand of large and spiky workloads, and adds cost and management overhead because you need to provision additional clusters and VPCs to meet scaling requirements. For more information, see https://github.com/kubernetes/community/blob/master/sig-scalability/configs-and-limits/thresholds.md[Kubernetes Scalability thresholds] on GitHub. - -[[cni-increase-ip-addresses-compatability,cni-increase-ip-addresses-compatability.title]] -==== Compatibility with [.noloc]`Amazon VPC CNI plugin for Kubernetes` features - -You can use IP prefixes with the following features: - - - -* IPv4 Source Network Address Translation - For more information, see <>. -* IPv6 addresses to clusters, Pods, and services - For more information, see <>. -* Restricting traffic using [.noloc]`Kubernetes` network policies - For more information, see <>. - -The following list provides information about the Amazon VPC CNI plugin settings that apply. For more information about each setting, see https://github.com/aws/amazon-vpc-cni-k8s/blob/master/README.md[amazon-vpc-cni-k8s] on [.noloc]`GitHub`. - - - -* `WARM_IP_TARGET` -* `MINIMUM_IP_TARGET` -* `WARM_PREFIX_TARGET` - - -[[cni-increase-ip-addresses-considerations,cni-increase-ip-addresses-considerations.title]] -==== Considerations - -Consider the following when you use this feature: - - - -* Each Amazon EC2 instance type supports a maximum number of [.noloc]`Pods`. If your managed node group consists of multiple instance types, the smallest number of maximum [.noloc]`Pods` for an instance in the cluster is applied to all nodes in the cluster. -* By default, the maximum number of `Pods` that you can run on a node is 110, but you can change that number. If you change the number and have an existing managed node group, the next AMI or launch template update of your node group results in new nodes coming up with the changed value. -* When transitioning from assigning IP addresses to assigning IP prefixes, we recommend that you create new node groups to increase the number of available IP addresses, rather than doing a rolling replacement of existing nodes. Running [.noloc]`Pods` on a node that has both IP addresses and prefixes assigned can lead to inconsistency in the advertised IP address capacity, impacting the future workloads on the node. For the recommended way of performing the transition, see https://github.com/aws/aws-eks-best-practices/blob/master/content/networking/prefix-mode/index_windows.md#replace-all-nodes-during-migration-from-secondary-ip-mode-to-prefix-delegation-mode-or-vice-versa[Replace all nodes during migration from Secondary IP mode to Prefix Delegation mode or vice versa] in the Amazon EKS best practices guide. -* The security group scope is at the node-level - For more information, see link:vpc/latest/userguide/VPC_SecurityGroups.html[Security group,type="documentation"]. -* IP prefixes assigned to a network interface support high [.noloc]`Pod` density per node and have the best launch time. -* IP prefixes and IP addresses are associated with standard Amazon EC2 elastic network interfaces. Pods requiring specific security groups are assigned the primary IP address of a branch network interface. You can mix [.noloc]`Pods` getting IP addresses, or IP addresses from IP prefixes with [.noloc]`Pods` getting branch network interfaces on the same node. -* For clusters with Linux nodes only. -+ -** After you configure the add-on to assign prefixes to network interfaces, you can't downgrade your [.noloc]`Amazon VPC CNI plugin for Kubernetes` add-on to a version lower than `1.9.0` (or `1.10.1`) without removing all nodes in all node groups in your cluster. -** If you're also using security groups for [.noloc]`Pods`, with `POD_SECURITY_GROUP_ENFORCING_MODE`=``standard`` and `AWS_VPC_K8S_CNI_EXTERNALSNAT`=``false``, when your [.noloc]`Pods` communicate with endpoints outside of your VPC, the node's security groups are used, rather than any security groups you've assigned to your [.noloc]`Pods`. -+ -If you're also using <>, with `POD_SECURITY_GROUP_ENFORCING_MODE`=``strict``, when your `Pods` communicate with endpoints outside of your VPC, the `Pod's` security groups are used. - - -[.topic] -[[cni-increase-ip-addresses-procedure,cni-increase-ip-addresses-procedure.title]] -==== Increase the available IP addresses for your Amazon EKS node - -You can increase the number of IP addresses that nodes can assign to [.noloc]`Pods` by assigning IP prefixes, rather than assigning individual secondary IP addresses to your nodes. - -Complete the following before you start the procedure: - - - -* Review the considerations. -* You need an existing cluster. To deploy one, see <>. -* The subnets that your Amazon EKS nodes are in must have sufficient contiguous `/28` (for `IPv4` clusters) or `/80` (for `IPv6` clusters) Classless Inter-Domain Routing (CIDR) blocks. You can only have Linux nodes in an `IPv6` cluster. Using IP prefixes can fail if IP addresses are scattered throughout the subnet CIDR. We recommend that following: -+ -** Using a subnet CIDR reservation so that even if any IP addresses within the reserved range are still in use, upon their release, the IP addresses aren't reassigned. This ensures that prefixes are available for allocation without segmentation. -** Use new subnets that are specifically used for running the workloads that IP prefixes are assigned to. Both [.noloc]`Windows` and [.noloc]`Linux` workloads can run in the same subnet when assigning IP prefixes. -* To assign IP prefixes to your nodes, your nodes must be {aws} Nitro-based. Instances that aren't Nitro-based continue to allocate individual secondary IP addresses, but have a significantly lower number of IP addresses to assign to [.noloc]`Pods` than [.noloc]`Nitro-based` instances do. -* *For clusters with [.noloc]`Linux` nodes only* – If your cluster is configured for the `IPv4` family, you must have version `1.9.0` or later of the [.noloc]`Amazon VPC CNI plugin for Kubernetes` add-on installed. You can check your current version with the following command. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl describe daemonset aws-node --namespace kube-system | grep Image | cut -d "/" -f 2 ----- -+ -If your cluster is configured for the `IPv6` family, you must have version `1.10.1` of the add-on installed. If your plugin version is earlier than the required versions, you must update it. For more information, see the updating sections of <>. -* *For clusters with [.noloc]`Windows` nodes only*:: - -+ -** Your cluster and its platform version must be at, or later than the versions in the following table. To upgrade your cluster version, see <>. If your cluster isn't at the minimum platform version, then you can't assign IP prefixes to your nodes until Amazon EKS has updated your platform version. -+ -[cols="1,1", options="header"] -|=== -|Kubernetes version -|Platform version - - -|`1.27` -|`eks.3` - -|`1.26` -|`eks.4` - -|`1.25` -|`eks.5` -|=== -+ -You can check your current [.noloc]`Kubernetes` and platform version by replacing [.replaceable]`my-cluster` in the following command with the name of your cluster and then running the modified command: `aws eks describe-cluster --name [.replaceable]``my-cluster`` --query 'cluster.{"Kubernetes Version": version, "Platform Version": platformVersion}'`. -** [.noloc]`Windows` support enabled for your cluster. For more information, see <>. -. Configure your cluster to assign IP address prefixes to nodes. Complete the procedure on the tab that matches your node's operating system. -+ -[.noloc]`Linux`::: -... Enable the parameter to assign prefixes to network interfaces for the Amazon VPC CNI [.noloc]`DaemonSet`. When you deploy a `1.21` or later cluster, version `1.10.1` or later of the [.noloc]`Amazon VPC CNI plugin for Kubernetes` add-on is deployed with it. If you created the cluster with the `IPv6` family, this setting was set to `true` by default. If you created the cluster with the `IPv4` family, this setting was set to `false` by default. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl set env daemonset aws-node -n kube-system ENABLE_PREFIX_DELEGATION=true ----- -+ -IMPORTANT: Even if your subnet has available IP addresses, if the subnet does not have any contiguous `/28` blocks available, you will see the following error in the [.noloc]`Amazon VPC CNI plugin for Kubernetes` logs. - -[source,bash,subs="verbatim,attributes"] ----- -InsufficientCidrBlocks: The specified subnet does not have enough free cidr blocks to satisfy the request ----- - -This can happen due to fragmentation of existing secondary IP addresses spread out across a subnet. To resolve this error, either create a new subnet and launch [.noloc]`Pods` there, or use an Amazon EC2 subnet CIDR reservation to reserve space within a subnet for use with prefix assignment. For more information, see link:vpc/latest/userguide/subnet-cidr-reservation.html[Subnet CIDR reservations,type="documentation"] in the Amazon VPC User Guide. -... If you plan to deploy a managed node group without a launch template, or with a launch template that you haven't specified an AMI ID in, and you're using a version of the [.noloc]`Amazon VPC CNI plugin for Kubernetes` at or later than the versions listed in the prerequisites, then skip to the next step. Managed node groups automatically calculates the maximum number of [.noloc]`Pods` for you. -+ -If you're deploying a self-managed node group or a managed node group with a launch template that you have specified an AMI ID in, then you must determine the Amazon EKS recommend number of maximum [.noloc]`Pods` for your nodes. Follow the instructions in <>, adding `--cni-prefix-delegation-enabled` to step 3. Note the output for use in a later step. -+ -IMPORTANT: Managed node groups enforces a maximum number on the value of `maxPods`. For instances with less than 30 vCPUs the maximum number is 110 and for all other instances the maximum number is 250. This maximum number is applied whether prefix delegation is enabled or not. -... If you're using a `1.21` or later cluster configured for `IPv6`, skip to the next step. -+ -Specify the parameters in one of the following options. To determine which option is right for you and what value to provide for it, see https://github.com/aws/amazon-vpc-cni-k8s/blob/master/docs/prefix-and-ip-target.md[WARM_PREFIX_TARGET, WARM_IP_TARGET, and MINIMUM_IP_TARGET] on [.noloc]`GitHub`. -+ -You can replace the [.replaceable]`example values` with a value greater than zero. -+ -**** `WARM_PREFIX_TARGET` -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl set env ds aws-node -n kube-system WARM_PREFIX_TARGET=1 ----- -**** `WARM_IP_TARGET` or `MINIMUM_IP_TARGET` – If either value is set, it overrides any value set for `WARM_PREFIX_TARGET`. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl set env ds aws-node -n kube-system WARM_IP_TARGET=5 ----- -[source,bash,subs="verbatim,attributes"] ----- -kubectl set env ds aws-node -n kube-system MINIMUM_IP_TARGET=2 ----- -... Create one of the following types of node groups with at least one Amazon EC2 Nitro Amazon Linux 2 instance type. For a list of Nitro instance types, see link:AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances[Instances built on the Nitro System,type="documentation"] in the Amazon EC2 User Guide. This capability is not supported on [.noloc]`Windows`. For the options that include [.replaceable]`110`, replace it with either the value from step 3 (recommended), or your own value. -+ -**** *Self-managed* – Deploy the node group using the instructions in <>. Specify the following text for the *BootstrapArguments* parameter. -+ -[source,bash,subs="verbatim,attributes"] ----- ---use-max-pods false --kubelet-extra-args '--max-pods=110' ----- -+ -If you're using `eksctl` to create the node group, you can use the following command. -+ -[source,bash,subs="verbatim,attributes"] ----- -eksctl create nodegroup --cluster my-cluster --managed=false --max-pods-per-node 110 ----- -**** *Managed* – Deploy your node group using one of the following options: -+ -***** *Without a launch template or with a launch template without an AMI ID specified* – Complete the procedure in <>. Managed node groups automatically calculates the Amazon EKS recommended `max-pods` value for you. -***** *With a launch template with a specified AMI ID* – In your launch template, specify an Amazon EKS optimized AMI ID, or a custom AMI built off the Amazon EKS optimized AMI, then <> and provide the following user data in the launch template. This user data passes arguments into the `bootstrap.sh` file. For more information about the bootstrap file, see https://github.com/awslabs/amazon-eks-ami/blob/main/templates/al2/runtime/bootstrap.sh[bootstrap.sh] on [.noloc]`GitHub`. -+ -[source,bash,subs="verbatim,attributes"] ----- -/etc/eks/bootstrap.sh my-cluster \ - --use-max-pods false \ - --kubelet-extra-args '--max-pods=110' ----- -+ -If you're using `eksctl` to create the node group, you can use the following command. -+ -[source,bash,subs="verbatim,attributes"] ----- -eksctl create nodegroup --cluster my-cluster --max-pods-per-node 110 ----- -+ -If you've created a custom AMI that is not built off the Amazon EKS optimized AMI, then you need to custom create the configuration yourself. -+ -NOTE: If you also want to assign IP addresses to [.noloc]`Pods` from a different subnet than the instance's, then you need to enable the capability in this step. For more information, see <>. - - -[.noloc]`Windows`::: -... Enable assignment of IP prefixes. -+ -.... Open the `amazon-vpc-cni` `ConfigMap` for editing. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl edit configmap -n kube-system amazon-vpc-cni -o yaml ----- -.... Add the following line to the `data` section. -+ -[source,yaml,subs="verbatim,attributes"] ----- - enable-windows-prefix-delegation: "true" ----- -.... Save the file and close the editor. -.... Confirm that the line was added to the `ConfigMap`. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl get configmap -n kube-system amazon-vpc-cni -o "jsonpath={.data.enable-windows-prefix-delegation}" ----- -+ -If the returned output isn't `true`, then there might have been an error. Try completing the step again. -+ -IMPORTANT: Even if your subnet has available IP addresses, if the subnet does not have any contiguous `/28` blocks available, you will see the following error in the node events. - -[source,bash,subs="verbatim,attributes"] ----- -"failed to allocate a private IP/Prefix address: InsufficientCidrBlocks: The specified subnet does not have enough free cidr blocks to satisfy the request" ----- - -This can happen due to fragmentation of existing secondary IP addresses spread out across a subnet. To resolve this error, either create a new subnet and launch [.noloc]`Pods` there, or use an Amazon EC2 subnet CIDR reservation to reserve space within a subnet for use with prefix assignment. For more information, see link:vpc/latest/userguide/subnet-cidr-reservation.html[Subnet CIDR reservations,type="documentation"] in the Amazon VPC User Guide. -... (Optional) Specify additional configuration for controlling the pre-scaling and dynamic scaling behavior for your cluster. For more information, see https://github.com/aws/amazon-vpc-resource-controller-k8s/blob/master/docs/windows/prefix_delegation_config_options.md[Configuration options with Prefix Delegation mode on Windows] on GitHub. -+ -.... Open the `amazon-vpc-cni` `ConfigMap` for editing. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl edit configmap -n kube-system amazon-vpc-cni -o yaml ----- -.... Replace the [.replaceable]`example values` with a value greater than zero and add the entries that you require to the `data` section of the `ConfigMap`. If you set a value for either `warm-ip-target` or `minimum-ip-target`, the value overrides any value set for `warm-prefix-target`. -+ -[source,yaml,subs="verbatim,attributes"] ----- - warm-prefix-target: "1" - warm-ip-target: "5" - minimum-ip-target: "2" ----- -.... Save the file and close the editor. -... Create [.noloc]`Windows` node groups with at least one Amazon EC2 [.noloc]`Nitro` instance type. For a list of [.noloc]`Nitro` instance types, see link:AWSEC2/latest/WindowsGuide/instance-types.html#ec2-nitro-instances[Instances built on the Nitro System,type="documentation"] in the Amazon EC2 User Guide. By default, the maximum number of [.noloc]`Pods` that you can deploy to a node is 110. If you want to increase or decrease that number, specify the following in the user data for the bootstrap configuration. Replace [.replaceable]`max-pods-quantity` with your max pods value. -+ -[source,bash,subs="verbatim,attributes"] ----- --KubeletExtraArgs '--max-pods=max-pods-quantity' ----- -+ -If you're deploying managed node groups, this configuration needs to be added in the launch template. For more information, see <>. For more information about the configuration parameters for [.noloc]`Windows` bootstrap script, see <>. -. Once your nodes are deployed, view the nodes in your cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl get nodes ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -NAME STATUS ROLES AGE VERSION -ip-192-168-22-103.region-code.compute.internal Ready 19m v1.XX.X-eks-6b7464 -ip-192-168-97-94.region-code.compute.internal Ready 19m v1.XX.X-eks-6b7464 ----- -. Describe one of the nodes to determine the value of `max-pods` for the node and the number of available IP addresses. Replace [.replaceable]`192.168.30.193` with the `IPv4` address in the name of one of your nodes returned in the previous output. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl describe node ip-192-168-30-193.region-code.compute.internal | grep 'pods\|PrivateIPv4Address' ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -pods: 110 -vpc.amazonaws.com/PrivateIPv4Address: 144 ----- -+ -In the previous output, `110` is the maximum number of [.noloc]`Pods` that [.noloc]`Kubernetes` will deploy to the node, even though [.replaceable]`144` IP addresses are available. - - -[.topic] -[[security-groups-for-pods,security-groups-for-pods.title]] -=== Assign security groups to individual [.noloc]`pods` - -[abstract] --- -Learn how to configure security groups for [.noloc]`Pods` on Amazon EKS, integrating Amazon EC2 security groups with [.noloc]`Kubernetes` [.noloc]`Pods` to define network traffic rules. Discover the considerations, setup process, and deploy a sample application with assigned security groups. --- - -*Applies to*: [.noloc]`Linux` nodes with Amazon EC2 instances - -*Applies to*: Private subnets - -Security groups for [.noloc]`Pods` integrate Amazon EC2 security groups with [.noloc]`Kubernetes` [.noloc]`Pods`. You can use Amazon EC2 security groups to define rules that allow inbound and outbound network traffic to and from [.noloc]`Pods` that you deploy to nodes running on many Amazon EC2 instance types and Fargate. For a detailed explanation of this capability, see the link:containers/introducing-security-groups-for-pods[Introducing security groups for Pods,type="blog"] blog post. - -[[security-groups-for-pods-compatability,security-groups-for-pods-compatability.title]] -==== Compatibility with [.noloc]`Amazon VPC CNI plugin for Kubernetes` features - -You can use security groups for [.noloc]`Pods` with the following features: - - - -* IPv4 Source Network Address Translation - For more information, see <>. -* IPv6 addresses to clusters, Pods, and services - For more information, see <>. -* Restricting traffic using [.noloc]`Kubernetes` network policies - For more information, see <>. - - -[[sg-pods-considerations,sg-pods-considerations.title]] -==== Considerations - -Before deploying security groups for [.noloc]`Pods`, consider the following limitations and conditions: - - - -* Security groups for [.noloc]`Pods` can't be used with [.noloc]`Windows` nodes. -* Security groups for [.noloc]`Pods` can be used with clusters configured for the `IPv6` family that contain Amazon EC2 nodes by using version 1.16.0 or later of the Amazon VPC CNI plugin. You can use security groups for [.noloc]`Pods` with clusters configure `IPv6` family that contain only Fargate nodes by using version 1.7.7 or later of the Amazon VPC CNI plugin. For more information, see <> -* Security groups for [.noloc]`Pods` are supported by most link:AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances[Nitro-based,type="documentation"] Amazon EC2 instance families, though not by all generations of a family. For example, the `m5`, `c5`, `r5`, `m6g`, `c6g`, and `r6g` instance family and generations are supported. No instance types in the `t` family are supported. For a complete list of supported instance types, see the https://github.com/aws/amazon-vpc-resource-controller-k8s/blob/v1.5.0/pkg/aws/vpc/limits.go[limits.go] file on [.noloc]`GitHub`. Your nodes must be one of the listed instance types that have `IsTrunkingCompatible: true` in that file. -* If you're also using [.noloc]`Pod` security policies to restrict access to [.noloc]`Pod` mutation, then the `eks:vpc-resource-controller` [.noloc]`Kubernetes` user must be specified in the [.noloc]`Kubernetes` `ClusterRoleBinding` for the `role` that your `psp` is assigned to. If you're using the default Amazon EKS `psp`, `role`, and `ClusterRoleBinding`, this is the `eks:podsecuritypolicy:authenticated` `ClusterRoleBinding`. For example, you add the user to the `subjects:` section, as shown in the following example: -+ -[source,yaml,subs="verbatim,attributes"] ----- -[...] -subjects: - - kind: Group - apiGroup: rbac.authorization.k8s.io - name: system:authenticated - - apiGroup: rbac.authorization.k8s.io - kind: User - name: eks:vpc-resource-controller - - kind: ServiceAccount - name: eks-vpc-resource-controller ----- -* If you're using custom networking and security groups for [.noloc]`Pods` together, the security group specified by security groups for [.noloc]`Pods` is used instead of the security group specified in the `ENIConfig`. -* If you're using version `1.10.2` or earlier of the Amazon VPC CNI plugin and you include the `terminationGracePeriodSeconds` setting in your [.noloc]`Pod` spec, the value for the setting can't be zero. -* If you're using version `1.10` or earlier of the Amazon VPC CNI plugin, or version `1.11` with `POD_SECURITY_GROUP_ENFORCING_MODE`=``strict``, which is the default setting, then [.noloc]`Kubernetes` services of type `NodePort` and `LoadBalancer` using instance targets with an `externalTrafficPolicy` set to `Local` aren't supported with [.noloc]`Pods` that you assign security groups to. For more information about using a load balancer with instance targets, see <>. -* If you're using version `1.10` or earlier of the Amazon VPC CNI plugin or version `1.11` with `POD_SECURITY_GROUP_ENFORCING_MODE`=``strict``, which is the default setting, source NAT is disabled for outbound traffic from [.noloc]`Pods` with assigned security groups so that outbound security group rules are applied. To access the internet, [.noloc]`Pods` with assigned security groups must be launched on nodes that are deployed in a private subnet configured with a NAT gateway or instance. [.noloc]`Pods` with assigned security groups deployed to public subnets are not able to access the internet. -+ -If you're using version `1.11` or later of the plugin with `POD_SECURITY_GROUP_ENFORCING_MODE`=``standard``, then [.noloc]`Pod` traffic destined for outside of the VPC is translated to the IP address of the instance's primary network interface. For this traffic, the rules in the security groups for the primary network interface are used, rather than the rules in the [.noloc]`Pod's` security groups. -* To use [.noloc]`Calico` network policy with [.noloc]`Pods` that have associated security groups, you must use version `1.11.0` or later of the Amazon VPC CNI plugin and set `POD_SECURITY_GROUP_ENFORCING_MODE`=``standard``. Otherwise, traffic flow to and from [.noloc]`Pods` with associated security groups are not subjected to [.noloc]`Calico` network policy enforcement and are limited to Amazon EC2 security group enforcement only. To update your Amazon VPC CNI version, see <> -* [.noloc]`Pods` running on Amazon EC2 nodes that use security groups in clusters that use https://kubernetes.io/docs/tasks/administer-cluster/nodelocaldns/[NodeLocal DNSCache] are only supported with version `1.11.0` or later of the Amazon VPC CNI plugin and with `POD_SECURITY_GROUP_ENFORCING_MODE`=``standard``. To update your Amazon VPC CNI plugin version, see <> -* Security groups for [.noloc]`Pods` might lead to higher [.noloc]`Pod` startup latency for [.noloc]`Pods` with high churn. This is due to rate limiting in the resource controller. -* The EC2 security group scope is at the [.noloc]`Pod`-level - For more information, see link:vpc/latest/userguide/VPC_SecurityGroups.html[Security group,type="documentation"]. -+ -If you set `POD_SECURITY_GROUP_ENFORCING_MODE=standard` and `AWS_VPC_K8S_CNI_EXTERNALSNAT=false`, traffic destined for endpoints outside the VPC use the node's security groups, not the [.noloc]`Pod's` security groups. - - -[.topic] -[[security-groups-pods-deployment,security-groups-pods-deployment.title]] -==== Configure the [.noloc]`Amazon VPC CNI plugin for Kubernetes` for security groups for Amazon EKS [.noloc]`Pods` - -If you use [.noloc]`Pods` with Amazon EC2 instances, you need to configure the [.noloc]`Amazon VPC CNI plugin for Kubernetes` for security groups - -If you use Fargate [.noloc]`Pods` only, and don't have any Amazon EC2 nodes in your cluster, see <>. - -. Check your current [.noloc]`Amazon VPC CNI plugin for Kubernetes` version with the following command: -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl describe daemonset aws-node --namespace kube-system | grep amazon-k8s-cni: | cut -d : -f 3 ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -v1.7.6 ----- -+ -If your [.noloc]`Amazon VPC CNI plugin for Kubernetes` version is earlier than `1.7.7`, then update the plugin to version `1.7.7` or later. For more information, see <> -. Add the link:iam/home#/policies/arn:aws:iam::aws:policy/AmazonEKSVPCResourceController[AmazonEKSVPCResourceController,type="console"] managed IAM policy to the <> that is associated with your Amazon EKS cluster. The policy allows the role to manage network interfaces, their private IP addresses, and their attachment and detachment to and from network instances. -+ -.. Retrieve the name of your cluster IAM role and store it in a variable. Replace [.replaceable]`my-cluster` with the name of your cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -cluster_role=$(aws eks describe-cluster --name my-cluster --query cluster.roleArn --output text | cut -d / -f 2) ----- -.. Attach the policy to the role. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws iam attach-role-policy --policy-arn {arn-aws}iam::aws:policy/AmazonEKSVPCResourceController --role-name $cluster_role ----- -. Enable the Amazon VPC CNI add-on to manage network interfaces for [.noloc]`Pods` by setting the `ENABLE_POD_ENI` variable to `true` in the `aws-node` [.noloc]`DaemonSet`. Once this setting is set to `true`, for each node in the cluster the add-on creates a `cninode` custom resource. The VPC resource controller creates and attaches one special network interface called a _trunk network interface_ with the description `aws-k8s-trunk-eni`. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl set env daemonset aws-node -n kube-system ENABLE_POD_ENI=true ----- -+ -NOTE: The trunk network interface is included in the maximum number of network interfaces supported by the instance type. For a list of the maximum number of network interfaces supported by each instance type, see link:AWSEC2/latest/UserGuide/using-eni.html#AvailableIpPerENI[IP addresses per network interface per instance type,type="documentation"] in the _Amazon EC2 User Guide_. If your node already has the maximum number of standard network interfaces attached to it then the VPC resource controller will reserve a space. You will have to scale down your running [.noloc]`Pods` enough for the controller to detach and delete a standard network interface, create the trunk network interface, and attach it to the instance. -. You can see which of your nodes have a `CNINode` custom resource with the following command. If `No resources found` is returned, then wait several seconds and try again. The previous step requires restarting the [.noloc]`Amazon VPC CNI plugin for` Kubernetes Pods`, which takes several seconds. -+ -[source,shell,subs="verbatim,attributes"] ----- -kubectl get cninode -A - NAME FEATURES - ip-192-168-64-141.us-west-2.compute.internal [{"name":"SecurityGroupsForPods"}] - ip-192-168-7-203.us-west-2.compute.internal [{"name":"SecurityGroupsForPods"}] ----- -+ -If you are using VPC CNI versions older than `1.15`, node labels were used instead of the `CNINode` custom resource. You can see which of your nodes have the node label `aws-k8s-trunk-eni` set to `true` with the following command. If `No resources found` is returned, then wait several seconds and try again. The previous step requires restarting the [.noloc]`Amazon VPC CNI plugin for Kubernetes Pods`, which takes several seconds. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl get nodes -o wide -l vpc.amazonaws.com/has-trunk-attached=true -- ----- -+ -Once the trunk network interface is created, [.noloc]`Pods` are assigned secondary IP addresses from the trunk or standard network interfaces. The trunk interface is automatically deleted if the node is deleted. -+ -When you deploy a security group for a [.noloc]`Pod` in a later step, the VPC resource controller creates a special network interface called a _branch network interface_ with a description of `aws-k8s-branch-eni` and associates the security groups to it. Branch network interfaces are created in addition to the standard and trunk network interfaces attached to the node. -+ -If you are using liveness or readiness probes, then you also need to disable _TCP early demux_, so that the `kubelet` can connect to [.noloc]`Pods` on branch network interfaces using TCP. To disable _TCP early demux_, run the following command: -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl patch daemonset aws-node -n kube-system \ - -p '{"spec": {"template": {"spec": {"initContainers": [{"env":[{"name":"DISABLE_TCP_EARLY_DEMUX","value":"true"}],"name":"aws-vpc-cni-init"}]}}}}' ----- -+ -NOTE: If you're using `1.11.0` or later of the [.noloc]`Amazon VPC CNI plugin for Kubernetes` add-on and set `POD_SECURITY_GROUP_ENFORCING_MODE`=``standard``, as described in the next step, then you don't need to run the previous command. -. If your cluster uses `NodeLocal DNSCache`, or you want to use [.noloc]`Calico` network policy with your [.noloc]`Pods` that have their own security groups, or you have [.noloc]`Kubernetes` services of type `NodePort` and `LoadBalancer` using instance targets with an `externalTrafficPolicy` set to `Local` for [.noloc]`Pods` that you want to assign security groups to, then you must be using version `1.11.0` or later of the [.noloc]`Amazon VPC CNI plugin for Kubernetes` add-on, and you must enable the following setting: -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl set env daemonset aws-node -n kube-system POD_SECURITY_GROUP_ENFORCING_MODE=standard ----- -+ -IMPORTANT: -** [.noloc]`Pod` security group rules aren't applied to traffic between [.noloc]`Pods` or between [.noloc]`Pods` and [.noloc]`services`, such as `kubelet` or `nodeLocalDNS`, that are on the same node. Pods using different security groups on the same node can't communicate because they are configured in different subnets, and routing is disabled between these subnets. -** Outbound traffic from [.noloc]`Pods` to addresses outside of the VPC is network address translated to the IP address of the instance's primary network interface (unless you've also set `AWS_VPC_K8S_CNI_EXTERNALSNAT=true`). For this traffic, the rules in the security groups for the primary network interface are used, rather than the rules in the [.noloc]`Pod's` security groups. -** For this setting to apply to existing [.noloc]`Pods`, you must restart the [.noloc]`Pods` or the nodes that the [.noloc]`Pods` are running on. - -. To see how to use a security group policy for your [.noloc]`Pod`, see <>. - - -[.topic] -[[sg-pods-example-deployment,sg-pods-example-deployment.title]] -==== Use a security group policy for an Amazon EKS [.noloc]`Pod` - -To use security groups for [.noloc]`Pods`, you must have an existing security group. The following steps show you how to use the security group policy for a [.noloc]`Pod`. Unless otherwise noted, complete all steps from the same terminal because variables are used in the following steps that don't persist across terminals. - -If you have a [.noloc]`Pod` with Amazon EC2 instances, you must configure the plugin before you use this procedure. For more information, see <>. - -. Create a [.noloc]`Kubernetes` namespace to deploy resources to. You can replace [.replaceable]`my-namespace` with the name of a namespace that you want to use. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl create namespace my-namespace ----- -. [[deploy-securitygrouppolicy]]Deploy an Amazon EKS `SecurityGroupPolicy` to your cluster. -+ -.. Copy the following contents to your device. You can replace [.replaceable]`podSelector` with `serviceAccountSelector` if you'd rather select [.noloc]`Pods` based on service account labels. You must specify one selector or the other. An empty `podSelector` (example: `podSelector: {}`) selects all [.noloc]`Pods` in the namespace. You can change [.replaceable]`my-role` to the name of your role. An empty `serviceAccountSelector` selects all service accounts in the namespace. You can replace [.replaceable]`my-security-group-policy` with a name for your `SecurityGroupPolicy` and [.replaceable]`my-namespace` with the namespace that you want to create the `SecurityGroupPolicy` in. -+ -You must replace [.replaceable]`my_pod_security_group_id` with the ID of an existing security group. If you don't have an existing security group, then you must create one. For more information, see link:AWSEC2/latest/UserGuide/ec2-security-groups.html[Amazon EC2 security groups for Linux instances,type="documentation"] in the link:AWSEC2/latest/UserGuide/[Amazon EC2 User Guide,type="documentation"]. You can specify 1-5 security group IDs. If you specify more than one ID, then the combination of all the rules in all the security groups are effective for the selected [.noloc]`Pods`. -+ -[source,yaml,subs="verbatim,attributes"] ----- -cat >my-security-group-policy.yaml <sample-application.yaml < -my-deployment-5df6f7687b-j9fl4 1/1 Running 0 7m51s 192.168.70.145 ip-192-168-92-33.region-code.compute.internal -my-deployment-5df6f7687b-rjxcz 1/1 Running 0 7m51s 192.168.73.207 ip-192-168-92-33.region-code.compute.internal -my-deployment-5df6f7687b-zmb42 1/1 Running 0 7m51s 192.168.63.27 ip-192-168-33-28.region-code.compute.internal ----- -+ -[NOTE] -==== -Try these tips if any [.noloc]`Pods` are stuck. - -* If any [.noloc]`Pods` are stuck in the `Waiting` state, then run `kubectl describe pod [.replaceable]``my-deployment-xxxxxxxxxx-xxxxx`` -n [.replaceable]``my-namespace```. If you see `Insufficient permissions: Unable to create Elastic Network Interface.`, confirm that you added the IAM policy to the IAM cluster role in a previous step. -* If any [.noloc]`Pods` are stuck in the `Pending` state, confirm that your node instance type is listed in https://github.com/aws/amazon-vpc-resource-controller-k8s/blob/master/pkg/aws/vpc/limits.go[limits.go] and that the product of the maximum number of branch network interfaces supported by the instance type multiplied times the number of nodes in your node group hasn't already been met. For example, an `m5.large` instance supports nine branch network interfaces. If your node group has five nodes, then a maximum of 45 branch network interfaces can be created for the node group. The 46th [.noloc]`Pod` that you attempt to deploy will sit in `Pending` state until another [.noloc]`Pod` that has associated security groups is deleted. - -==== -+ -If you run `kubectl describe pod [.replaceable]``my-deployment-xxxxxxxxxx-xxxxx`` -n [.replaceable]``my-namespace``` and see a message similar to the following message, then it can be safely ignored. This message might appear when the [.noloc]`Amazon VPC CNI plugin for Kubernetes` tries to set up host networking and fails while the network interface is being created. The plugin logs this event until the network interface is created. -+ -[source,bash,subs="verbatim,attributes"] ----- -Failed to create Pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "e24268322e55c8185721f52df6493684f6c2c3bf4fd59c9c121fd4cdc894579f" network for Pod "my-deployment-5df6f7687b-4fbjm": networkPlugin -cni failed to set up Pod "my-deployment-5df6f7687b-4fbjm-c89wx_my-namespace" network: add cmd: failed to assign an IP address to container ----- -+ -You can't exceed the maximum number of [.noloc]`Pods` that can be run on the instance type. For a list of the maximum number of [.noloc]`Pods` that you can run on each instance type, see https://github.com/awslabs/amazon-eks-ami/blob/main/templates/shared/runtime/eni-max-pods.txt[eni-max-pods.txt] on [.noloc]`GitHub`. When you delete a [.noloc]`Pod` that has associated security groups, or delete the node that the [.noloc]`Pod` is running on, the VPC resource controller deletes the branch network interface. If you delete a cluster with [.noloc]`Pods` using [.noloc]`Pods` for security groups, then the controller doesn't delete the branch network interfaces, so you'll need to delete them yourself. For information about how to delete network interfaces, see link:AWSEC2/latest/UserGuide/using-eni.html#delete_eni[Delete a network interface,type="documentation"] in the Amazon EC2 User Guide. -. In a separate terminal, shell into one of the [.noloc]`Pods`. For the remainder of this topic, this terminal is referred to as `TerminalB`. Replace [.replaceable]`5df6f7687b-4fbjm` with the ID of one of the [.noloc]`Pods` returned in your output from the previous step. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl exec -it -n my-namespace my-deployment-5df6f7687b-4fbjm -- /bin/bash ----- -. From the shell in `TerminalB`, confirm that the sample application works. -+ -[source,bash,subs="verbatim,attributes"] ----- -curl my-app ----- -+ -An example output is as follows. -+ -[source,html,subs="verbatim"] ----- - - - -Welcome to nginx! -[...] ----- -+ -You received the output because all [.noloc]`Pods` running the application are associated with the security group that you created. That group contains a rule that allows all traffic between all [.noloc]`Pods` that the security group is associated to. DNS traffic is allowed outbound from that security group to the cluster security group, which is associated with your nodes. The nodes are running the [.noloc]`CoreDNS` [.noloc]`Pods`, which your [.noloc]`Pods` did a name lookup to. -. From `TerminalA`, remove the security group rules that allow DNS communication to the cluster security group from your security group. If you didn't add the DNS rules to the cluster security group in a previous step, then replace [.replaceable]`$my_cluster_security_group_id` with the ID of the security group that you created the rules in. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws ec2 revoke-security-group-ingress --group-id $my_cluster_security_group_id --security-group-rule-ids $my_tcp_rule_id -aws ec2 revoke-security-group-ingress --group-id $my_cluster_security_group_id --security-group-rule-ids $my_udp_rule_id ----- -. From `TerminalB`, attempt to access the application again. -+ -[source,bash,subs="verbatim,attributes"] ----- -curl my-app ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -curl: (6) Could not resolve host: my-app ----- -+ -The attempt fails because the [.noloc]`Pod` is no longer able to access the [.noloc]`CoreDNS` [.noloc]`Pods`, which have the cluster security group associated to them. The cluster security group no longer has the security group rules that allow DNS communication from the security group associated to your [.noloc]`Pod`. -+ -If you attempt to access the application using the IP addresses returned for one of the [.noloc]`Pods` in a previous step, you still receive a response because all ports are allowed between [.noloc]`Pods` that have the security group associated to them and a name lookup isn't required. -. Once you've finished experimenting, you can remove the sample security group policy, application, and security group that you created. Run the following commands from `TerminalA`. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl delete namespace my-namespace -aws ec2 revoke-security-group-ingress --group-id $my_pod_security_group_id --security-group-rule-ids $my_inbound_self_rule_id -wait -sleep 45s -aws ec2 delete-security-group --group-id $my_pod_security_group_id ----- - - -[.topic] -[[pod-multiple-network-interfaces,pod-multiple-network-interfaces.title]] -=== Attach multiple network interfaces to [.noloc]`Pods` with [.noloc]`Multus` - -[abstract] --- -Learn how to use Multus CNI to attach multiple network interfaces to a [.noloc]`Pod` in Amazon EKS for advanced networking scenarios, while leveraging the [.noloc]`Amazon VPC CNI` plugin for primary networking. --- - -Multus CNI is a container network interface (CNI) plugin for Amazon EKS that enables attaching multiple network interfaces to a [.noloc]`Pod`. For more information, see the https://github.com/k8snetworkplumbingwg/multus-cni[Multus-CNI] documentation on [.noloc]`GitHub`. - -In Amazon EKS, each [.noloc]`Pod` has one network interface assigned by the Amazon VPC CNI plugin. With Multus, you can create a multi-homed [.noloc]`Pod` that has multiple interfaces. This is accomplished by Multus acting as a "meta-plugin"; a CNI plugin that can call multiple other CNI plugins. {aws} support for Multus comes configured with the Amazon VPC CNI plugin as the default delegate plugin. - -* Amazon EKS won't be building and publishing single root I/O virtualization (SR-IOV) and Data Plane Development Kit (DPDK) CNI plugins. However, you can achieve packet acceleration by connecting directly to Amazon EC2 Elastic Network Adapters (ENA) through Multus managed host-device and `ipvlan` plugins. -* Amazon EKS is supporting Multus, which provides a generic process that enables simple chaining of additional CNI plugins. Multus and the process of chaining is supported, but {aws} won't provide support for all compatible CNI plugins that can be chained, or issues that may arise in those CNI plugins that are unrelated to the chaining configuration. -* Amazon EKS is providing support and life cycle management for the Multus plugin, but isn't responsible for any IP address or additional management associated with the additional network interfaces. The IP address and management of the default network interface utilizing the Amazon VPC CNI plugin remains unchanged. -* Only the Amazon VPC CNI plugin is officially supported as the default delegate plugin. You need to modify the published Multus installation manifest to reconfigure the default delegate plugin to an alternate CNI if you choose not to use the Amazon VPC CNI plugin for primary networking. -* Multus is only supported when using the Amazon VPC CNI as the primary CNI. We do not support the Amazon VPC CNI when used for higher order interfaces, secondary or otherwise. -* To prevent the Amazon VPC CNI plugin from trying to manage additional network interfaces assigned to [.noloc]`Pods`, add the following tag to the network interface: -+ -*key*:: -: `node.k8s.amazonaws.com/no_manage` -+ -*value*:: -: `true` -* Multus is compatible with network policies, but the policy has to be enriched to include ports and IP addresses that may be part of additional network interfaces attached to [.noloc]`Pods`. - -For an implementation walk through, see the https://github.com/aws-samples/eks-install-guide-for-multus/blob/main/README.md[Multus Setup Guide] on [.noloc]`GitHub`. - - -[.topic] -[[alternate-cni-plugins,alternate-cni-plugins.title]] -== Alternate CNI plugins for Amazon EKS clusters - -[abstract] --- -Learn how to use alternate network and security plugins on Amazon EKS to customize networking for your [.noloc]`Kubernetes` clusters on Amazon EC2 nodes. --- - -The https://github.com/aws/amazon-vpc-cni-plugins[Amazon VPC CNI plugin for Kubernetes] is the only CNI plugin supported by Amazon EKS with Amazon EC2 nodes. Amazon EKS supports the core capabilities of Cilium and Calico for Amazon EKS Hybrid Nodes. Amazon EKS runs upstream [.noloc]`Kubernetes`, so you can install alternate compatible CNI plugins to Amazon EC2 nodes in your cluster. If you have Fargate nodes in your cluster, the [.noloc]`Amazon VPC CNI plugin for Kubernetes` is already on your Fargate nodes. It's the only CNI plugin you can use with Fargate nodes. An attempt to install an alternate CNI plugin on Fargate nodes fails. - -If you plan to use an alternate CNI plugin on Amazon EC2 nodes, we recommend that you obtain commercial support for the plugin or have the in-house expertise to troubleshoot and contribute fixes to the CNI plugin project. - -Amazon EKS maintains relationships with a network of partners that offer support for alternate compatible CNI plugins. For details about the versions, qualifications, and testing performed, see the following partner documentation. - -[cols="1,1,1", options="header"] -|=== -|Partner -|Product -|Documentation - - -|Tigera -|https://www.tigera.io/partners/aws/[Calico] -|https://docs.projectcalico.org/getting-started/kubernetes/managed-public-cloud/eks[Installation instructions] - -|Isovalent -|https://cilium.io[Cilium] -|https://docs.cilium.io/en/stable/gettingstarted/k8s-install-default/[Installation instructions] - -|Juniper -|https://www.juniper.net/us/en/products/sdn-and-orchestration/contrail/cloud-native-contrail-networking.html[Cloud-Native Contrail Networking (CN2)] -|https://www.juniper.net/documentation/us/en/software/cn-cloud-native23.2/cn-cloud-native-eks-install-and-lcm/index.html[Installation instructions] - -|VMware -|https://antrea.io/[Antrea] -|https://antrea.io/docs/main/docs/eks-installation[Installation instructions] -|=== - -Amazon EKS aims to give you a wide selection of options to cover all use cases. - - -[[alternate-network-policy-plugins,alternate-network-policy-plugins.title]] -=== Alternate compatible network policy plugins - -https://www.tigera.io/project-calico[Calico] is a widely adopted solution for container networking and security. Using [.noloc]`Calico` on EKS provides a fully compliant network policy enforcement for your EKS clusters. Additionally, you can opt to use [.noloc]`Calico's` networking, which conserve IP addresses from your underlying VPC. https://www.tigera.io/tigera-products/calico-cloud/[Calico Cloud] enhances the features of [.noloc]`Calico Open Source`, providing advanced security and observability capabilities. - -Traffic flow to and from [.noloc]`Pods` with associated security groups are not subjected to [.noloc]`Calico` network policy enforcement and are limited to Amazon VPC security group enforcement only. - -If you use [.noloc]`Calico` network policy enforcement, we recommend that you set the environment variable `ANNOTATE_POD_IP` to `true` to avoid a known issue with [.noloc]`Kubernetes`. To use this feature, you must add `patch` permission for pods to the `aws-node` [.noloc]`ClusterRole`. Note that adding patch permissions to the `aws-node` [.noloc]`DaemonSet` increases the security scope for the plugin. For more information, see https://github.com/aws/amazon-vpc-cni-k8s/?tab=readme-ov-file#annotate_pod_ip-v193[ANNOTATE_POD_IP] in the VPC CNI repo on GitHub. - -=== Considerations for Amazon EKS Auto Mode - -Amazon EKS Auto Mode does not support alternate CNI plugins or network policy plugins. For more information, see <>. - -[.topic] -[[aws-load-balancer-controller,aws-load-balancer-controller.title]] -== Route internet traffic with {aws} Load Balancer Controller - -[abstract] --- -Learn how to configure and use the [.noloc]`{aws} Load Balancer Controller` to expose [.noloc]`Kubernetes` cluster apps to the internet with {aws} Elastic Load Balancing for [.noloc]`Kubernetes` [.noloc]`services` and [.noloc]`ingresses`. --- - -The [.noloc]`{aws} Load Balancer Controller` manages {aws} Elastic Load Balancers for a [.noloc]`Kubernetes` cluster. You can use the controller to expose your cluster apps to the internet. The controller provisions {aws} load balancers that point to cluster Service or Ingress resources. In other words, the controller creates a single IP address or DNS name that points to multiple pods in your cluster. - - - -image::images/lbc-overview.png["Architecture diagram. Illustration of traffic coming from internet users, to Amazon Load Balancer. Amazon Load Balancer distributes traffic to pods in the cluster.",scaledwidth=50%] - -The controller watches for [.noloc]`Kubernetes` [.noloc]`Ingress` or [.noloc]`Service` resources. In response, it creates the appropriate {aws} Elastic Load Balancing resources. You can configure the specific behavior of the load balancers by applying annotations to the [.noloc]`Kubernetes` resources. For example, you can attach {aws} security groups to load balancers using annotations. - -The controller provisions the following resources: - - - -*[.noloc]`Kubernetes` `Ingress`*:: -The LBC creates an link:elasticloadbalancing/latest/application/introduction.html[{aws} Application Load Balancer (ALB),type="documentation"] when you create a [.noloc]`Kubernetes` `Ingress`. https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/ingress/annotations/[Review the annotations you can apply to an Ingress resource.] - - -*[.noloc]`Kubernetes` service of the `LoadBalancer` type*:: -The LBC creates an link:elasticloadbalancing/latest/network/introduction.html[{aws} Network Load Balancer (NLB),type="documentation"]when you create a [.noloc]`Kubernetes` service of type `LoadBalancer`. https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/service/annotations/[Review the annotations you can apply to a Service resource.] -+ -In the past, the [.noloc]`Kubernetes` network load balancer was used for _instance_ targets, but the LBC was used for _IP_ targets. With the [.noloc]`{aws} Load Balancer Controller` version `2.3.0` or later, you can create NLBs using either target type. For more information about NLB target types, see link:elasticloadbalancing/latest/network/load-balancer-target-groups.html#target-type[Target type,type="documentation"] in the User Guide for Network Load Balancers. - -The controller is an https://github.com/kubernetes-sigs/aws-load-balancer-controller[open-source project] managed on [.noloc]`GitHub`. - -Before deploying the controller, we recommend that you review the prerequisites and considerations in <> and <>. In those topics, you will deploy a sample app that includes an {aws} load balancer. - - -[[lbc-overview,lbc-overview.title]] -=== Install the controller - -You can use one of the following procedures to install the [.noloc]`{aws} Load Balancer Controller`: - - - -* If you are new to Amazon EKS we recommend that you use Helm for the installation because it simplifies the [.noloc]`{aws} Load Balancer Controller` installation. For more information, see <>. -* For advanced configurations, such as clusters with restricted network access to public container registries, use [.noloc]`Kubernetes` Manifests. For more information, see <>. - - -[[lbc-deprecated,lbc-deprecated.title]] -=== Migrate from deprecated controller versions - -* If you have deprecated versions of the [.noloc]`{aws} Load Balancer Controller` installed, see <>. -* Deprecated versions cannot be upgraded. They must be removed and a current version of the [.noloc]`{aws} Load Balancer Controller` installed. -+ -[[lbc-deprecated-list]] -* Deprecated versions include: -+ -** {aws} ALB Ingress Controller for [.noloc]`Kubernetes` ("Ingress Controller"), a predecessor to the [.noloc]`{aws} Load Balancer Controller`. -** Any `0.1.[.replaceable]``x``` version of the [.noloc]`{aws} Load Balancer Controller` - - -[[lbc-legacy,lbc-legacy.title]] -=== Legacy cloud provider - -[.noloc]`Kubernetes` includes a legacy cloud provider for {aws}. The legacy cloud provider is capable of provisioning {aws} load balancers, similar to the [.noloc]`{aws} Load Balancer Controller`. The legacy cloud provider creates Classic Load Balancers. If you do not install the [.noloc]`{aws} Load Balancer Controller`, [.noloc]`Kubernetes` will default to using the legacy cloud provider. You should install the [.noloc]`{aws} Load Balancer Controller` and avoid using the legacy cloud provider. - -[IMPORTANT] -==== - -In versions 2.5 and newer, the [.noloc]`{aws} Load Balancer Controller` becomes the default controller for [.noloc]`Kubernetes` _service_ resources with the `type: LoadBalancer` and makes an {aws} Network Load Balancer (NLB) for each service. It does this by making a mutating webhook for services, which sets the `spec.loadBalancerClass` field to `service.k8s.aws/nlb` for new services of `type: LoadBalancer`. You can turn off this feature and revert to using the https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/service/annotations/#legacy-cloud-provider[legacy Cloud Provider] as the default controller, by setting the helm chart value `enableServiceMutatorWebhook` to `false`. The cluster won't provision new Classic Load Balancers for your services unless you turn off this feature. Existing Classic Load Balancers will continue to work. - -==== - - -[.topic] -[[lbc-helm,lbc-helm.title]] -=== Install [.noloc]`{aws} Load Balancer Controller` with [.noloc]`Helm` - -[abstract] --- -Learn how to install the [.noloc]`{aws} Load Balancer Controller` on Amazon EKS using Helm to manage K8s load balancing with {aws} Cloud. Discover the prerequisites and steps for creating an IAM role, installing with Helm, and verifying the controller deployment. --- - -[TIP] -==== -With Amazon EKS Auto Mode, you don't need to install or upgrade networking add-ons. Auto Mode includes pod networking and load balancing capabilities. - -For more information, see <>. -==== - -This topic describes how to install the [.noloc]`{aws} Load Balancer Controller` using Helm, a package manager for [.noloc]`Kubernetes`, and `eksctl`. The controller is installed with default options. For more information about the controller, including details on configuring it with annotations, see the https://kubernetes-sigs.github.io/aws-load-balancer-controller/[{aws} Load Balancer Controller Documentation] on [.noloc]`GitHub`. - -In the following steps, replace the [.replaceable]`example values` with your own values. - -[[lbc-prereqs,lbc-prereqs.title]] -==== Prerequisites - -Before starting this tutorial, you must install and configure the following tools and resources that you need to create and manage an Amazon EKS cluster. - - - -* An existing Amazon EKS cluster. To deploy one, see <>. -* An existing {aws} Identity and Access Management (IAM) [.noloc]`OpenID Connect` ([.noloc]`OIDC`) provider for your cluster. To determine whether you already have one, or to create one, see <>. -* Make sure that your [.noloc]`Amazon VPC CNI plugin for Kubernetes`, `kube-proxy`, and [.noloc]`CoreDNS` add-ons are at the minimum versions listed in <>. -* Familiarity with {aws} Elastic Load Balancing. For more information, see the link:elasticloadbalancing/latest/userguide/[Elastic Load Balancing User Guide,type="documentation"]. -* Familiarity with Kubernetes https://kubernetes.io/docs/concepts/services-networking/service/[service] and https://kubernetes.io/docs/concepts/services-networking/ingress/[ingress] resources. - - -* https://helm.sh/docs/helm/helm_install/[Helm] installed locally. - - -[[lbc-helm-iam,lbc-helm-iam.title]] -==== Step 1: Create IAM Role using `eksctl` - -[NOTE] -==== - -You only need to create an IAM Role for the [.noloc]`{aws} Load Balancer Controller` once per {aws-account}. Check if `AmazonEKSLoadBalancerControllerRole` exists in the link:iam[IAM Console,type="console"]. If this role exists, skip to <>. - -==== - -[NOTE] -==== - -Below example is referring to the [.noloc]`{aws} Load Balancer Controller` **v2.11.0** release version. For more information about all releases, see the https://github.com/kubernetes-sigs/aws-load-balancer-controller/releases/[{aws} Load Balancer Controller Release Page] on [.noloc]`GitHub`. - -==== - -. Download an IAM policy for the [.noloc]`{aws} Load Balancer Controller` that allows it to make calls to {aws} APIs on your behalf. -+ -==== -[role="tablist"] -{aws}::: -+ -[source,shell,subs="verbatim,attributes"] ----- -curl -O https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.11.0/docs/install/iam_policy.json ----- - - -{aws} GovCloud (US)::: -+ -[source,shell,subs="verbatim,attributes"] ----- -curl -O https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.11.0/docs/install/iam_policy_us-gov.json ----- -+ -[source,shell,subs="verbatim,attributes"] ----- -mv iam_policy_us-gov.json iam_policy.json ----- -==== -+ -. Create an IAM policy using the policy downloaded in the previous step. -+ -[source,shell,subs="verbatim,attributes"] ----- -aws iam create-policy \ - --policy-name AWSLoadBalancerControllerIAMPolicy \ - --policy-document file://iam_policy.json ----- -+ -NOTE: If you view the policy in the {aws-management-console}, the console shows warnings for the *ELB* service, but not for the *ELB v2* service. This happens because some of the actions in the policy exist for *ELB v2*, but not for *ELB*. You can ignore the warnings for *ELB*. -. Replace [.replaceable]`my-cluster` with the name of your cluster, [.replaceable]`111122223333` with your account ID, and then run the command. If your cluster is in the {aws} GovCloud (US-East) or {aws} GovCloud (US-West) {aws} Regions, then replace `{arn-aws}` with `arn:aws-us-gov:`. -+ -[source,shell,subs="verbatim,attributes"] ----- -eksctl create iamserviceaccount \ - --cluster=my-cluster \ - --namespace=kube-system \ - --name=aws-load-balancer-controller \ - --role-name AmazonEKSLoadBalancerControllerRole \ - --attach-policy-arn={arn-aws}iam::111122223333:policy/AWSLoadBalancerControllerIAMPolicy \ - --approve ----- - - -[[lbc-helm-install,lbc-helm-install.title]] -==== Step 2: Install [.noloc]`{aws} Load Balancer Controller` - -. Add the `eks-charts` Helm chart repository. {aws} maintains https://github.com/aws/eks-charts[this repository] on GitHub. -+ -[source,shell,subs="verbatim,attributes"] ----- -helm repo add eks https://aws.github.io/eks-charts ----- -. Update your local repo to make sure that you have the most recent charts. -+ -[source,shell,subs="verbatim,attributes"] ----- -helm repo update eks ----- -. Install the [.noloc]`{aws} Load Balancer Controller`. -+ -If you're deploying the controller to Amazon EC2 nodes that have https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node[restricted access to the Amazon EC2 instance metadata service (IMDS)], or if you're deploying to Fargate or Amazon EKS Hybrid Nodes, then add the following flags to the `helm` command that follows: -+ -*** `--set region=[.replaceable]``region-code``` -*** `--set vpcId=[.replaceable]``vpc-xxxxxxxx``` -+ -Replace [.replaceable]`my-cluster` with the name of your cluster. In the following command, `aws-load-balancer-controller` is the [.noloc]`Kubernetes` service account that you created in a previous step. -+ -For more information about configuring the helm chart, see https://github.com/aws/eks-charts/blob/master/stable/aws-load-balancer-controller/values.yaml[values.yaml] on GitHub. -+ -[source,shell,subs="verbatim,attributes"] ----- -helm install aws-load-balancer-controller eks/aws-load-balancer-controller \ - -n kube-system \ - --set clusterName=my-cluster \ - --set serviceAccount.create=false \ - --set serviceAccount.name=aws-load-balancer-controller ----- - - -[IMPORTANT] -==== -The deployed chart doesn't receive security updates automatically. You need to manually upgrade to a newer chart when it becomes available. When upgrading, change [.replaceable]`install` to `upgrade` in the previous command. -==== - -The `helm install` command automatically installs the custom resource definitions ([.noloc]`CRDs`) for the controller. The `helm upgrade` command does not. If you use `helm upgrade,` you must manually install the [.noloc]`CRDs`. Run the following command to install the [.noloc]`CRDs`: - -[source,shell,subs="verbatim,attributes"] ----- -wget https://raw.githubusercontent.com/aws/eks-charts/master/stable/aws-load-balancer-controller/crds/crds.yaml -kubectl apply -f crds.yaml ----- - - -[[lbc-helm-verify,lbc-helm-verify.title]] -==== Step 3: Verify that the controller is installed -. Verify that the controller is installed. -+ -[source,shell,subs="verbatim,attributes"] ----- -kubectl get deployment -n kube-system aws-load-balancer-controller ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -NAME READY UP-TO-DATE AVAILABLE AGE -aws-load-balancer-controller 2/2 2 2 84s ----- -+ -You receive the previous output if you deployed using Helm. If you deployed using the [.noloc]`Kubernetes` manifest, you only have one replica. -. Before using the controller to provision {aws} resources, your cluster must meet specific requirements. For more information, see <> and <>. - -// GDC Must Fix - -[.topic] -[[lbc-manifest,lbc-manifest.title]] -=== Install [.noloc]`{aws} Load Balancer Controller` with manifests - -[abstract] --- -Install the [.noloc]`{aws} Load Balancer Controller` add-on for Amazon EKS using [.noloc]`Kubernetes` manifests to provision Elastic Load Balancing resources. Configure IAM role and install `cert-manager` before applying controller manifest. --- - -[TIP] -==== -With Amazon EKS Auto Mode, you don't need to install or upgrade networking add-ons. Auto Mode includes pod networking and load balancing capabilities. - -For more information, see <>. -==== - -This topic describes how to install the controller by downloading and applying [.noloc]`Kubernetes` manifests. You can view the full https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/[documentation] for the controller on [.noloc]`GitHub`. - -In the following steps, replace the [.replaceable]`example values` with your own values. - -[[lbc-manifest-prereqs,lbc-manifest-prereqs.title]] -==== Prerequisites - -Before starting this tutorial, you must install and configure the following tools and resources that you need to create and manage an Amazon EKS cluster. - - - -* An existing Amazon EKS cluster. To deploy one, see <>. -* An existing {aws} Identity and Access Management (IAM) [.noloc]`OpenID Connect` ([.noloc]`OIDC`) provider for your cluster. To determine whether you already have one, or to create one, see <>. -* Make sure that your [.noloc]`Amazon VPC CNI plugin for Kubernetes`, `kube-proxy`, and [.noloc]`CoreDNS` add-ons are at the minimum versions listed in <>. -* Familiarity with {aws} Elastic Load Balancing. For more information, see the link:elasticloadbalancing/latest/userguide/[Elastic Load Balancing User Guide,type="documentation"]. -* Familiarity with Kubernetes https://kubernetes.io/docs/concepts/services-networking/service/[service] and https://kubernetes.io/docs/concepts/services-networking/ingress/[ingress] resources. - - -[[lbc-iam,lbc-iam.title]] -==== Step 1: Configure IAM - -[NOTE] -==== - -You only need to create a role for the [.noloc]`{aws} Load Balancer Controller` one per {aws} account. Check if `AmazonEKSLoadBalancerControllerRole` exists in the link:iam[IAM Console,type="console"]. If this role exists, skip to <>. - -==== - -[NOTE] -==== - -Below example is referring to the [.noloc]`{aws} Load Balancer Controller` **v2.11.0** release version. For more inforamtion about all releases, see the https://github.com/kubernetes-sigs/aws-load-balancer-controller/releases/[{aws} Load Balancer Controller Release Page] on [.noloc]`GitHub`. - -==== - -. Download an IAM policy for the [.noloc]`{aws} Load Balancer Controller` that allows it to make calls to {aws} APIs on your behalf. -+ -==== -[role="tablist"] -{aws}::: -+ -[source,shell,subs="verbatim,attributes"] ----- -curl -O https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.11.0/docs/install/iam_policy.json ----- - - -{aws} GovCloud (US)::: -+ -[source,shell,subs="verbatim,attributes"] ----- -curl -O https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.11.0/docs/install/iam_policy_us-gov.json ----- -+ -[source,shell,subs="verbatim,attributes"] ----- -mv iam_policy_us-gov.json iam_policy.json ----- -==== -. Create an IAM policy using the policy downloaded in the previous step. -+ -[source,shell,subs="verbatim,attributes"] ----- -aws iam create-policy \ - --policy-name AWSLoadBalancerControllerIAMPolicy \ - --policy-document file://iam_policy.json ----- -+ -NOTE: If you view the policy in the {aws-management-console}, the console shows warnings for the *ELB* service, but not for the *ELB v2* service. This happens because some of the actions in the policy exist for *ELB v2*, but not for *ELB*. You can ignore the warnings for *ELB*. - -==== -[role="tablist"] -eksctl::: -.. Replace [.replaceable]`my-cluster` with the name of your cluster, [.replaceable]`111122223333` with your account ID, and then run the command. If your cluster is in the {aws} GovCloud (US-East) or {aws} GovCloud (US-West) {aws} Regions, then replace `{arn-aws}` with `arn:aws-us-gov:`. -+ -[source,shell,subs="verbatim,attributes"] ----- -eksctl create iamserviceaccount \ - --cluster=my-cluster \ - --namespace=kube-system \ - --name=aws-load-balancer-controller \ - --role-name AmazonEKSLoadBalancerControllerRole \ - --attach-policy-arn={arn-aws}iam::111122223333:policy/AWSLoadBalancerControllerIAMPolicy \ - --approve ----- - - -{aws} CLI and kubectl::: -.. Retrieve your cluster's [.noloc]`OIDC` provider ID and store it in a variable. -+ -[source,bash,subs="verbatim,attributes"] ----- -oidc_id=$(aws eks describe-cluster --name my-cluster --query "cluster.identity.oidc.issuer" --output text | cut -d '/' -f 5) ----- -.. Determine whether an IAM [.noloc]`OIDC` provider with your cluster's ID is already in your account. You need [.noloc]`OIDC` configured for both the cluster and IAM. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws iam list-open-id-connect-providers | grep $oidc_id | cut -d "/" -f4 ----- -+ -If output is returned, then you already have an IAM [.noloc]`OIDC` provider for your cluster. If no output is returned, then you must create an IAM [.noloc]`OIDC` provider for your cluster. For more information, see <>. -.. Copy the following contents to your device. Replace [.replaceable]`111122223333` with your account ID. Replace [.replaceable]`region-code` with the {aws} Region that your cluster is in. Replace [.replaceable]`EXAMPLED539D4633E53DE1B71EXAMPLE` with the output returned in the previous step. If your cluster is in the {aws} GovCloud (US-East) or {aws} GovCloud (US-West) {aws} Regions, then replace `{arn-aws}` with `arn:aws-us-gov:`. After replacing the text, run the modified command to create the `load-balancer-role-trust-policy.json` file. -+ -[source,json,subs="verbatim,attributes"] ----- -cat >load-balancer-role-trust-policy.json <aws-load-balancer-controller-service-account.yaml <>. -+ -[source,bash,subs="verbatim,attributes"] ----- -quay.io/jetstack/cert-manager-cainjector:v1.13.5 -quay.io/jetstack/cert-manager-controller:v1.13.5 -quay.io/jetstack/cert-manager-webhook:v1.13.5 ----- -.. Replace `quay.io` in the manifest for the three images with your own registry name. The following command assumes that your private repository's name is the same as the source repository. Replace [.replaceable]`111122223333.dkr.ecr.region-code.amazonaws.com` with your private registry. -+ -[source,shell,subs="verbatim,attributes"] ----- -sed -i.bak -e 's|quay.io|111122223333.dkr.ecr.region-code.amazonaws.com|' ./cert-manager.yaml ----- -.. Apply the manifest. -+ -[source,shell,subs="verbatim,attributes"] ----- -kubectl apply \ - --validate=false \ - -f ./cert-manager.yaml ----- -==== - - -[[lbc-install,lbc-install.title]] -==== Step 3: Install [.noloc]`{aws} Load Balancer Controller` -. Download the controller specification. For more information about the controller, see the https://kubernetes-sigs.github.io/aws-load-balancer-controller/[documentation] on [.noloc]`GitHub`. -+ -[source,bash,subs="verbatim,attributes"] ----- -curl -Lo v2_11_0_full.yaml https://github.com/kubernetes-sigs/aws-load-balancer-controller/releases/download/v2.11.0/v2_11_0_full.yaml ----- -. Make the following edits to the file. -+ -.. If you downloaded the `v2_11_0_full.yaml` file, run the following command to remove the `ServiceAccount` section in the manifest. If you don't remove this section, the required annotation that you made to the service account in a previous step is overwritten. Removing this section also preserves the service account that you created in a previous step if you delete the controller. -+ -[source,shell,subs="verbatim,attributes"] ----- -sed -i.bak -e '690,698d' ./v2_11_0_full.yaml ----- -+ -If you downloaded a different file version, then open the file in an editor and remove the following lines. -+ -[source,yaml,subs="verbatim,attributes"] ----- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: aws-load-balancer-controller - name: aws-load-balancer-controller - namespace: kube-system ---- ----- -.. Replace `your-cluster-name` in the `Deployment` `spec` section of the file with the name of your cluster by replacing [.replaceable]`my-cluster` with the name of your cluster. -+ -[source,shell,subs="verbatim,attributes"] ----- -sed -i.bak -e 's|your-cluster-name|my-cluster|' ./v2_11_0_full.yaml ----- -.. If your nodes don't have access to the Amazon EKS Amazon ECR image repositories, then you need to pull the following image and push it to a repository that your nodes have access to. For more information on how to pull, tag, and push an image to your own repository, see <>. -+ -[source,bash,subs="verbatim,attributes"] ----- -public.ecr.aws/eks/aws-load-balancer-controller:v2.11.0 ----- -+ -Add your registry's name to the manifest. The following command assumes that your private repository's name is the same as the source repository and adds your private registry's name to the file. Replace [.replaceable]`111122223333.dkr.ecr.region-code.amazonaws.com` with your registry. This line assumes that you named your private repository the same as the source repository. If not, change the `eks/aws-load-balancer-controller` text after your private registry name to your repository name. -+ -[source,shell,subs="verbatim,attributes"] ----- -sed -i.bak -e 's|public.ecr.aws/eks/aws-load-balancer-controller|111122223333.dkr.ecr.region-code.amazonaws.com/eks/aws-load-balancer-controller|' ./v2_11_0_full.yaml ----- -.. (Required only for Fargate or Restricted IMDS) -+ -If you're deploying the controller to Amazon EC2 nodes that have https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node[restricted access to the Amazon EC2 instance metadata service (IMDS)], or if you're deploying to Fargate or Amazon EKS Hybrid Nodes, then add the `following parameters` under `- args:`. -+ -[source,yaml,subs="verbatim,attributes"] ----- -[...] -spec: - containers: - - args: - - --cluster-name=your-cluster-name - - --ingress-class=alb - - --aws-vpc-id=vpc-xxxxxxxx - - --aws-region=region-code - - -[...] ----- -. Apply the file. -+ -[source,shell,subs="verbatim,attributes"] ----- -kubectl apply -f v2_11_0_full.yaml ----- -. Download the `IngressClass` and `IngressClassParams` manifest to your cluster. -+ -[source,shell,subs="verbatim,attributes"] ----- -curl -Lo v2_11_0_ingclass.yaml https://github.com/kubernetes-sigs/aws-load-balancer-controller/releases/download/v2.11.0/v2_11_0_ingclass.yaml ----- -. Apply the manifest to your cluster. -+ -[source,shell,subs="verbatim,attributes"] ----- -kubectl apply -f v2_11_0_ingclass.yaml ----- - - -[[lbc-verify,lbc-verify.title]] -==== Step 4: Verify that the controller is installed -. Verify that the controller is installed. -+ -[source,shell,subs="verbatim,attributes"] ----- -kubectl get deployment -n kube-system aws-load-balancer-controller ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -NAME READY UP-TO-DATE AVAILABLE AGE -aws-load-balancer-controller 2/2 2 2 84s ----- -+ -You receive the previous output if you deployed using Helm. If you deployed using the [.noloc]`Kubernetes` manifest, you only have one replica. -. Before using the controller to provision {aws} resources, your cluster must meet specific requirements. For more information, see <> and <>. - - -[.topic] -[[lbc-remove,lbc-remove.title]] -=== Migrate apps from deprecated ALB [.noloc]`Ingress Controller` - -[abstract] --- -Learn how to migrate from the deprecated ALB Ingress Controller to the latest [.noloc]`{aws} Load Balancer Controller` release, ensuring smooth transition and uninterrupted load balancing capabilities. --- - -This topic describes how to migrate from deprecated controller versions. More specifically, it describes how to remove deprecated versions of the [.noloc]`{aws} Load Balancer Controller`. - - - -* Deprecated versions cannot be upgraded. You must remove them first, and then install a current version. -+ -[[lbc-deprecated-list]] -* Deprecated versions include: -+ -** {aws} ALB Ingress Controller for [.noloc]`Kubernetes` ("Ingress Controller"), a predecessor to the [.noloc]`{aws} Load Balancer Controller`. -** Any `0.1.[.replaceable]``x``` version of the [.noloc]`{aws} Load Balancer Controller` - - -[[lbc-remove-desc,lbc-remove-desc.title]] -==== Remove the deprecated controller version - -[NOTE] -==== - -You may have installed the deprecated version using Helm or manually with [.noloc]`Kubernetes` manifests. Complete the procedure using the tool that you originally installed it with. - -==== -. If you installed the `incubator/aws-alb-ingress-controller` Helm chart, uninstall it. -+ -[source,shell,subs="verbatim,attributes"] ----- -helm delete aws-alb-ingress-controller -n kube-system ----- -. If you have version `0.1.[.replaceable]``x``` of the `eks-charts/aws-load-balancer-controller` chart installed, uninstall it. The upgrade from `0.1.[.replaceable]``x``` to version `1.0.0` doesn't work due to incompatibility with the webhook API version. -+ -[source,shell,subs="verbatim,attributes"] ----- -helm delete aws-load-balancer-controller -n kube-system ----- -. Check to see if the controller is currently installed. -+ -[source,shell,subs="verbatim,attributes"] ----- -kubectl get deployment -n kube-system alb-ingress-controller ----- -+ -This is the output if the controller isn't installed. -+ -+ -This is the output if the controller is installed. -+ -[source,bash,subs="verbatim,attributes"] ----- -NAME READY UP-TO-DATE AVAILABLE AGE -alb-ingress-controller 1/1 1 1 122d ----- -. Enter the following commands to remove the controller. -+ -[source,shell,subs="verbatim,attributes"] ----- -kubectl delete -f https://raw.githubusercontent.com/kubernetes-sigs/aws-alb-ingress-controller/v1.1.8/docs/examples/alb-ingress-controller.yaml -kubectl delete -f https://raw.githubusercontent.com/kubernetes-sigs/aws-alb-ingress-controller/v1.1.8/docs/examples/rbac-role.yaml ----- - - -[[lbc-migrate,lbc-migrate.title]] -==== Migrate to [.noloc]`{aws} Load Balancer Controller` - -To migrate from the ALB Ingress Controller for [.noloc]`Kubernetes` to the [.noloc]`{aws} Load Balancer Controller`, you need to: - -. Remove the ALB Ingress Controller (see above). -. <> -. Add an additional policy to the IAM Role used by the [.noloc]`{aws} Load Balancer Controller`. This policy permits the LBC to manage resources created by the ALB Ingress Controller for [.noloc]`Kubernetes`. -. Download the IAM policy. This policy permits the [.noloc]`{aws} Load Balancer Controller` to manage resources created by the ALB Ingress Controller for [.noloc]`Kubernetes`. You can also https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/main/docs/install/iam_policy_v1_to_v2_additional.json[view the policy]. -+ -[source,shell,subs="verbatim,attributes"] ----- -curl -O https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.11.0/docs/install/iam_policy_v1_to_v2_additional.json ----- -. If your cluster is in the {aws} GovCloud (US-East) or {aws} GovCloud (US-West) {aws} Regions, then replace `{arn-aws}` with `arn:aws-us-gov:`.. -+ -[source,shell,subs="verbatim,attributes"] ----- -sed -i.bak -e 's|{arn-aws}|arn:aws-us-gov:|' iam_policy_v1_to_v2_additional.json ----- -. Create the IAM policy and note the ARN that is returned. -+ -[source,shell,subs="verbatim,attributes"] ----- -aws iam create-policy \ - --policy-name AWSLoadBalancerControllerAdditionalIAMPolicy \ - --policy-document file://iam_policy_v1_to_v2_additional.json ----- -. Attach the IAM policy to the IAM role used by the [.noloc]`{aws} Load Balancer Controller`. Replace [.replaceable]`your-role-name` with the name of the role, such as `AmazonEKSLoadBalancerControllerRole`. -+ -If you created the role using `eksctl`, then to find the role name that was created, open the link:cloudformation[{aws} CloudFormation console,type="console"] and select the *eksctl-[.replaceable]`my-cluster`-addon-iamserviceaccount-kube-system-aws-load-balancer-controller* stack. Select the *Resources* tab. The role name is in the *Physical ID* column. If your cluster is in the {aws} GovCloud (US-East) or {aws} GovCloud (US-West) {aws} Regions, then replace `{arn-aws}` with `arn:aws-us-gov:`. -+ -[source,shell,subs="verbatim,attributes"] ----- -aws iam attach-role-policy \ - --role-name your-role-name \ - --policy-arn {arn-aws}iam::111122223333:policy/AWSLoadBalancerControllerAdditionalIAMPolicy ----- - - -[.topic] -[[managing-coredns,managing-coredns.title]] -== Manage CoreDNS for DNS in Amazon EKS clusters - -[abstract] --- -Learn how to manage the [.noloc]`CoreDNS` Amazon EKS add-on for DNS service discovery in [.noloc]`Kubernetes` clusters with configuration updates and version upgrades. --- - -[TIP] -==== -With Amazon EKS Auto Mode, you don't need to install or upgrade networking add-ons. Auto Mode includes pod networking and load balancing capabilities. - -For more information, see <>. -==== - -[.noloc]`CoreDNS` is a flexible, extensible DNS server that can serve as the [.noloc]`Kubernetes` cluster DNS. When you launch an Amazon EKS cluster with at least one node, two replicas of the [.noloc]`CoreDNS` image are deployed by default, regardless of the number of nodes deployed in your cluster. The [.noloc]`CoreDNS` [.noloc]`Pods` provide name resolution for all [.noloc]`Pods` in the cluster. The [.noloc]`CoreDNS` [.noloc]`Pods` can be deployed to Fargate nodes if your cluster includes a Fargate Profile with a namespace that matches the namespace for the [.noloc]`CoreDNS` `deployment`. For more information on Fargate Profiles, see <>. For more information about [.noloc]`CoreDNS`, see https://kubernetes.io/docs/tasks/administer-cluster/coredns/[Using CoreDNS for Service Discovery] in the [.noloc]`Kubernetes` documentation. - -[[coredns-versions,coredns-versions.title]] -=== [.noloc]`CoreDNS` versions - -The following table lists the latest version of the Amazon EKS add-on type for each [.noloc]`Kubernetes` version. - -[options="header"] -|=== -| Kubernetes version | [.noloc]`CoreDNS` version -| 1.31 | v1.11.4-eksbuild.2 -| 1.30 | v1.11.4-eksbuild.2 -| 1.29 | v1.11.4-eksbuild.2 -| 1.28 | v1.10.1-eksbuild.17 -| 1.27 | v1.10.1-eksbuild.17 -| 1.26 | v1.9.3-eksbuild.21 -| 1.25 | v1.9.3-eksbuild.21 -| 1.24 | v1.9.3-eksbuild.21 -| 1.23 | v1.8.7-eksbuild.20 -|=== - -[IMPORTANT] -==== - -If you're self-managing this add-on, the versions in the table might not be the same as the available self-managed versions. For more information about updating the self-managed type of this add-on, see <>. - -==== - -[[coredns-upgrade,coredns-upgrade.title]] -=== Important [.noloc]`CoreDNS` upgrade considerations - -* To improve the stability and availability of the [.noloc]`CoreDNS` [.noloc]`Deployment`, versions `v1.9.3-eksbuild.6` and later and `v1.10.1-eksbuild.3` are deployed with a `PodDisruptionBudget`. If you've deployed an existing `PodDisruptionBudget`, your upgrade to these versions might fail. If the upgrade fails, completing one of the following tasks should resolve the issue: -+ -** When doing the upgrade of the Amazon EKS add-on, choose to override the existing settings as your conflict resolution option. If you've made other custom settings to the [.noloc]`Deployment`, make sure to back up your settings before upgrading so that you can reapply your other custom settings after the upgrade. -** Remove your existing `PodDisruptionBudget` and try the upgrade again. -* In EKS add-on versions `v1.9.3-eksbuild.3` and later and `v1.10.1-eksbuild.6` and later, the [.noloc]`CoreDNS` [.noloc]`Deployment` sets the `readinessProbe` to use the `/ready` endpoint. This endpoint is enabled in the `Corefile` configuration file for [.noloc]`CoreDNS`. -+ -If you use a custom `Corefile`, you must add the `ready` plugin to the config, so that the `/ready` endpoint is active in [.noloc]`CoreDNS` for the probe to use. -* In EKS add-on versions `v1.9.3-eksbuild.7` and later and `v1.10.1-eksbuild.4` and later, you can change the `PodDisruptionBudget`. You can edit the add-on and change these settings in the *Optional configuration settings* using the fields in the following example. This example shows the default `PodDisruptionBudget`. -+ -[source,json,subs="verbatim,attributes"] ----- -{ - "podDisruptionBudget": { - "enabled": true, - "maxUnavailable": 1 - } -} ----- -// Not using [.noloc]`Kubernetes` here because the _ causes issues with the rendering. -+ -You can set `maxUnavailable` or `minAvailable`, but you can't set both in a single `PodDisruptionBudget`. For more information about `PodDisruptionBudgets`, see https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget[Specifying a PodDisruptionBudget] in the _Kubernetes documentation_. -+ -Note that if you set `enabled` to `false`, the `PodDisruptionBudget` isn't removed. After you set this field to `false`, you must delete the `PodDisruptionBudget` object. Similarly, if you edit the add-on to use an older version of the add-on (downgrade the add-on) after upgrading to a version with a `PodDisruptionBudget`, the `PodDisruptionBudget` isn't removed. To delete the `PodDisruptionBudget`, you can run the following command: -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl delete poddisruptionbudget coredns -n kube-system ----- -* In EKS add-on versions `v1.10.1-eksbuild.5` and later, change the default toleration from `node-role.kubernetes.io/master:NoSchedule` to `node-role.kubernetes.io/control-plane:NoSchedule` to comply with KEP 2067. For more information about KEP 2067, see https://github.com/kubernetes/enhancements/tree/master/keps/sig-cluster-lifecycle/kubeadm/2067-rename-master-label-taint#renaming-the-node-rolekubernetesiomaster-node-taint[KEP-2067: Rename the kubeadm "master" label and taint] in the _Kubernetes Enhancement Proposals (KEPs)_ on [.noloc]`GitHub`. -+ -In EKS add-on versions `v1.8.7-eksbuild.8` and later and `v1.9.3-eksbuild.9` and later, both tolerations are set to be compatible with every [.noloc]`Kubernetes` version. -* In EKS add-on versions `v1.9.3-eksbuild.11` and `v1.10.1-eksbuild.7` and later, the [.noloc]`CoreDNS` [.noloc]`Deployment` sets a default value for `topologySpreadConstraints`. The default value ensures that the [.noloc]`CoreDNS` [.noloc]`Pods` are spread across the Availability Zones if there are nodes in multiple Availability Zones available. You can set a custom value that will be used instead of the default value. The default value follows: -+ -[source,yaml,subs="verbatim,attributes"] ----- -topologySpreadConstraints: - - maxSkew: 1 - topologyKey: topology.kubernetes.io/zone - whenUnsatisfiable: ScheduleAnyway - labelSelector: - matchLabels: - k8s-app: kube-dns ----- - - -[[coredns-upgrade-1.11,coredns-upgrade-1.11.title]] -==== [.noloc]`CoreDNS` `v1.11` upgrade considerations - -* In EKS add-on versions `v1.11.1-eksbuild.4` and later, the container image is based on a https://gallery.ecr.aws/eks-distro-build-tooling/eks-distro-minimal-base[minimal base image] maintained by Amazon EKS Distro, which contains minimal packages and doesn't have shells. For more information, see https://distro.eks.amazonaws.com/[Amazon EKS Distro]. The usage and troubleshooting of the [.noloc]`CoreDNS` image remains the same. - - -[.topic] -[[coredns-add-on-create,coredns-add-on-create.title]] -=== Create the [.noloc]`CoreDNS` Amazon EKS add-on - -Create the [.noloc]`CoreDNS` Amazon EKS add-on. You must have a cluster before you create the add-on. For more information, see <>. - -. See which version of the add-on is installed on your cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl describe deployment coredns --namespace kube-system | grep coredns: | cut -d : -f 3 ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -v1.10.1-eksbuild.13 ----- -. See which type of the add-on is installed on your cluster. Depending on the tool that you created your cluster with, you might not currently have the Amazon EKS add-on type installed on your cluster. Replace [.replaceable]`my-cluster` with the name of your cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks describe-addon --cluster-name my-cluster --addon-name coredns --query addon.addonVersion --output text ----- -+ -If a version number is returned, you have the Amazon EKS type of the add-on installed on your cluster and don't need to complete the remaining steps in this procedure. If an error is returned, you don't have the Amazon EKS type of the add-on installed on your cluster. Complete the remaining steps of this procedure to install it. -. Save the configuration of your currently installed add-on. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl get deployment coredns -n kube-system -o yaml > aws-k8s-coredns-old.yaml ----- -. Create the add-on using the {aws} CLI. If you want to use the {aws-management-console} or `eksctl` to create the add-on, see <> and specify `coredns` for the add-on name. Copy the command that follows to your device. Make the following modifications to the command, as needed, and then run the modified command. -+ -** Replace [.replaceable]`my-cluster` with the name of your cluster. -** Replace [.replaceable]`v1.11.3-eksbuild.1` with the latest version listed in the <> for your cluster version. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks create-addon --cluster-name my-cluster --addon-name coredns --addon-version v1.11.3-eksbuild.1 ----- -+ -If you've applied custom settings to your current add-on that conflict with the default settings of the Amazon EKS add-on, creation might fail. If creation fails, you receive an error that can help you resolve the issue. Alternatively, you can add `--resolve-conflicts OVERWRITE` to the previous command. This allows the add-on to overwrite any existing custom settings. Once you've created the add-on, you can update it with your custom settings. -. Confirm that the latest version of the add-on for your cluster's [.noloc]`Kubernetes` version was added to your cluster. Replace [.replaceable]`my-cluster` with the name of your cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks describe-addon --cluster-name my-cluster --addon-name coredns --query addon.addonVersion --output text ----- -+ -It might take several seconds for add-on creation to complete. -+ -An example output is as follows. -+ -[source,json,subs="verbatim,attributes"] ----- -v1.11.3-eksbuild.1 ----- -. If you made custom settings to your original add-on, before you created the Amazon EKS add-on, use the configuration that you saved in a previous step to update the Amazon EKS add-on with your custom settings. For instructions to update the add-on, see <>. - - -[.topic] -[[coredns-add-on-update,coredns-add-on-update.title]] -=== Update the [.noloc]`CoreDNS` Amazon EKS add-on - -Update the Amazon EKS type of the add-on. If you haven't added the Amazon EKS add-on to your cluster, either <> or see <>. - -Before you begin, review the upgrade considerations. For more information, see <>. - -. See which version of the add-on is installed on your cluster. Replace [.replaceable]`my-cluster` with your cluster name. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks describe-addon --cluster-name my-cluster --addon-name coredns --query "addon.addonVersion" --output text ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -v1.10.1-eksbuild.13 ----- -+ -If the version returned is the same as the version for your cluster's [.noloc]`Kubernetes` version in the <>, then you already have the latest version installed on your cluster and don't need to complete the rest of this procedure. If you receive an error, instead of a version number in your output, then you don't have the Amazon EKS type of the add-on installed on your cluster. You need to <> before you can update it with this procedure. -. Save the configuration of your currently installed add-on. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl get deployment coredns -n kube-system -o yaml > aws-k8s-coredns-old.yaml ----- -. Update your add-on using the {aws} CLI. If you want to use the {aws-management-console} or `eksctl` to update the add-on, see <>. Copy the command that follows to your device. Make the following modifications to the command, as needed, and then run the modified command. -+ -** Replace [.replaceable]`my-cluster` with the name of your cluster. -** Replace [.replaceable]`v1.11.3-eksbuild.1` with the latest version listed in the <> for your cluster version. -** The `--resolve-conflicts[.replaceable]``PRESERVE``` option preserves existing configuration values for the add-on. If you've set custom values for add-on settings, and you don't use this option, Amazon EKS overwrites your values with its default values. If you use this option, then we recommend testing any field and value changes on a non-production cluster before updating the add-on on your production cluster. If you change this value to `OVERWRITE`, all settings are changed to Amazon EKS default values. If you've set custom values for any settings, they might be overwritten with Amazon EKS default values. If you change this value to `none`, Amazon EKS doesn't change the value of any settings, but the update might fail. If the update fails, you receive an error message to help you resolve the conflict. -** If you're not updating a configuration setting, remove `--configuration-values '{[.replaceable]``"replicaCount":3``}'` from the command. If you're updating a configuration setting, replace [.replaceable]`"replicaCount":3` with the setting that you want to set. In this example, the number of replicas of [.noloc]`CoreDNS` is set to `3`. The value that you specify must be valid for the configuration schema. If you don't know the configuration schema, run `aws eks describe-addon-configuration --addon-name coredns --addon-version [.replaceable]``v1.11.3-eksbuild.1```, replacing [.replaceable]`v1.11.3-eksbuild.1` with the version number of the add-on that you want to see the configuration for. The schema is returned in the output. If you have any existing custom configuration, want to remove it all, and set the values for all settings back to Amazon EKS defaults, remove [.replaceable]`"replicaCount":3` from the command, so that you have empty `{}`. For more information about [.noloc]`CoreDNS` settings, see https://kubernetes.io/docs/tasks/administer-cluster/dns-custom-nameservers/[Customizing DNS Service] in the [.noloc]`Kubernetes` documentation. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks update-addon --cluster-name my-cluster --addon-name coredns --addon-version v1.11.3-eksbuild.1 \ - --resolve-conflicts PRESERVE --configuration-values '{"replicaCount":3}' ----- -+ -It might take several seconds for the update to complete. -. Confirm that the add-on version was updated. Replace [.replaceable]`my-cluster` with the name of your cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks describe-addon --cluster-name my-cluster --addon-name coredns ----- -+ -It might take several seconds for the update to complete. -+ -An example output is as follows. -+ -[source,json,subs="verbatim,attributes"] ----- -{ - "addon": { - "addonName": "coredns", - "clusterName": "my-cluster", - "status": "ACTIVE", - "addonVersion": "v1.11.3-eksbuild.1", - "health": { - "issues": [] - }, - "addonArn": "{arn-aws}eks:region:111122223333:addon/my-cluster/coredns/d2c34f06-1111-2222-1eb0-24f64ce37fa4", - "createdAt": "2023-03-01T16:41:32.442000+00:00", - "modifiedAt": "2023-03-01T18:16:54.332000+00:00", - "tags": {}, - "configurationValues": "{\"replicaCount\":3}" - } -} ----- - - -[.topic] -[[coredns-add-on-self-managed-update,coredns-add-on-self-managed-update.title]] -=== Update the [.noloc]`CoreDNS` Amazon EKS self-managed add-on - -[IMPORTANT] -==== - -We recommend adding the Amazon EKS type of the add-on to your cluster instead of using the self-managed type of the add-on. If you're not familiar with the difference between the types, see <>. For more information about adding an Amazon EKS add-on to your cluster, see <>. If you're unable to use the Amazon EKS add-on, we encourage you to submit an issue about why you can't to the https://github.com/aws/containers-roadmap/issues[Containers roadmap GitHub repository]. - -==== - -Before you begin, review the upgrade considerations. For more information, see <>. - -. Confirm that you have the self-managed type of the add-on installed on your cluster. Replace [.replaceable]`my-cluster` with the name of your cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks describe-addon --cluster-name my-cluster --addon-name coredns --query addon.addonVersion --output text ----- -+ -If an error message is returned, you have the self-managed type of the add-on installed on your cluster. Complete the remaining steps in this procedure. If a version number is returned, you have the Amazon EKS type of the add-on installed on your cluster. To update the Amazon EKS type of the add-on, use the procedure in <>, rather than using this procedure. If you're not familiar with the differences between the add-on types, see <>. -. See which version of the container image is currently installed on your cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl describe deployment coredns -n kube-system | grep Image | cut -d ":" -f 3 ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -v1.8.7-eksbuild.2 ----- -. If your current [.noloc]`CoreDNS` version is `v1.5.0` or later, but earlier than the version listed in the <> table, then skip this step. If your current version is earlier than `1.5.0`, then you need to modify the `ConfigMap` for [.noloc]`CoreDNS` to use the forward add-on, rather than the proxy add-on. -+ -.. Open the `ConfigMap` with the following command. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl edit configmap coredns -n kube-system ----- -.. Replace `proxy` in the following line with `forward`. Save the file and exit the editor. -+ -[source,bash,subs="verbatim,attributes"] ----- -proxy . /etc/resolv.conf ----- -. If you originally deployed your cluster on [.noloc]`Kubernetes` `1.17` or earlier, then you may need to remove a discontinued line from your [.noloc]`CoreDNS` manifest. -+ -IMPORTANT: You must complete this step before updating to [.noloc]`CoreDNS` version `1.7.0`, but it's recommended that you complete this step even if you're updating to an earlier version. -+ -.. Check to see if your [.noloc]`CoreDNS` manifest has the line. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl get configmap coredns -n kube-system -o jsonpath='{$.data.Corefile}' | grep upstream ----- -+ -If no output is returned, your manifest doesn't have the line and you can skip to the next step to update [.noloc]`CoreDNS`. If output is returned, then you need to remove the line. -.. Edit the `ConfigMap` with the following command, removing the line in the file that has the word `upstream` in it. Do not change anything else in the file. Once the line is removed, save the changes. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl edit configmap coredns -n kube-system -o yaml ----- -. Retrieve your current [.noloc]`CoreDNS` image version: -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl describe deployment coredns -n kube-system | grep Image ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -602401143452.dkr.ecr.region-code.amazonaws.com/eks/coredns:v1.8.7-eksbuild.2 ----- -. If you're updating to [.noloc]`CoreDNS` `1.8.3` or later, then you need to add the `endpointslices` permission to the `system:coredns` [.noloc]`Kubernetes` `clusterrole`. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl edit clusterrole system:coredns -n kube-system ----- -+ -Add the following lines under the existing permissions lines in the `rules` section of the file. -+ -[source,yaml,subs="verbatim,attributes"] ----- -[...] -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - list - - watch -[...] ----- -. Update the [.noloc]`CoreDNS` add-on by replacing [.replaceable]`602401143452` and [.replaceable]`region-code` with the values from the output returned in a previous step. Replace [.replaceable]`v1.11.3-eksbuild.1` with the [.noloc]`CoreDNS` version listed in the <> for your [.noloc]`Kubernetes` version. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl set image deployment.apps/coredns -n kube-system coredns=602401143452.dkr.ecr.region-code.amazonaws.com/eks/coredns:v1.11.3-eksbuild.1 ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -deployment.apps/coredns image updated ----- -. Check the container image version again to confirm that it was updated to the version that you specified in the previous step. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl describe deployment coredns -n kube-system | grep Image | cut -d ":" -f 3 ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -v1.11.3-eksbuild.1 ----- - - -[.topic] -[[coredns-autoscaling,coredns-autoscaling.title]] -=== Scale [.noloc]`CoreDNS` [.noloc]`Pods` for high DNS traffic - -[abstract] --- -Learn how the Amazon EKS add-on for [.noloc]`CoreDNS` autoscales to handle increased load on DNS pods, improving application availability and cluster scalability. --- - -When you launch an Amazon EKS cluster with at least one node, a [.noloc]`Deployment` of two replicas of the [.noloc]`CoreDNS` image are deployed by default, regardless of the number of nodes deployed in your cluster. The [.noloc]`CoreDNS` Pods provide name resolution for all Pods in the cluster. Applications use name resolution to connect to pods and services in the cluster as well as connecting to services outside the cluster. As the number of requests for name resolution (queries) from pods increase, the [.noloc]`CoreDNS` pods can get overwhelmed and slow down, and reject requests that the pods can`'t handle. - -To handle the increased load on the [.noloc]`CoreDNS` pods, consider an autoscaling system for [.noloc]`CoreDNS`. Amazon EKS can manage the autoscaling of the [.noloc]`CoreDNS` Deployment in the EKS Add-on version of [.noloc]`CoreDNS`. This [.noloc]`CoreDNS` autoscaler continuously monitors the cluster state, including the number of nodes and CPU cores. Based on that information, the controller will dynamically adapt the number of replicas of the [.noloc]`CoreDNS` deployment in an EKS cluster. This feature works for [.noloc]`CoreDNS` `v1.9` and EKS release version `1.25` and later. For more information about which versions are compatible with [.noloc]`CoreDNS` Autoscaling, see the following section. - -We recommend using this feature in conjunction with other https://aws.github.io/aws-eks-best-practices/cluster-autoscaling/[EKS Cluster Autoscaling best practices] to improve overall application availability and cluster scalability. - -[[coredns-autoscaling-prereqs,coredns-autoscaling-prereqs.title]] -==== Prerequisites - -For Amazon EKS to scale your [.noloc]`CoreDNS` deployment, there are three prerequisites: - - - -* You must be using the _EKS Add-on_ version of [.noloc]`CoreDNS`. -* Your cluster must be running at least the minimum cluster versions and platform versions. -* Your cluster must be running at least the minimum version of the EKS Add-on of [.noloc]`CoreDNS`. - - -[[coredns-autoscaling-cluster-version,coredns-autoscaling-cluster-version.title]] -===== Minimum cluster version - -Autoscaling of [.noloc]`CoreDNS` is done by a new component in the cluster control plane, managed by Amazon EKS. Because of this, you must upgrade your cluster to an EKS release that supports the minimum platform version that has the new component. - -A new Amazon EKS cluster. To deploy one, see <>. The cluster must be [.noloc]`Kubernetes` version `1.25` or later. The cluster must be running one of the [.noloc]`Kubernetes` versions and platform versions listed in the following table or a later version. Note that any [.noloc]`Kubernetes` and platform versions later than those listed are also supported. You can check your current [.noloc]`Kubernetes` version by replacing [.replaceable]`my-cluster` in the following command with the name of your cluster and then running the modified command: - -[source,bash,subs="verbatim,attributes"] ----- -aws eks describe-cluster - --name my-cluster --query cluster.version --output - text ----- - -[cols="1,1", options="header"] -|=== -|Kubernetes version -|Platform version - - -|`1.29.3` -|`eks.7` - -|`1.28.8` -|`eks.13` - -|`1.27.12` -|`eks.17` - -|`1.26.15` -|`eks.18` - -|`1.25.16` -|`eks.19` -|=== - -[NOTE] -==== - -Every platform version of later [.noloc]`Kubernetes` versions are also supported, for example [.noloc]`Kubernetes` version `1.30` from `eks.1` and on. - -==== - -[[coredns-autoscaling-coredns-version,coredns-autoscaling-coredns-version.title]] -===== Minimum EKS Add-on version - -[cols="1,1,1,1,1,1", options="header"] -|=== -|Kubernetes version -|1.29 -|1.28 -|1.27 -|1.26 -|1.25 - - -| -|`v1.11.1-eksbuild.9` -|`v1.10.1-eksbuild.11` -|`v1.10.1-eksbuild.11` -|`v1.9.3-eksbuild.15` -|`v1.9.3-eksbuild.15` -|=== - - -[[coredns-autoscaling-console,coredns-autoscaling-console.title]] -.Configuring [.noloc]`CoreDNS` autoscaling in the {aws-management-console} -[%collapsible] -==== -. Ensure that your cluster is at or above the minimum cluster version. -+ -Amazon EKS upgrades clusters between platform versions of the same [.noloc]`Kubernetes` version automatically, and you can`'t start this process yourself. Instead, you can upgrade your cluster to the next [.noloc]`Kubernetes` version, and the cluster will be upgraded to that K8s version and the latest platform version. For example, if you upgrade from `1.25` to `1.26`, the cluster will upgrade to `1.26.15 eks.18`. -+ -New [.noloc]`Kubernetes` versions sometimes introduce significant changes. Therefore, we recommend that you test the behavior of your applications by using a separate cluster of the new [.noloc]`Kubernetes` version before you update your production clusters. -+ -To upgrade a cluster to a new [.noloc]`Kubernetes` version, follow the procedure in <>. -. Ensure that you have the EKS Add-on for [.noloc]`CoreDNS`, not the self-managed [.noloc]`CoreDNS` Deployment. -+ -Depending on the tool that you created your cluster with, you might not currently have the Amazon EKS add-on type installed on your cluster. To see which type of the add-on is installed on your cluster, you can run the following command. Replace `my-cluster` with the name of your cluster. -+ -[source,shell,subs="verbatim,attributes"] ----- -aws eks describe-addon --cluster-name my-cluster --addon-name coredns --query addon.addonVersion --output text ----- -+ -If a version number is returned, you have the Amazon EKS type of the add-on installed on your cluster and you can continue with the next step. If an error is returned, you don't have the Amazon EKS type of the add-on installed on your cluster. Complete the remaining steps of the procedure <> to replace the self-managed version with the Amazon EKS add-on. -. Ensure that your EKS Add-on for [.noloc]`CoreDNS` is at a version the same or higher than the minimum EKS Add-on version. -+ -See which version of the add-on is installed on your cluster. You can check in the {aws-management-console} or run the following command: -+ -[source,shell,subs="verbatim,attributes"] ----- -kubectl describe deployment coredns --namespace kube-system | grep coredns: | cut -d : -f 3 ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -v1.10.1-eksbuild.13 ----- -+ -Compare this version with the minimum EKS Add-on version in the previous section. If needed, upgrade the EKS Add-on to a higher version by following the procedure <>. -. Add the autoscaling configuration to the *Optional configuration settings* of the EKS Add-on. -+ -.. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. -.. In the left navigation pane, select *Clusters*, and then select the name of the cluster that you want to configure the add-on for. -.. Choose the *Add-ons* tab. -.. Select the box in the top right of the [.noloc]`CoreDNS` add-on box and then choose *Edit*. -.. On the *Configure [.noloc]`CoreDNS`* page: -+ -... Select the *Version* that you'd like to use. We recommend that you keep the same version as the previous step, and update the version and configuration in separate actions. -... Expand the *Optional configuration settings*. -... Enter the JSON key `"autoscaling":` and value of a nested JSON object with a key `"enabled":` and value `true` in *Configuration values*. The resulting text must be a valid JSON object. If this key and value are the only data in the text box, surround the key and value with curly braces `{ }`. The following example shows autoscaling is enabled: -+ -[source,json,subs="verbatim,attributes"] ----- -{ - "autoScaling": { - "enabled": true - } -} ----- -... (Optional) You can provide minimum and maximum values that autoscaling can scale the number of [.noloc]`CoreDNS` pods to. -+ -The following example shows autoscaling is enabled and all of the optional keys have values. We recommend that the minimum number of [.noloc]`CoreDNS` pods is always greater than 2 to provide resilience for the DNS service in the cluster. -+ -[source,json,subs="verbatim,attributes"] ----- -{ - "autoScaling": { - "enabled": true, - "minReplicas": 2, - "maxReplicas": 10 - } -} ----- -.. To apply the new configuration by replacing the [.noloc]`CoreDNS` pods, choose *Save changes*. -+ -Amazon EKS applies changes to the EKS Add-ons by using a _rollout_ of the [.noloc]`Kubernetes` Deployment for CoreDNS. You can track the status of the rollout in the *Update history* of the add-on in the {aws-management-console} and with `kubectl rollout status deployment/coredns --namespace kube-system`. -+ -`kubectl rollout` has the following commands: -+ -[source,shell,subs="verbatim,attributes"] ----- -kubectl rollout - -history -- View rollout history -pause -- Mark the provided resource as paused -restart -- Restart a resource -resume -- Resume a paused resource -status -- Show the status of the rollout -undo -- Undo a previous rollout ----- -+ -If the rollout takes too long, Amazon EKS will undo the rollout, and a message with the type of *Addon Update* and a status of *Failed* will be added to the *Update history* of the add-on. To investigate any issues, start from the history of the rollout, and run `kubectl logs` on a [.noloc]`CoreDNS` pod to see the logs of [.noloc]`CoreDNS`. -. If the new entry in the *Update history* has a status of *Successful*, then the rollout has completed and the add-on is using the new configuration in all of the [.noloc]`CoreDNS` pods. As you change the number of nodes and CPU cores of nodes in the cluster, Amazon EKS scales the number of replicas of the [.noloc]`CoreDNS` deployment. - -==== - -[[coredns-autoscaling-cli,coredns-autoscaling-cli.title]] -.Configuring [.noloc]`CoreDNS` autoscaling in the {aws} Command Line Interface -[%collapsible] -==== -. Ensure that your cluster is at or above the minimum cluster version. -+ -Amazon EKS upgrades clusters between platform versions of the same [.noloc]`Kubernetes` version automatically, and you can`'t start this process yourself. Instead, you can upgrade your cluster to the next [.noloc]`Kubernetes` version, and the cluster will be upgraded to that K8s version and the latest platform version. For example, if you upgrade from `1.25` to `1.26`, the cluster will upgrade to `1.26.15 eks.18`. -+ -New [.noloc]`Kubernetes` versions sometimes introduce significant changes. Therefore, we recommend that you test the behavior of your applications by using a separate cluster of the new [.noloc]`Kubernetes` version before you update your production clusters. -+ -To upgrade a cluster to a new [.noloc]`Kubernetes` version, follow the procedure in <>. -. Ensure that you have the EKS Add-on for [.noloc]`CoreDNS`, not the self-managed [.noloc]`CoreDNS` Deployment. -+ -Depending on the tool that you created your cluster with, you might not currently have the Amazon EKS add-on type installed on your cluster. To see which type of the add-on is installed on your cluster, you can run the following command. Replace `my-cluster` with the name of your cluster. -+ -[source,shell,subs="verbatim,attributes"] ----- -aws eks describe-addon --cluster-name my-cluster --addon-name coredns --query addon.addonVersion --output text ----- -+ -If a version number is returned, you have the Amazon EKS type of the add-on installed on your cluster. If an error is returned, you don't have the Amazon EKS type of the add-on installed on your cluster. Complete the remaining steps of the procedure <> to replace the self-managed version with the Amazon EKS add-on. -. Ensure that your EKS Add-on for [.noloc]`CoreDNS` is at a version the same or higher than the minimum EKS Add-on version. -+ -See which version of the add-on is installed on your cluster. You can check in the {aws-management-console} or run the following command: -+ -[source,shell,subs="verbatim,attributes"] ----- -kubectl describe deployment coredns --namespace kube-system | grep coredns: | cut -d : -f 3 ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -v1.10.1-eksbuild.13 ----- -+ -Compare this version with the minimum EKS Add-on version in the previous section. If needed, upgrade the EKS Add-on to a higher version by following the procedure <>. -. Add the autoscaling configuration to the *Optional configuration settings* of the EKS Add-on. -+ -Run the following {aws} CLI command. Replace `my-cluster` with the name of your cluster and the IAM role ARN with the role that you are using. -+ -[source,shell,subs="verbatim,attributes"] ----- -aws eks update-addon --cluster-name my-cluster --addon-name coredns \ - --resolve-conflicts PRESERVE --configuration-values '{"autoScaling":{"enabled":true}}' ----- -+ -Amazon EKS applies changes to the EKS Add-ons by using a _rollout_ of the [.noloc]`Kubernetes` Deployment for CoreDNS. You can track the status of the rollout in the *Update history* of the add-on in the {aws-management-console} and with `kubectl rollout status deployment/coredns --namespace kube-system`. -+ -`kubectl rollout` has the following commands: -+ -[source,shell,subs="verbatim,attributes"] ----- -kubectl rollout - -history -- View rollout history -pause -- Mark the provided resource as paused -restart -- Restart a resource -resume -- Resume a paused resource -status -- Show the status of the rollout -undo -- Undo a previous rollout ----- -+ -If the rollout takes too long, Amazon EKS will undo the rollout, and a message with the type of *Addon Update* and a status of *Failed* will be added to the *Update history* of the add-on. To investigate any issues, start from the history of the rollout, and run `kubectl logs` on a [.noloc]`CoreDNS` pod to see the logs of [.noloc]`CoreDNS`. -. (Optional) You can provide minimum and maximum values that autoscaling can scale the number of [.noloc]`CoreDNS` pods to. -+ -The following example shows autoscaling is enabled and all of the optional keys have values. We recommend that the minimum number of [.noloc]`CoreDNS` pods is always greater than 2 to provide resilience for the DNS service in the cluster. -+ -[source,shell,subs="verbatim,attributes"] ----- -aws eks update-addon --cluster-name my-cluster --addon-name coredns \ - --resolve-conflicts PRESERVE --configuration-values '{"autoScaling":{"enabled":true,"minReplicas":2,"maxReplicas":10}}' ----- -. Check the status of the update to the add-on by running the following command: -+ -[source,shell,subs="verbatim,attributes"] ----- -aws eks describe-addon --cluster-name my-cluster --addon-name coredns \ ----- -+ -If you see this line: `"status": "ACTIVE"`, then the rollout has completed and the add-on is using the new configuration in all of the [.noloc]`CoreDNS` pods. As you change the number of nodes and CPU cores of nodes in the cluster, Amazon EKS scales the number of replicas of the [.noloc]`CoreDNS` deployment. - -==== - -[.topic] -[[coredns-metrics,coredns-metrics.title]] -=== Monitor [.noloc]`Kubernetes` DNS resolution with [.noloc]`CoreDNS` metrics - -[abstract] --- -Learn how to collect [.noloc]`CoreDNS` metrics in Amazon EKS using Prometheus or CloudWatch Agent, enabling monitoring and observability for your [.noloc]`Kubernetes` DNS resolution. --- - -[.noloc]`CoreDNS` as an EKS add-on exposes the metrics from [.noloc]`CoreDNS` on port `9153` in the Prometheus format in the `kube-dns` service. You can use Prometheus, the Amazon CloudWatch agent, or any other compatible system to scrape (collect) these metrics. - -For an example _scrape configuration_ that is compatible with both Prometheus and the CloudWatch agent, see link:AmazonCloudWatch/latest/monitoring/ContainerInsights-Prometheus-Setup-configure.html[CloudWatch agent configuration for Prometheus,type="documentation"] in the _Amazon CloudWatch User Guide_. - -[.topic] -[[managing-kube-proxy,managing-kube-proxy.title]] -== Manage `kube-proxy` in Amazon EKS clusters - -[abstract] --- -Learn how to manage the `kube-proxy` add-on on your Amazon EKS cluster to manage network rules and enable network communication to your Pods. --- - -[TIP] -==== -With Amazon EKS Auto Mode, you don't need to install or upgrade networking add-ons. Auto Mode includes pod networking and load balancing capabilities. - -For more information, see <>. -==== - -//GDC: Need DF to review - - -We recommend adding the Amazon EKS type of the add-on to your cluster instead of using the self-managed type of the add-on. If you're not familiar with the difference between the types, see <>. For more information about adding an Amazon EKS add-on to your cluster, see <>. If you're unable to use the Amazon EKS add-on, we encourage you to submit an issue about why you can't to the https://github.com/aws/containers-roadmap/issues[Containers roadmap GitHub repository]. - -The `kube-proxy` add-on is deployed on each Amazon EC2 node in your Amazon EKS cluster. It maintains network rules on your nodes and enables network communication to your [.noloc]`Pods`. The add-on isn't deployed to Fargate nodes in your cluster. For more information, see https://kubernetes.io/docs/reference/command-line-tools-reference/kube-proxy/[kube-proxy] in the [.noloc]`Kubernetes` documentation. - -=== Install as Amazon EKS Add-on - - -[[kube-proxy-versions,kube-proxy-versions.title]] -=== `kube-proxy` versions - -The following table lists the latest version of the Amazon EKS add-on type for each [.noloc]`Kubernetes` version. - -[options="header"] -|=== -| Kubernetes version | `kube-proxy` version -| 1.31 | v1.31.3-eksbuild.2 -| 1.30 | v1.30.7-eksbuild.2 -| 1.29 | v1.29.11-eksbuild.2 -| 1.28 | v1.28.15-eksbuild.4 -| 1.27 | v1.27.16-eksbuild.14 -| 1.26 | v1.26.15-eksbuild.19 -| 1.25 | v1.25.16-eksbuild.22 -| 1.24 | v1.24.17-eksbuild.19 -| 1.23 | v1.23.17-eksbuild.20 -|=== - -[NOTE] -==== - -An earlier version of the documentation was incorrect. `kube-proxy` versions `v1.28.5`, `v1.27.9`, and `v1.26.12` aren't available. - -If you're self-managing this add-on, the versions in the table might not be the same as the available self-managed versions. - -==== - -[[managing-kube-proxy-images,managing-kube-proxy-images.title]] -=== `kube-proxy` container image migration - -There are two types of the `kube-proxy` container image available for each Amazon EKS cluster version: - - - -* *Default* – This image type is based on a Debian-based Docker image that is maintained by the [.noloc]`Kubernetes` upstream community. -* *Minimal* – This image type is based on a https://gallery.ecr.aws/eks-distro-build-tooling/eks-distro-minimal-base-iptables[minimal base image] maintained by Amazon EKS Distro, which contains minimal packages and doesn't have shells. For more information, see https://distro.eks.amazonaws.com/[Amazon EKS Distro]. - -The following table lists the latest available self-managed `kube-proxy` container image version for each Amazon EKS cluster version. - -// GDC Update - -[options="header"] -|=== -| Version | kube-proxy (default type) | kube-proxy (minimal type) -| 1.31 | Only minimal type is available | v1.31.2-minimal-eksbuild.3 -| 1.30 | Only minimal type is available | v1.30.6-minimal-eksbuild.3 -| 1.29 | Only minimal type is available | v1.29.10-minimal-eksbuild.3 -| 1.28 | Only minimal type is available | v1.28.15-minimal-eksbuild.4 -| 1.27 | Only minimal type is available | v1.27.16-minimal-eksbuild.14 -| 1.26 | Only minimal type is available | v1.26.15-minimal-eksbuild.19 -| 1.25 | Only minimal type is available | v1.25.16-minimal-eksbuild.22 -| 1.24 | v1.24.10-eksbuild.2 | v1.24.17-minimal-eksbuild.19 -| 1.23 | v1.23.16-eksbuild.2 | v1.23.17-minimal-eksbuild.20 -|=== - - -* The default image type isn't available for [.noloc]`Kubernetes` version `1.25` and later. You must use the minimal image type. -* When you <>, you specify a valid Amazon EKS add-on version, which might not be a version listed in this table. This is because <> versions don't always match container image versions specified when updating the self-managed type of this add-on. When you update the self-managed type of this add-on, you specify a valid container image version listed in this table. - - -[.topic] -[[kube-proxy-add-on-self-managed-update,kube-proxy-add-on-self-managed-update.title]] -=== Update the Kubernetes `kube-proxy` self-managed add-on - -[IMPORTANT] -==== - -We recommend adding the Amazon EKS type of the add-on to your cluster instead of using the self-managed type of the add-on. If you're not familiar with the difference between the types, see <>. For more information about adding an Amazon EKS add-on to your cluster, see <>. If you're unable to use the Amazon EKS add-on, we encourage you to submit an issue about why you can't to the https://github.com/aws/containers-roadmap/issues[Containers roadmap GitHub repository]. - -==== +include::alternate-cni-plugins.adoc[leveloffset=+1] -[[managing-kube-proxy-prereqs,managing-kube-proxy-prereqs.title]] -==== Prerequisites +include::pod-multus.adoc[leveloffset=+1] -* An existing Amazon EKS cluster. To deploy one, see <>. +include::aws-load-balancer-controller.adoc[leveloffset=+1] +include::managing-coredns.adoc[leveloffset=+1] -[[managing-kube-proxy-considerations,managing-kube-proxy-considerations.title]] -==== Considerations +include::managing-kube-proxy.adoc[leveloffset=+1] -* `Kube-proxy` on an Amazon EKS cluster has the same https://kubernetes.io/releases/version-skew-policy/#kube-proxy[compatibility and skew policy as Kubernetes]. Learn how to <>. -. Confirm that you have the self-managed type of the add-on installed on your cluster. Replace [.replaceable]`my-cluster` with the name of your cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks describe-addon --cluster-name my-cluster --addon-name kube-proxy --query addon.addonVersion --output text ----- -+ -If an error message is returned, you have the self-managed type of the add-on installed on your cluster. The remaining steps in this topic are for updating the self-managed type of the add-on. If a version number is returned, you have the Amazon EKS type of the add-on installed on your cluster. To update it, use the procedure in <>, rather than using the procedure in this topic. If you're not familiar with the differences between the add-on types, see <>. -. See which version of the container image is currently installed on your cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl describe daemonset kube-proxy -n kube-system | grep Image ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -Image: 602401143452.dkr.ecr.region-code.amazonaws.com/eks/kube-proxy:v1.29.1-eksbuild.2 ----- -+ -In the example output, [.replaceable]`v1.29.1-eksbuild.2` is the version installed on the cluster. -. Update the `kube-proxy` add-on by replacing [.replaceable]`602401143452` and [.replaceable]`region-code` with the values from your output in the previous step. Replace [.replaceable]`v1.30.6-eksbuild.3` with the `kube-proxy` version listed in the <> table. -+ -IMPORTANT: The manifests for each image type are different and not compatible between the _default_ or _minimal_ image types. You must use the same image type as the previous image, so that the entrypoint and arguments match. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl set image daemonset.apps/kube-proxy -n kube-system kube-proxy=602401143452.dkr.ecr.region-code.amazonaws.com/eks/kube-proxy:v1.30.6-eksbuild.3 ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -daemonset.apps/kube-proxy image updated ----- -. Confirm that the new version is now installed on your cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl describe daemonset kube-proxy -n kube-system | grep Image | cut -d ":" -f 3 ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -v1.30.0-eksbuild.3 ----- -. If you're using `x86` and `Arm` nodes in the same cluster and your cluster was deployed before August 17, 2020. Then, edit your `kube-proxy` manifest to include a node selector for multiple hardware architectures with the following command. This is a one-time operation. After you've added the selector to your manifest, you don't need to add it each time you update the add-on. If your cluster was deployed on or after August 17, 2020, then `kube-proxy` is already multi-architecture capable. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl edit -n kube-system daemonset/kube-proxy ----- -+ -Add the following node selector to the file in the editor and then save the file. For an example of where to include this text in the editor, see the https://github.com/aws/amazon-vpc-cni-k8s/blob/release-1.11/config/master/aws-k8s-cni.yaml#L265-#L269[CNI manifest] file on [.noloc]`GitHub`. This enables [.noloc]`Kubernetes` to pull the correct hardware image based on the node's hardware architecture. -+ -[source,yaml,subs="verbatim,attributes"] ----- -- key: "kubernetes.io/arch" - operator: In - values: - - amd64 - - arm64 ----- -. If your cluster was originally created with [.noloc]`Kubernetes` version `1.14` or later, then you can skip this step because `kube-proxy` already includes this `Affinity Rule`. If you originally created an Amazon EKS cluster with [.noloc]`Kubernetes` version `1.13` or earlier and intend to use Fargate nodes in your cluster, then edit your `kube-proxy` manifest to include a `NodeAffinity` rule to prevent `kube-proxy` [.noloc]`Pods` from scheduling on Fargate nodes. This is a one-time edit. Once you've added the `Affinity Rule` to your manifest, you don't need to add it each time that you update the add-on. Edit your `kube-proxy` [.noloc]`DaemonSet`. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl edit -n kube-system daemonset/kube-proxy ----- -+ -Add the following `Affinity Rule` to the [.noloc]`DaemonSet`spec`` section of the file in the editor and then save the file. For an example of where to include this text in the editor, see the https://github.com/aws/amazon-vpc-cni-k8s/blob/release-1.11/config/master/aws-k8s-cni.yaml#L270-#L273[CNI manifest] file on [.noloc]`GitHub`. -+ -[source,yaml,subs="verbatim,attributes"] ----- -- key: eks.amazonaws.com/compute-type - operator: NotIn - values: - - fargate ----- diff --git a/latest/ug/networking/eks-networking.adoc b/latest/ug/networking/eks-networking.adoc index 75fcb437d..7055667d7 100644 --- a/latest/ug/networking/eks-networking.adoc +++ b/latest/ug/networking/eks-networking.adoc @@ -1,22 +1,8 @@ -//!!NODE_ROOT -[[eks-networking,eks-networking.title]] +include::../attributes.txt[] + +[#eks-networking] = Configure networking for Amazon EKS clusters -:doctype: book -:sectnums: -:toc: left -:icons: font -:experimental: -:idprefix: -:idseparator: - -:sourcedir: . -:info_doctype: chapter -:info_title: Configure networking for Amazon EKS clusters :info_titleabbrev: Configure networking -:info_abstract: Learn how to configure networking for your Amazon EKS cluster using a VPC, subnets, \ - security groups, and networking add-ons to ensure secure and efficient \ - communication. - -include::../attributes.txt[] [abstract] -- @@ -28,13 +14,23 @@ Your Amazon EKS cluster is created in a VPC. Pod networking is provided by the A [.topiclist] [[Topic List]] -include::network-reqs.adoc[leveloffset=+1] +[#add-existing-subnet] +== Add an existing VPC Subnet to an Amazon EKS cluster from the management console +. Navigate to your cluster in the management console +. From the *Networking* tab select *Manage VPC Resources* +. From the *Subnets* dropdown, select additional subnets from the VPC of your cluster. -include::creating-a-vpc.adoc[leveloffset=+1] +To create a new VPC Subnet: +* xref:network-requirements-subnets[Review EKS Subnet Requirements] +* See link:vpc/latest/userguide/create-subnets.html["Create a Subnet",type="documentation"] in the Amazon Virtual Private Cloud User Guide. -include::sec-group-reqs.adoc[leveloffset=+1] +include::network-reqs.adoc[leveloffset=+1] + +include::creating-a-vpc.adoc[leveloffset=+1] +include::sec-group-reqs.adoc[leveloffset=+1] include::eks-networking-add-ons.adoc[leveloffset=+1] + diff --git a/latest/ug/networking/external-snat.adoc b/latest/ug/networking/external-snat.adoc new file mode 100644 index 000000000..4edf836fb --- /dev/null +++ b/latest/ug/networking/external-snat.adoc @@ -0,0 +1,52 @@ +include::../attributes.txt[] + +[.topic] +[#external-snat] += Enable outbound internet access for Pods +:info_titleabbrev: Outbound traffic + +[abstract] +-- +Learn how Amazon EKS manages external communication for Pods using Source Network Address Translation (SNAT), allowing Pods to access internet resources or networks connected via VPC peering, Transit Gateway, or {aws} Direct Connect. +-- + +*Applies to*: Linux `IPv4` Fargate nodes, Linux nodes with Amazon EC2 instances + +If you deployed your cluster using the `IPv6` family, then the information in this topic isn't applicable to your cluster, because `IPv6` addresses are not network translated. For more information about using `IPv6` with your cluster, see <>. + +By default, each Pod in your cluster is assigned a link:AWSEC2/latest/UserGuide/using-instance-addressing.html#concepts-private-addresses[private,type="documentation"] `IPv4` address from a classless inter-domain routing (CIDR) block that is associated with the VPC that the Pod is deployed in. Pods in the same VPC communicate with each other using these private IP addresses as end points. When a Pod communicates to any `IPv4` address that isn't within a CIDR block that's associated to your VPC, the Amazon VPC CNI plugin (for both https://github.com/aws/amazon-vpc-cni-k8s#amazon-vpc-cni-k8s[Linux] or https://github.com/aws/amazon-vpc-cni-plugins/tree/master/plugins/vpc-bridge[Windows]) translates the Pod's `IPv4` address to the primary private `IPv4` address of the primary link:AWSEC2/latest/UserGuide/using-eni.html#eni-basics[elastic network interface,type="documentation"] of the node that the Pod is running on, by default <>. + +[NOTE] +==== + +For Windows nodes, there are additional details to consider. By default, the https://github.com/aws/amazon-vpc-cni-plugins/tree/master/plugins/vpc-bridge[VPC CNI plugin for Windows] is defined with a networking configuration in which the traffic to a destination within the same VPC is excluded for SNAT. This means that internal VPC communication has SNAT disabled and the IP address allocated to a Pod is routable inside the VPC. But traffic to a destination outside of the VPC has the source Pod IP SNAT'ed to the instance ENI's primary IP address. This default configuration for Windows ensures that the pod can access networks outside of your VPC in the same way as the host instance. + +==== + +Due to this behavior: + +* Your Pods can communicate with internet resources only if the node that they're running on has a link:AWSEC2/latest/UserGuide/using-instance-addressing.html#concepts-public-addresses[public,type="documentation"] or link:vpc/latest/userguide/vpc-eips.html[elastic,type="documentation"] IP address assigned to it and is in a link:vpc/latest/userguide/configure-subnets.html#subnet-basics[public subnet,type="documentation"]. A public subnet's associated link:vpc/latest/userguide/VPC_Route_Tables.html[route table,type="documentation"] has a route to an internet gateway. We recommend deploying nodes to private subnets, whenever possible. +* For versions of the plugin earlier than `1.8.0`, resources that are in networks or VPCs that are connected to your cluster VPC using link:vpc/latest/peering/what-is-vpc-peering.html[VPC peering,type="documentation"], a link:whitepapers/latest/aws-vpc-connectivity-options/transit-vpc-option.html[transit VPC,type="documentation"], or link:directconnect/latest/UserGuide/Welcome.html[{aws} Direct Connect,type="documentation"] can't initiate communication to your Pods behind secondary elastic network interfaces. Your Pods can initiate communication to those resources and receive responses from them, though. + +If either of the following statements are true in your environment, then change the default configuration with the command that follows. + +* You have resources in networks or VPCs that are connected to your cluster VPC using link:vpc/latest/peering/what-is-vpc-peering.html[VPC peering,type="documentation"], a link:whitepapers/latest/aws-vpc-connectivity-options/transit-vpc-option.html[transit VPC,type="documentation"], or link:directconnect/latest/UserGuide/Welcome.html[{aws} Direct Connect,type="documentation"] that need to initiate communication with your Pods using an `IPv4` address and your plugin version is earlier than `1.8.0`. +* Your Pods are in a link:vpc/latest/userguide/configure-subnets.html#subnet-basics[private subnet,type="documentation"] and need to communicate outbound to the internet. The subnet has a route to a link:vpc/latest/userguide/vpc-nat-gateway.html[NAT gateway,type="documentation"]. + + +[source,bash,subs="verbatim,attributes"] +---- +kubectl set env daemonset -n kube-system aws-node AWS_VPC_K8S_CNI_EXTERNALSNAT=true +---- + +[NOTE] +==== + +The `AWS_VPC_K8S_CNI_EXTERNALSNAT` and `AWS_VPC_K8S_CNI_EXCLUDE_SNAT_CIDRS` CNI configuration variables aren't applicable to Windows nodes. Disabling SNAT isn't supported for Windows. As for excluding a list of `IPv4` CIDRs from SNAT, you can define this by specifying the `ExcludedSnatCIDRs` parameter in the Windows bootstrap script. For more information on using this parameter, see <>. + +==== + +[#snat-exception] +== Host networking + +{asterisk} If a Pod's spec contains `hostNetwork=true` (default is `false`), then its IP address isn't translated to a different address. This is the case for the `kube-proxy` and Amazon VPC CNI plugin for Kubernetes Pods that run on your cluster, by default. For these Pods, the IP address is the same as the node's primary IP address, so the Pod's IP address isn't translated. For more information about a Pod's `hostNetwork` setting, see https://kubernetes.io/docs/reference/generated/kubernetes-api/v{k8s-n}/#podspec-v1-core[PodSpec v1 core] in the Kubernetes API reference. \ No newline at end of file diff --git a/latest/ug/networking/kube-proxy-add-on-self-managed-update.adoc b/latest/ug/networking/kube-proxy-add-on-self-managed-update.adoc new file mode 100644 index 000000000..cbf3e9cf4 --- /dev/null +++ b/latest/ug/networking/kube-proxy-add-on-self-managed-update.adoc @@ -0,0 +1,108 @@ +include::../attributes.txt[] + +[.topic] +[#kube-proxy-add-on-self-managed-update] += Update the Kubernetes `kube-proxy` self-managed add-on +:info_titleabbrev: Update + +[IMPORTANT] +==== + +We recommend adding the Amazon EKS type of the add-on to your cluster instead of using the self-managed type of the add-on. If you're not familiar with the difference between the types, see <>. For more information about adding an Amazon EKS add-on to your cluster, see <>. If you're unable to use the Amazon EKS add-on, we encourage you to submit an issue about why you can't to the https://github.com/aws/containers-roadmap/issues[Containers roadmap GitHub repository]. + +==== + +[#managing-kube-proxy-prereqs] +== Prerequisites + +* An existing Amazon EKS cluster. To deploy one, see <>. + + +[#managing-kube-proxy-considerations] +== Considerations + +* `Kube-proxy` on an Amazon EKS cluster has the same https://kubernetes.io/releases/version-skew-policy/#kube-proxy[compatibility and skew policy as Kubernetes]. Learn how to <>. +. Confirm that you have the self-managed type of the add-on installed on your cluster. Replace [.replaceable]`my-cluster` with the name of your cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks describe-addon --cluster-name my-cluster --addon-name kube-proxy --query addon.addonVersion --output text +---- ++ +If an error message is returned, you have the self-managed type of the add-on installed on your cluster. The remaining steps in this topic are for updating the self-managed type of the add-on. If a version number is returned, you have the Amazon EKS type of the add-on installed on your cluster. To update it, use the procedure in <>, rather than using the procedure in this topic. If you're not familiar with the differences between the add-on types, see <>. +. See which version of the container image is currently installed on your cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl describe daemonset kube-proxy -n kube-system | grep Image +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +Image: 602401143452.dkr.ecr.region-code.amazonaws.com/eks/kube-proxy:v1.29.1-eksbuild.2 +---- ++ +In the example output, [.replaceable]`v1.29.1-eksbuild.2` is the version installed on the cluster. +. Update the `kube-proxy` add-on by replacing [.replaceable]`602401143452` and [.replaceable]`region-code` with the values from your output in the previous step. Replace [.replaceable]`v1.30.6-eksbuild.3` with the `kube-proxy` version listed in the <> table. ++ +IMPORTANT: The manifests for each image type are different and not compatible between the _default_ or _minimal_ image types. You must use the same image type as the previous image, so that the entrypoint and arguments match. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl set image daemonset.apps/kube-proxy -n kube-system kube-proxy=602401143452.dkr.ecr.region-code.amazonaws.com/eks/kube-proxy:v1.30.6-eksbuild.3 +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +daemonset.apps/kube-proxy image updated +---- +. Confirm that the new version is now installed on your cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl describe daemonset kube-proxy -n kube-system | grep Image | cut -d ":" -f 3 +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +v1.30.0-eksbuild.3 +---- +. If you're using `x86` and `Arm` nodes in the same cluster and your cluster was deployed before August 17, 2020. Then, edit your `kube-proxy` manifest to include a node selector for multiple hardware architectures with the following command. This is a one-time operation. After you've added the selector to your manifest, you don't need to add it each time you update the add-on. If your cluster was deployed on or after August 17, 2020, then `kube-proxy` is already multi-architecture capable. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl edit -n kube-system daemonset/kube-proxy +---- ++ +Add the following node selector to the file in the editor and then save the file. For an example of where to include this text in the editor, see the https://github.com/aws/amazon-vpc-cni-k8s/blob/release-1.11/config/master/aws-k8s-cni.yaml#L265-#L269[CNI manifest] file on GitHub. This enables Kubernetes to pull the correct hardware image based on the node's hardware architecture. ++ +[source,yaml,subs="verbatim,attributes"] +---- +- key: "kubernetes.io/arch" + operator: In + values: + - amd64 + - arm64 +---- +. If your cluster was originally created with Kubernetes version `1.14` or later, then you can skip this step because `kube-proxy` already includes this `Affinity Rule`. If you originally created an Amazon EKS cluster with Kubernetes version `1.13` or earlier and intend to use Fargate nodes in your cluster, then edit your `kube-proxy` manifest to include a `NodeAffinity` rule to prevent `kube-proxy` Pods from scheduling on Fargate nodes. This is a one-time edit. Once you've added the `Affinity Rule` to your manifest, you don't need to add it each time that you update the add-on. Edit your `kube-proxy` DaemonSet. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl edit -n kube-system daemonset/kube-proxy +---- ++ +Add the following `Affinity Rule` to the DaemonSet `spec` section of the file in the editor and then save the file. For an example of where to include this text in the editor, see the https://github.com/aws/amazon-vpc-cni-k8s/blob/release-1.11/config/master/aws-k8s-cni.yaml#L270-#L273[CNI manifest] file on GitHub. ++ +[source,yaml,subs="verbatim,attributes"] +---- +- key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate +---- \ No newline at end of file diff --git a/latest/ug/networking/lbc-helm.adoc b/latest/ug/networking/lbc-helm.adoc new file mode 100644 index 000000000..080021479 --- /dev/null +++ b/latest/ug/networking/lbc-helm.adoc @@ -0,0 +1,151 @@ +include::../attributes.txt[] + +[.topic] +[#lbc-helm] += Install {aws} Load Balancer Controller with Helm +:info_titleabbrev: Install with Helm + +[abstract] +-- +Learn how to install the {aws} Load Balancer Controller on Amazon EKS using Helm to manage K8s load balancing with {aws} Cloud. Discover the prerequisites and steps for creating an IAM role, installing with Helm, and verifying the controller deployment. +-- + +[TIP] +==== +With Amazon EKS Auto Mode, you don't need to install or upgrade networking add-ons. Auto Mode includes pod networking and load balancing capabilities. + +For more information, see <>. +==== + +This topic describes how to install the {aws} Load Balancer Controller using Helm, a package manager for Kubernetes, and `eksctl`. The controller is installed with default options. For more information about the controller, including details on configuring it with annotations, see the https://kubernetes-sigs.github.io/aws-load-balancer-controller/[{aws} Load Balancer Controller Documentation] on GitHub. + +In the following steps, replace the example values with your own values. + +[#lbc-prereqs] +== Prerequisites + +Before starting this tutorial, you must complete the following steps: + +* Create an Amazon EKS cluster. To create one, see <>. +* Install https://helm.sh/docs/helm/helm_install/[Helm] on your local machine. +* Make sure that your Amazon VPC CNI plugin for Kubernetes, `kube-proxy`, and CoreDNS add-ons are at the minimum versions listed in <>. +* Learn about {aws} Elastic Load Balancing concepts. For more information, see the link:elasticloadbalancing/latest/userguide/[Elastic Load Balancing User Guide,type="documentation"]. +* Learn about Kubernetes https://kubernetes.io/docs/concepts/services-networking/service/[service] and https://kubernetes.io/docs/concepts/services-networking/ingress/[ingress] resources. + +[#lbc-considerations] +=== Considerations + +Before proceeding with the configuration steps on this page, consider the following: + +* The IAM policy and role (`AmazonEKSLoadBalancerControllerRole`) can be reused across multiple EKS clusters in the same {aws} account. +* If you're installing the controller on the same cluster where the role (`AmazonEKSLoadBalancerControllerRole`) was originally created, go to <> after verifying the role exists. +* If you're using IAM Roles for Service Accounts (IRSA), IRSA must be set up for each cluster, and the OpenID Connect (OIDC) provider ARN in the role's trust policy is specific to each EKS cluster. Additionally, if you're installing the controller on a new cluster with an existing `AmazonEKSLoadBalancerControllerRole`, update the role's trust policy to include the new cluster's OIDC provider and create a new service account with the appropriate role annotation. To determine whether you already have an OIDC provider, or to create one, see <>. + +[#lbc-helm-iam] +== Step 1: Create IAM Role using `eksctl` +The following steps refer to the {aws} Load Balancer Controller *v2.14.1* release version. For more information about all releases, see the https://github.com/kubernetes-sigs/aws-load-balancer-controller/releases/[{aws} Load Balancer Controller Release Page] on GitHub. + +. Download an IAM policy for the {aws} Load Balancer Controller that allows it to make calls to {aws} APIs on your behalf. ++ +[source,shell,subs="verbatim,attributes"] +---- +curl -O https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.14.1/docs/install/iam_policy.json +---- +** If you are a non-standard {aws} partition, such as a Government or China region, https://github.com/kubernetes-sigs/aws-load-balancer-controller/tree/main/docs/install[review the policies on GitHub] and download the appropriate policy for your region. +. Create an IAM policy using the policy downloaded in the previous step. ++ +[source,shell,subs="verbatim,attributes"] +---- +aws iam create-policy \ + --policy-name AWSLoadBalancerControllerIAMPolicy \ + --policy-document file://iam_policy.json +---- ++ +NOTE: If you view the policy in the {aws-management-console}, the console shows warnings for the *ELB* service, but not for the *ELB v2* service. This happens because some of the actions in the policy exist for *ELB v2*, but not for *ELB*. You can ignore the warnings for *ELB*. +. Replace the values for cluster name, region code, and account ID. ++ +[source,shell,subs="verbatim,attributes"] +---- +eksctl create iamserviceaccount \ + --cluster= \ + --namespace=kube-system \ + --name=aws-load-balancer-controller \ + --attach-policy-arn={arn-aws}iam:::policy/AWSLoadBalancerControllerIAMPolicy \ + --override-existing-serviceaccounts \ + --region \ + --approve +---- + + +[#lbc-helm-install] +== Step 2: Install {aws} Load Balancer Controller + +. Add the `eks-charts` Helm chart repository. {aws} maintains https://github.com/aws/eks-charts[this repository] on GitHub. ++ +[source,shell,subs="verbatim,attributes"] +---- +helm repo add eks https://aws.github.io/eks-charts +---- +. Update your local repo to make sure that you have the most recent charts. ++ +[source,shell,subs="verbatim,attributes"] +---- +helm repo update eks +---- +. Install the {aws} Load Balancer Controller. ++ +If you're deploying the controller to Amazon EC2 nodes that have https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node[restricted access to the Amazon EC2 instance metadata service (IMDS)], or if you're deploying to Fargate or Amazon EKS Hybrid Nodes, then add the following flags to the `helm` command that follows: ++ +*** `--set region=[.replaceable]``region-code``` +*** `--set vpcId=[.replaceable]``vpc-xxxxxxxx``` ++ +Replace [.replaceable]`my-cluster` with the name of your cluster. In the following command, `aws-load-balancer-controller` is the Kubernetes service account that you created in a previous step. ++ +For more information about configuring the helm chart, see https://github.com/aws/eks-charts/blob/master/stable/aws-load-balancer-controller/values.yaml[values.yaml] on GitHub. ++ +[source,shell,subs="verbatim,attributes"] +---- +helm install aws-load-balancer-controller eks/aws-load-balancer-controller \ + -n kube-system \ + --set clusterName=my-cluster \ + --set serviceAccount.create=false \ + --set serviceAccount.name=aws-load-balancer-controller \ + --version 1.14.0 +---- + + +[IMPORTANT] +==== +The deployed chart doesn't receive security updates automatically. You need to manually upgrade to a newer chart when it becomes available. When upgrading, change [.replaceable]`install` to `upgrade` in the previous command. +==== + +The `helm install` command automatically installs the custom resource definitions (CRDs) for the controller. The `helm upgrade` command does not. If you use `helm upgrade,` you must manually install the CRDs. Run the following command to install the CRDs: + +[source,shell,subs="verbatim,attributes"] +---- +wget https://raw.githubusercontent.com/aws/eks-charts/master/stable/aws-load-balancer-controller/crds/crds.yaml +kubectl apply -f crds.yaml +---- + + +[#lbc-helm-verify] +== Step 3: Verify that the controller is installed +. Verify that the controller is installed. ++ +[source,shell,subs="verbatim,attributes"] +---- +kubectl get deployment -n kube-system aws-load-balancer-controller +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +NAME READY UP-TO-DATE AVAILABLE AGE +aws-load-balancer-controller 2/2 2 2 84s +---- ++ +You receive the previous output if you deployed using Helm. If you deployed using the Kubernetes manifest, you only have one replica. +. Before using the controller to provision {aws} resources, your cluster must meet specific requirements. For more information, see <> and <>. + + diff --git a/latest/ug/networking/lbc-manifest.adoc b/latest/ug/networking/lbc-manifest.adoc new file mode 100644 index 000000000..eea269e12 --- /dev/null +++ b/latest/ug/networking/lbc-manifest.adoc @@ -0,0 +1,324 @@ +include::../attributes.txt[] + +[.topic] +[#lbc-manifest] += Install {aws} Load Balancer Controller with manifests +:info_titleabbrev: Install with manifests + +[abstract] +-- +Install the {aws} Load Balancer Controller add-on for Amazon EKS using Kubernetes manifests to provision Elastic Load Balancing resources. Configure IAM role and install `cert-manager` before applying controller manifest. +-- + +[TIP] +==== +With Amazon EKS Auto Mode, you don't need to install or upgrade networking add-ons. Auto Mode includes pod networking and load balancing capabilities. + +For more information, see <>. +==== + +This topic describes how to install the controller by downloading and applying Kubernetes manifests. You can view the full https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/[documentation] for the controller on GitHub. + +In the following steps, replace the example values with your own values. + +[#lbc-manifest-prereqs] +== Prerequisites + +Before starting this tutorial, you must complete the following steps: + +* Create an Amazon EKS cluster. To create one, see <>. +* Install https://helm.sh/docs/helm/helm_install/[Helm] on your local machine. +* Make sure that your Amazon VPC CNI plugin for Kubernetes, `kube-proxy`, and CoreDNS add-ons are at the minimum versions listed in <>. +* Learn about {aws} Elastic Load Balancing concepts. For more information, see the link:elasticloadbalancing/latest/userguide/[Elastic Load Balancing User Guide,type="documentation"]. +* Learn about Kubernetes https://kubernetes.io/docs/concepts/services-networking/service/[service] and https://kubernetes.io/docs/concepts/services-networking/ingress/[ingress] resources. + +[#lbc-manifest-considerations] +=== Considerations + +Before proceeding with the configuration steps on this page, consider the following: + +* The IAM policy and role (`AmazonEKSLoadBalancerControllerRole`) can be reused across multiple EKS clusters in the same {aws} account. +* If you're installing the controller on the same cluster where the role (`AmazonEKSLoadBalancerControllerRole`) was originally created, go to <> after verifying the role exists. +* If you're using IAM Roles for Service Accounts (IRSA), IRSA must be set up for each cluster, and the OpenID Connect (OIDC) provider ARN in the role's trust policy is specific to each EKS cluster. Additionally, if you're installing the controller on a new cluster with an existing `AmazonEKSLoadBalancerControllerRole`, update the role's trust policy to include the new cluster's OIDC provider and create a new service account with the appropriate role annotation. To determine whether you already have an OIDC provider, or to create one, see <>. + +[#lbc-iam] +== Step 1: Configure IAM +The following steps refer to the {aws} Load Balancer Controller *v2.14.1* release version. For more information about all releases, see the https://github.com/kubernetes-sigs/aws-load-balancer-controller/releases/[{aws} Load Balancer Controller Release Page] on GitHub. + +. Download an IAM policy for the {aws} Load Balancer Controller that allows it to make calls to {aws} APIs on your behalf. ++ +==== +[role="tablist"] +{aws}::: ++ +[source,shell,subs="verbatim,attributes"] +---- +curl -O https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.14.1/docs/install/iam_policy.json +---- + + +{aws} GovCloud (US)::: ++ +[source,shell,subs="verbatim,attributes"] +---- +curl -O https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.14.1/docs/install/iam_policy_us-gov.json +---- ++ +[source,shell,subs="verbatim,attributes"] +---- +mv iam_policy_us-gov.json iam_policy.json +---- +==== +. Create an IAM policy using the policy downloaded in the previous step. ++ +[source,shell,subs="verbatim,attributes"] +---- +aws iam create-policy \ + --policy-name AWSLoadBalancerControllerIAMPolicy \ + --policy-document file://iam_policy.json +---- ++ +NOTE: If you view the policy in the {aws-management-console}, the console shows warnings for the *ELB* service, but not for the *ELB v2* service. This happens because some of the actions in the policy exist for *ELB v2*, but not for *ELB*. You can ignore the warnings for *ELB*. + +==== +[role="tablist"] +eksctl::: +.. Replace [.replaceable]`my-cluster` with the name of your cluster, [.replaceable]`111122223333` with your account ID, and then run the command. ++ +[source,shell,subs="verbatim,attributes"] +---- +eksctl create iamserviceaccount \ + --cluster=my-cluster \ + --namespace=kube-system \ + --name=aws-load-balancer-controller \ + --role-name AmazonEKSLoadBalancerControllerRole \ + --attach-policy-arn={arn-aws}iam::111122223333:policy/AWSLoadBalancerControllerIAMPolicy \ + --approve +---- + + +{aws} CLI and kubectl::: +.. Retrieve your cluster's OIDC provider ID and store it in a variable. ++ +[source,bash,subs="verbatim,attributes"] +---- +oidc_id=$(aws eks describe-cluster --name my-cluster --query "cluster.identity.oidc.issuer" --output text | cut -d '/' -f 5) +---- +.. Determine whether an IAM OIDC provider with your cluster's ID is already in your account. You need OIDC configured for both the cluster and IAM. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam list-open-id-connect-providers | grep $oidc_id | cut -d "/" -f4 +---- ++ +If output is returned, then you already have an IAM OIDC provider for your cluster. If no output is returned, then you must create an IAM OIDC provider for your cluster. For more information, see <>. +.. Copy the following contents to your device. Replace [.replaceable]`111122223333` with your account ID. Replace [.replaceable]`region-code` with the {aws} Region that your cluster is in. Replace [.replaceable]`EXAMPLED539D4633E53DE1B71EXAMPLE` with the output returned in the previous step. ++ +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:960915e9-fdc0-403a-b346-1818e22e9709[] +---- +.. Create the IAM role. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam create-role \ + --role-name AmazonEKSLoadBalancerControllerRole \ + --assume-role-policy-document file://"load-balancer-role-trust-policy.json" +---- +.. Attach the required Amazon EKS managed IAM policy to the IAM role. Replace [.replaceable]`111122223333` with your account ID. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam attach-role-policy \ + --policy-arn {arn-aws}iam::111122223333:policy/AWSLoadBalancerControllerIAMPolicy \ + --role-name AmazonEKSLoadBalancerControllerRole +---- +.. Copy the following contents to your device. Replace [.replaceable]`111122223333` with your account ID. After replacing the text, run the modified command to create the `aws-load-balancer-controller-service-account.yaml` file. ++ +[source,yaml,subs="verbatim,attributes"] +---- +cat >aws-load-balancer-controller-service-account.yaml <>. ++ +[source,bash,subs="verbatim,attributes"] +---- +quay.io/jetstack/cert-manager-cainjector:v1.13.5 +quay.io/jetstack/cert-manager-controller:v1.13.5 +quay.io/jetstack/cert-manager-webhook:v1.13.5 +---- +.. Replace `quay.io` in the manifest for the three images with your own registry name. The following command assumes that your private repository's name is the same as the source repository. Replace [.replaceable]`111122223333.dkr.ecr.region-code.amazonaws.com` with your private registry. ++ +[source,shell,subs="verbatim,attributes"] +---- +sed -i.bak -e 's|quay.io|111122223333.dkr.ecr.region-code.amazonaws.com|' ./cert-manager.yaml +---- +.. Apply the manifest. ++ +[source,shell,subs="verbatim,attributes"] +---- +kubectl apply \ + --validate=false \ + -f ./cert-manager.yaml +---- +==== + + +[#lbc-install] +== Step 3: Install {aws} Load Balancer Controller +. Download the controller specification. For more information about the controller, see the https://kubernetes-sigs.github.io/aws-load-balancer-controller/[documentation] on GitHub. ++ +[source,bash,subs="verbatim,attributes"] +---- +curl -Lo v2_14_1_full.yaml https://github.com/kubernetes-sigs/aws-load-balancer-controller/releases/download/v2.14.1/v2_14_1_full.yaml +---- +. Make the following edits to the file. ++ +.. If you downloaded the `v2_14_1_full.yaml` file, run the following command to remove the `ServiceAccount` section in the manifest. If you don't remove this section, the required annotation that you made to the service account in a previous step is overwritten. Removing this section also preserves the service account that you created in a previous step if you delete the controller. ++ +[source,shell,subs="verbatim,attributes"] +---- +sed -i.bak -e '764,773d' ./v2_14_1_full.yaml +---- ++ +If you downloaded a different file version, then open the file in an editor and remove the following lines. ++ +[source,yaml,subs="verbatim,attributes"] +---- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: aws-load-balancer-controller + name: aws-load-balancer-controller + namespace: kube-system +--- +---- +.. Replace `your-cluster-name` in the `Deployment` `spec` section of the file with the name of your cluster by replacing [.replaceable]`my-cluster` with the name of your cluster. ++ +[source,shell,subs="verbatim,attributes"] +---- +sed -i.bak -e 's|your-cluster-name|my-cluster|' ./v2_14_1_full.yaml +---- +.. If your nodes don't have access to the Amazon EKS Amazon ECR image repositories, then you need to pull the following image and push it to a repository that your nodes have access to. For more information on how to pull, tag, and push an image to your own repository, see <>. ++ +[source,bash,subs="verbatim,attributes"] +---- +public.ecr.aws/eks/aws-load-balancer-controller:v2.14.1 +---- ++ +Add your registry's name to the manifest. The following command assumes that your private repository's name is the same as the source repository and adds your private registry's name to the file. Replace [.replaceable]`111122223333.dkr.ecr.region-code.amazonaws.com` with your registry. This line assumes that you named your private repository the same as the source repository. If not, change the `eks/aws-load-balancer-controller` text after your private registry name to your repository name. ++ +[source,shell,subs="verbatim,attributes"] +---- +sed -i.bak -e 's|public.ecr.aws/eks/aws-load-balancer-controller|111122223333.dkr.ecr.region-code.amazonaws.com/eks/aws-load-balancer-controller|' ./v2_14_1_full.yaml +---- +.. (Required only for Fargate or Restricted IMDS) ++ +If you're deploying the controller to Amazon EC2 nodes that have https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node[restricted access to the Amazon EC2 instance metadata service (IMDS)], or if you're deploying to Fargate or Amazon EKS Hybrid Nodes, then add the `following parameters` under `- args:`. ++ +[source,yaml,subs="verbatim,attributes"] +---- +[...] +spec: + containers: + - args: + - --cluster-name=your-cluster-name + - --ingress-class=alb + - --aws-vpc-id=vpc-xxxxxxxx + - --aws-region=region-code + + +[...] +---- +. Apply the file. ++ +[source,shell,subs="verbatim,attributes"] +---- +kubectl apply -f v2_14_1_full.yaml +---- +. Download the `IngressClass` and `IngressClassParams` manifest to your cluster. ++ +[source,shell,subs="verbatim,attributes"] +---- +curl -Lo v2.14.1_ingclass.yaml https://github.com/kubernetes-sigs/aws-load-balancer-controller/releases/download/v2.14.1/v2_14_1_ingclass.yaml +---- +. Apply the manifest to your cluster. ++ +[source,shell,subs="verbatim,attributes"] +---- +kubectl apply -f v2_14_1_ingclass.yaml +---- + + +[#lbc-verify] +== Step 4: Verify that the controller is installed +. Verify that the controller is installed. ++ +[source,shell,subs="verbatim,attributes"] +---- +kubectl get deployment -n kube-system aws-load-balancer-controller +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +NAME READY UP-TO-DATE AVAILABLE AGE +aws-load-balancer-controller 2/2 2 2 84s +---- ++ +You receive the previous output if you deployed using Helm. If you deployed using the Kubernetes manifest, you only have one replica. +. Before using the controller to provision {aws} resources, your cluster must meet specific requirements. For more information, see <> and <>. diff --git a/latest/ug/networking/lbc-remove.adoc b/latest/ug/networking/lbc-remove.adoc new file mode 100644 index 000000000..3f7f6ea51 --- /dev/null +++ b/latest/ug/networking/lbc-remove.adoc @@ -0,0 +1,115 @@ +include::../attributes.txt[] + +[.topic] +[#lbc-remove] += Migrate apps from deprecated ALB Ingress Controller +:info_titleabbrev: Migrate from deprecated + +[abstract] +-- +Learn how to migrate from the deprecated ALB Ingress Controller to the latest {aws} Load Balancer Controller release, ensuring smooth transition and uninterrupted load balancing capabilities. +-- + +This topic describes how to migrate from deprecated controller versions. More specifically, it describes how to remove deprecated versions of the {aws} Load Balancer Controller. + + + +* Deprecated versions cannot be upgraded. You must remove them first, and then install a current version. ++ +[[lbc-deprecated-list]] +* Deprecated versions include: ++ +** {aws} ALB Ingress Controller for Kubernetes ("Ingress Controller"), a predecessor to the {aws} Load Balancer Controller. +** Any `0.1.[.replaceable]``x``` version of the {aws} Load Balancer Controller + + +[#lbc-remove-desc] +== Remove the deprecated controller version + +[NOTE] +==== + +You may have installed the deprecated version using Helm or manually with Kubernetes manifests. Complete the procedure using the tool that you originally installed it with. + +==== +. If you installed the `incubator/aws-alb-ingress-controller` Helm chart, uninstall it. ++ +[source,shell,subs="verbatim,attributes"] +---- +helm delete aws-alb-ingress-controller -n kube-system +---- +. If you have version `0.1.[.replaceable]``x``` of the `eks-charts/aws-load-balancer-controller` chart installed, uninstall it. The upgrade from `0.1.[.replaceable]``x``` to version `1.0.0` doesn't work due to incompatibility with the webhook API version. ++ +[source,shell,subs="verbatim,attributes"] +---- +helm delete aws-load-balancer-controller -n kube-system +---- +. Check to see if the controller is currently installed. ++ +[source,shell,subs="verbatim,attributes"] +---- +kubectl get deployment -n kube-system alb-ingress-controller +---- + ++ +This is the output if the controller isn't installed. ++ +[source,bash,subs="verbatim,attributes"] +---- +Error from server (NotFound): deployments.apps "alb-ingress-controller" not found +---- ++ +This is the output if the controller is installed. ++ +[source,bash,subs="verbatim,attributes"] +---- +NAME READY UP-TO-DATE AVAILABLE AGE +alb-ingress-controller 1/1 1 1 122d +---- +. Enter the following commands to remove the controller. ++ +[source,shell,subs="verbatim,attributes"] +---- +kubectl delete -f https://raw.githubusercontent.com/kubernetes-sigs/aws-alb-ingress-controller/v1.1.8/docs/examples/alb-ingress-controller.yaml +kubectl delete -f https://raw.githubusercontent.com/kubernetes-sigs/aws-alb-ingress-controller/v1.1.8/docs/examples/rbac-role.yaml +---- + + +[#lbc-migrate] +== Migrate to {aws} Load Balancer Controller + +To migrate from the ALB Ingress Controller for Kubernetes to the {aws} Load Balancer Controller, you need to: + +. Remove the ALB Ingress Controller (see above). +. <> +. Add an additional policy to the IAM Role used by the {aws} Load Balancer Controller. This policy permits the LBC to manage resources created by the ALB Ingress Controller for Kubernetes. +. Download the IAM policy. This policy permits the {aws} Load Balancer Controller to manage resources created by the ALB Ingress Controller for Kubernetes. You can also https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/main/docs/install/iam_policy_v1_to_v2_additional.json[view the policy]. ++ +[source,shell,subs="verbatim,attributes"] +---- +curl -O https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.14.1/docs/install/iam_policy_v1_to_v2_additional.json +---- +. If your cluster is in the {aws} GovCloud (US-East) or {aws} GovCloud (US-West) {aws} Regions, then replace `{arn-aws}` with `arn:aws-us-gov:`. ++ +[source,shell,subs="verbatim,attributes"] +---- +sed -i.bak -e 's|{arn-aws}|arn:aws-us-gov:|' iam_policy_v1_to_v2_additional.json +---- +. Create the IAM policy and note the ARN that is returned. ++ +[source,shell,subs="verbatim,attributes"] +---- +aws iam create-policy \ + --policy-name AWSLoadBalancerControllerAdditionalIAMPolicy \ + --policy-document file://iam_policy_v1_to_v2_additional.json +---- +. Attach the IAM policy to the IAM role used by the {aws} Load Balancer Controller. Replace [.replaceable]`your-role-name` with the name of the role, such as `AmazonEKSLoadBalancerControllerRole`. ++ +If you created the role using `eksctl`, then to find the role name that was created, open the link:cloudformation[{aws} CloudFormation console,type="console"] and select the *eksctl-[.replaceable]`my-cluster`-addon-iamserviceaccount-kube-system-aws-load-balancer-controller* stack. Select the *Resources* tab. The role name is in the *Physical ID* column. ++ +[source,shell,subs="verbatim,attributes"] +---- +aws iam attach-role-policy \ + --role-name your-role-name \ + --policy-arn {arn-aws}iam::111122223333:policy/AWSLoadBalancerControllerAdditionalIAMPolicy +---- \ No newline at end of file diff --git a/latest/ug/networking/managing-coredns.adoc b/latest/ug/networking/managing-coredns.adoc new file mode 100644 index 000000000..63417f6cc --- /dev/null +++ b/latest/ug/networking/managing-coredns.adoc @@ -0,0 +1,110 @@ +include::../attributes.txt[] + +[.topic] +[#managing-coredns] += Manage CoreDNS for DNS in Amazon EKS clusters +:info_titleabbrev: CoreDNS + +include::coredns-add-on-create.adoc[leveloffset=+1] + +include::coredns-add-on-update.adoc[leveloffset=+1] + +include::coredns-add-on-self-managed-update.adoc[leveloffset=+1] + +include::coredns-autoscaling.adoc[leveloffset=+1] + +include::coredns-metrics.adoc[leveloffset=+1] + +[abstract] +-- +Learn how to manage the CoreDNS Amazon EKS add-on for DNS service discovery in Kubernetes clusters with configuration updates and version upgrades. +-- + +[TIP] +==== +With Amazon EKS Auto Mode, you don't need to install or upgrade networking add-ons. Auto Mode includes pod networking and load balancing capabilities. + +For more information, see <>. +==== + +CoreDNS is a flexible, extensible DNS server that can serve as the Kubernetes cluster DNS. When you launch an Amazon EKS cluster with at least one node, two replicas of the CoreDNS image are deployed by default, regardless of the number of nodes deployed in your cluster. The CoreDNS Pods provide name resolution for all Pods in the cluster. The CoreDNS Pods can be deployed to Fargate nodes if your cluster includes a <> with a namespace that matches the namespace for the CoreDNS `deployment`. For more information about CoreDNS, see https://kubernetes.io/docs/tasks/administer-cluster/coredns/[Using CoreDNS for Service Discovery] in the Kubernetes documentation. + +[#coredns-versions] +== CoreDNS versions + +The following table lists the latest version of the Amazon EKS add-on type for each Kubernetes version. + +|=== +| Kubernetes version | CoreDNS version + +| 1.34 | v1.12.4-eksbuild.1 +| 1.33 | v1.12.4-eksbuild.1 +| 1.32 | v1.11.4-eksbuild.24 +| 1.31 | v1.11.4-eksbuild.24 +| 1.30 | v1.11.4-eksbuild.24 +| 1.29 | v1.11.4-eksbuild.24 +| 1.28 | v1.10.1-eksbuild.38 + +|=== + +[IMPORTANT] +==== + +If you're self-managing this add-on, the versions in the table might not be the same as the available self-managed versions. For more information about updating the self-managed type of this add-on, see <>. + +==== + +[#coredns-upgrade] +== Important CoreDNS upgrade considerations + +* CoreDNS updates utilize a PodDisruptionBudget to help maintain DNS service availability during the update process. + +* To improve the stability and availability of the CoreDNS Deployment, versions `v1.9.3-eksbuild.6` and later and `v1.10.1-eksbuild.3` are deployed with a `PodDisruptionBudget`. If you've deployed an existing `PodDisruptionBudget`, your upgrade to these versions might fail. If the upgrade fails, completing one of the following tasks should resolve the issue: ++ +** When doing the upgrade of the Amazon EKS add-on, choose to override the existing settings as your conflict resolution option. If you've made other custom settings to the Deployment, make sure to back up your settings before upgrading so that you can reapply your other custom settings after the upgrade. +** Remove your existing `PodDisruptionBudget` and try the upgrade again. +* In EKS add-on versions `v1.9.3-eksbuild.3` and later and `v1.10.1-eksbuild.6` and later, the CoreDNS Deployment sets the `readinessProbe` to use the `/ready` endpoint. This endpoint is enabled in the `Corefile` configuration file for CoreDNS. ++ +If you use a custom `Corefile`, you must add the `ready` plugin to the config, so that the `/ready` endpoint is active in CoreDNS for the probe to use. +* In EKS add-on versions `v1.9.3-eksbuild.7` and later and `v1.10.1-eksbuild.4` and later, you can change the `PodDisruptionBudget`. You can edit the add-on and change these settings in the *Optional configuration settings* using the fields in the following example. This example shows the default `PodDisruptionBudget`. ++ +[source,json,subs="verbatim,attributes"] +---- +{ + "podDisruptionBudget": { + "enabled": true, + "maxUnavailable": 1 + } +} +---- +// Not using Kubernetes here because the _ causes issues with the rendering. ++ +You can set `maxUnavailable` or `minAvailable`, but you can't set both in a single `PodDisruptionBudget`. For more information about `PodDisruptionBudgets`, see https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget[Specifying a PodDisruptionBudget] in the _Kubernetes documentation_. ++ +Note that if you set `enabled` to `false`, the `PodDisruptionBudget` isn't removed. After you set this field to `false`, you must delete the `PodDisruptionBudget` object. Similarly, if you edit the add-on to use an older version of the add-on (downgrade the add-on) after upgrading to a version with a `PodDisruptionBudget`, the `PodDisruptionBudget` isn't removed. To delete the `PodDisruptionBudget`, you can run the following command: ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl delete poddisruptionbudget coredns -n kube-system +---- +* In EKS add-on versions `v1.10.1-eksbuild.5` and later, change the default toleration from `node-role.kubernetes.io/master:NoSchedule` to `node-role.kubernetes.io/control-plane:NoSchedule` to comply with KEP 2067. For more information about KEP 2067, see https://github.com/kubernetes/enhancements/tree/master/keps/sig-cluster-lifecycle/kubeadm/2067-rename-master-label-taint#renaming-the-node-rolekubernetesiomaster-node-taint[KEP-2067: Rename the kubeadm "master" label and taint] in the _Kubernetes Enhancement Proposals (KEPs)_ on GitHub. ++ +In EKS add-on versions `v1.8.7-eksbuild.8` and later and `v1.9.3-eksbuild.9` and later, both tolerations are set to be compatible with every Kubernetes version. +* In EKS add-on versions `v1.9.3-eksbuild.11` and `v1.10.1-eksbuild.7` and later, the CoreDNS Deployment sets a default value for `topologySpreadConstraints`. The default value ensures that the CoreDNS Pods are spread across the Availability Zones if there are nodes in multiple Availability Zones available. You can set a custom value that will be used instead of the default value. The default value follows: ++ +[source,yaml,subs="verbatim,attributes"] +---- +topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + k8s-app: kube-dns +---- + + +[#coredns-upgrade-1.11] +=== CoreDNS `v1.11` upgrade considerations + +* In EKS add-on versions `v1.11.1-eksbuild.4` and later, the container image is based on a https://gallery.ecr.aws/eks-distro-build-tooling/eks-distro-minimal-base[minimal base image] maintained by Amazon EKS Distro, which contains minimal packages and doesn't have shells. For more information, see https://distro.eks.amazonaws.com/[Amazon EKS Distro]. The usage and troubleshooting of the CoreDNS image remains the same. diff --git a/latest/ug/networking/managing-kube-proxy.adoc b/latest/ug/networking/managing-kube-proxy.adoc new file mode 100644 index 000000000..a4b8f3bb5 --- /dev/null +++ b/latest/ug/networking/managing-kube-proxy.adoc @@ -0,0 +1,81 @@ +include::../attributes.txt[] + +[.topic] +[#managing-kube-proxy] += Manage `kube-proxy` in Amazon EKS clusters +:info_titleabbrev: kube-proxy + +[abstract] +-- +Learn how to manage the `kube-proxy` add-on on your Amazon EKS cluster to manage network rules and enable network communication to your Pods. +-- + +[TIP] +==== +With Amazon EKS Auto Mode, you don't need to install or upgrade networking add-ons. Auto Mode includes pod networking and load balancing capabilities. + +For more information, see <>. +==== + +//GDC: Need DF to review + + +We recommend adding the Amazon EKS type of the add-on to your cluster instead of using the self-managed type of the add-on. If you're not familiar with the difference between the types, see <>. For more information about adding an Amazon EKS add-on to your cluster, see <>. If you're unable to use the Amazon EKS add-on, we encourage you to submit an issue about why you can't to the https://github.com/aws/containers-roadmap/issues[Containers roadmap GitHub repository]. + +The `kube-proxy` add-on is deployed on each Amazon EC2 node in your Amazon EKS cluster. It maintains network rules on your nodes and enables network communication to your Pods. The add-on isn't deployed to Fargate nodes in your cluster. For more information, see https://kubernetes.io/docs/reference/command-line-tools-reference/kube-proxy/[kube-proxy] in the Kubernetes documentation. + +== Install as Amazon EKS Add-on + + +[#kube-proxy-versions] +== `kube-proxy` versions + +The following table lists the latest version of the Amazon EKS add-on type for each Kubernetes version. + +|=== +| Kubernetes version | `kube-proxy` version + +| 1.34 | v1.34.0-eksbuild.5 +| 1.33 | v1.33.3-eksbuild.11 +| 1.32 | v1.32.6-eksbuild.13 +| 1.31 | v1.31.10-eksbuild.13 +| 1.30 | v1.30.14-eksbuild.8 +| 1.29 | v1.29.15-eksbuild.16 +| 1.28 | v1.28.15-eksbuild.31 + +|=== + +[NOTE] +==== + +An earlier version of the documentation was incorrect. `kube-proxy` versions `v1.28.5`, `v1.27.9`, and `v1.26.12` aren't available. + +If you're self-managing this add-on, the versions in the table might not be the same as the available self-managed versions. + +==== + +[#managing-kube-proxy-images] +== `kube-proxy` container image + +The `kube-proxy` container image is based on a https://gallery.ecr.aws/eks-distro-build-tooling/eks-distro-minimal-base-iptables[minimal base image] maintained by Amazon EKS Distro, which contains minimal packages and doesn't have shells. For more information, see https://distro.eks.amazonaws.com/[Amazon EKS Distro]. + +The following table lists the latest available self-managed `kube-proxy` container image version for each Amazon EKS cluster version. + + +|=== +| Version | kube-proxy + +| 1.34 | v1.34.0-eksbuild.5 +| 1.33 | v1.33.3-minimal-eksbuild.11 +| 1.32 | v1.32.6-minimal-eksbuild.13 +| 1.31 | v1.31.10-minimal-eksbuild.13 +| 1.30 | v1.30.14-minimal-eksbuild.8 +| 1.29 | v1.29.15-minimal-eksbuild.16 +| 1.28 | v1.28.15-minimal-eksbuild.31 + +|=== + +When you <>, you specify a valid Amazon EKS add-on version, which might not be a version listed in this table. This is because <> versions don't always match container image versions specified when updating the self-managed type of this add-on. When you update the self-managed type of this add-on, you specify a valid container image version listed in this table. + +include::kube-proxy-add-on-self-managed-update.adoc[leveloffset=+1] + diff --git a/latest/ug/networking/managing-vpc-cni.adoc b/latest/ug/networking/managing-vpc-cni.adoc new file mode 100644 index 000000000..95475422b --- /dev/null +++ b/latest/ug/networking/managing-vpc-cni.adoc @@ -0,0 +1,73 @@ +include::../attributes.txt[] + +[.topic] +[#managing-vpc-cni] += Assign IPs to Pods with the Amazon VPC CNI +:info_titleabbrev: Amazon VPC CNI + +[abstract] +-- +Discover how the Amazon VPC CNI plugin for Kubernetes add-on works to assign private IP addresses and create network interfaces for Pods and services in your Amazon EKS cluster. +-- + +[TIP] +==== +With Amazon EKS Auto Mode, you don't need to install or upgrade networking add-ons. Auto Mode includes pod networking and load balancing capabilities. + +For more information, see <>. +==== + +The Amazon VPC CNI plugin for Kubernetes add-on is deployed on each Amazon EC2 node in your Amazon EKS cluster. The add-on creates link:AWSEC2/latest/UserGuide/using-eni.html[elastic network interfaces,type="documentation"] and attaches them to your Amazon EC2 nodes. The add-on also assigns a private `IPv4` or `IPv6` address from your VPC to each Pod. + +A version of the add-on is deployed with each Fargate node in your cluster, but you don't update it on Fargate nodes. Other compatible CNI plugins are available for use on Amazon EKS clusters, but this is the only CNI plugin supported by Amazon EKS for nodes that run on {aws} infrastructure. For more information about the other compatible CNI plugins, see <>. The VPC CNI isn't supported for use with hybrid nodes. For more information about your CNI options for hybrid nodes, see <>. + +The following table lists the latest available version of the Amazon EKS add-on type for each Kubernetes version. + +[#vpc-cni-latest-available-version] +== Amazon VPC CNI versions + +|=== +| Kubernetes version | Amazon EKS type of VPC CNI version + +| 1.34 | v1.20.4-eksbuild.1 +| 1.33 | v1.20.4-eksbuild.1 +| 1.32 | v1.20.4-eksbuild.1 +| 1.31 | v1.20.4-eksbuild.1 +| 1.30 | v1.20.4-eksbuild.1 +| 1.29 | v1.20.4-eksbuild.1 +| 1.28 | v1.20.4-eksbuild.1 +|=== + +[IMPORTANT] +==== + +If you're self-managing this add-on, the versions in the table might not be the same as the available self-managed versions. For more information about updating the self-managed type of this add-on, see <>. + +==== + +[IMPORTANT] +==== + +To upgrade to VPC CNI v1.12.0 or later, you must upgrade to VPC CNI v1.7.0 first. We recommend that you update one minor version at a time. + +==== + + +[#manage-vpc-cni-add-on-on-considerations] +== Considerations + +The following are considerations for using the feature. + +* Versions are specified as `major-version.minor-version.patch-version-eksbuild.build-number`. +* Check version compatibility for each feature. Some features of each release of the Amazon VPC CNI plugin for Kubernetes require certain Kubernetes versions. When using different Amazon EKS features, if a specific version of the add-on is required, then it's noted in the feature documentation. Unless you have a specific reason for running an earlier version, we recommend running the latest version. + +include::vpc-add-on-create.adoc[leveloffset=+1] + +include::vpc-add-on-update.adoc[leveloffset=+1] + +include::vpc-add-on-self-managed-update.adoc[leveloffset=+1] + +include::cni-iam-role.adoc[leveloffset=+1] + +include::pod-networking-use-cases.adoc[leveloffset=+1] + diff --git a/latest/ug/networking/network-policies-troubleshooting.adoc b/latest/ug/networking/network-policies-troubleshooting.adoc index 13d789267..6d6041350 100644 --- a/latest/ug/networking/network-policies-troubleshooting.adoc +++ b/latest/ug/networking/network-policies-troubleshooting.adoc @@ -1,22 +1,213 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[network-policies-troubleshooting,network-policies-troubleshooting.title]] -= Troubleshooting [.noloc]`Kubernetes` network policies For Amazon EKS +[#network-policies-troubleshooting] += Troubleshooting Kubernetes network policies For Amazon EKS :info_titleabbrev: Troubleshooting -include::../attributes.txt[] - [abstract] -- Learn how to troubleshoot and investigate network connections that use network policies. -- -You can troubleshoot and investigate network connections that use network policies by reading the <> and by running tools from the <>. +This is the troubleshooting guide for network policy feature of the Amazon VPC CNI. + +//Manual Table of Contents, b/c the ToC gets long if each issue gets an anchor +This guide covers: + +* Install information, CRD and RBAC permissions <> +* Logs to examine when diagnosing network policy problems <> +* Running the eBPF SDK collection of tools to troubleshoot +* Known issues and solutions <> + +[NOTE] +==== +Note that network policies are only applied to pods that are made by Kubernetes _Deployments_. For more limitations of the network policies in the VPC CNI, see <>. +==== + +You can troubleshoot and investigate network connections that use network policies by reading the <> and by running tools from the <>. + +[#network-policies-troubleshooting-permissions] +== New `policyendpoints` CRD and permissions + +* CRD: `policyendpoints.networking.k8s.aws` +* Kubernetes API: `apiservice` called `v1.networking.k8s.io` +* Kubernetes resource: `Kind: NetworkPolicy` +* RBAC: `ClusterRole` called `aws-node` (VPC CNI), `ClusterRole` called `eks:network-policy-controller` (network policy controller in EKS cluster control plane) + +For network policy, the VPC CNI creates a new `CustomResourceDefinition` (CRD) called `policyendpoints.networking.k8s.aws`. The VPC CNI must have permissions to create the CRD and create CustomResources (CR) of this and the other CRD installed by the VPC CNI (`eniconfigs.crd.k8s.amazonaws.com`). Both of the CRDs are available in the link:https://github.com/aws/amazon-vpc-cni-k8s/blob/master/charts/aws-vpc-cni/crds/customresourcedefinition.yaml[`crds.yaml` file] on GitHub. Specifically, the VPC CNI must have "get", "list", and "watch" verb permissions for `policyendpoints`. + +The Kubernetes _Network Policy_ is part of the `apiservice` called `v1.networking.k8s.io`, and this is `apiversion: networking.k8s.io/v1` in your policy YAML files. The VPC CNI `DaemonSet` must have permissions to use this part of the Kubernetes API. + +The VPC CNI permissions are in a `ClusterRole` called `aws-node`. Note that `ClusterRole` objects aren't grouped in namespaces. The following shows the `aws-node` of a cluster: + +[source,bash] +---- +kubectl get clusterrole aws-node -o yaml +---- + +[source,yaml] +---- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/instance: aws-vpc-cni + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: aws-node + app.kubernetes.io/version: v1.19.4 + helm.sh/chart: aws-vpc-cni-1.19.4 + k8s-app: aws-node + name: aws-node +rules: +- apiGroups: + - crd.k8s.amazonaws.com + resources: + - eniconfigs + verbs: + - list + - watch + - get +- apiGroups: + - "" + resources: + - namespaces + verbs: + - list + - watch + - get +- apiGroups: + - "" + resources: + - pods + verbs: + - list + - watch + - get +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch + - get +- apiGroups: + - "" + - events.k8s.io + resources: + - events + verbs: + - create + - patch + - list +- apiGroups: + - networking.k8s.aws + resources: + - policyendpoints + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.aws + resources: + - policyendpoints/status + verbs: + - get +- apiGroups: + - vpcresources.k8s.aws + resources: + - cninodes + verbs: + - get + - list + - watch + - patch +---- + +Also, a new controller runs in the control plane of each EKS cluster. The controller uses the permissions of the `ClusterRole` called `eks:network-policy-controller`. The following shows the `eks:network-policy-controller` of a cluster: + +[source,bash] +---- +kubectl get clusterrole eks:network-policy-controller -o yaml +---- + +[source,yaml] +---- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: amazon-network-policy-controller-k8s + name: eks:network-policy-controller +rules: +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.aws + resources: + - policyendpoints + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - networking.k8s.aws + resources: + - policyendpoints/finalizers + verbs: + - update +- apiGroups: + - networking.k8s.aws + resources: + - policyendpoints/status + verbs: + - get + - patch + - update +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - patch + - update + - watch +---- -[[network-policies-troubleshooting-flowlogs,network-policies-troubleshooting-flowlogs.title]] +[#network-policies-troubleshooting-flowlogs] == Network policy logs -Whether connections are allowed or denied by a network policies is logged in _flow logs_. The network policy logs on each node include the flow logs for every pod that has a network policy. Network policy logs are stored at `/var/log/aws-routed-eni/network-policy-agent.log`. The following example is from a `network-policy-agent.log` file: +Each decision by the VPC CNI whether connections are allowed or denied by a network policies is logged in _flow logs_. The network policy logs on each node include the flow logs for every pod that has a network policy. Network policy logs are stored at `/var/log/aws-routed-eni/network-policy-agent.log`. The following example is from a `network-policy-agent.log` file: [source,bash,subs="verbatim,attributes"] ---- @@ -30,11 +221,11 @@ Network policy logs are disabled by default. To enable the network policy logs, [NOTE] ==== -Network policy logs require an additional 1 vCPU for the `aws-network-policy-agent` container in the VPC CNI `aws-node` daemonset manifest. +Network policy logs require an additional 1 vCPU for the `aws-network-policy-agent` container in the VPC CNI `aws-node` `DaemonSet` manifest. ==== -[[cni-network-policy-flowlogs-addon,cni-network-policy-flowlogs-addon.title]] +[#cni-network-policy-flowlogs-addon] === Amazon EKS add-on *{aws-management-console}*:: @@ -43,7 +234,7 @@ Network policy logs require an additional 1 vCPU for the `aws-network-policy-age .. In the left navigation pane, select *Clusters*, and then select the name of the cluster that you want to configure the Amazon VPC CNI add-on for. .. Choose the *Add-ons* tab. .. Select the box in the top right of the add-on box and then choose *Edit*. -.. On the *Configure [.replaceable]`name of addon`* page: +.. On the *Configure [.replaceable]`Amazon VPC CNI`* page: + ... Select a `v1.14.0-eksbuild.3` or later version in the *Version* dropdown list. ... Expand the *Optional configuration settings*. @@ -75,12 +266,12 @@ aws eks update-addon --cluster-name my-cluster --addon-name vpc-cni --addon-vers ---- -[[cni-network-policy-flowlogs-selfmanaged,cni-network-policy-flowlogs-selfmanaged.title]] +[#cni-network-policy-flowlogs-selfmanaged] === Self-managed add-on Helm:: -If you have installed the [.noloc]`Amazon VPC CNI plugin for Kubernetes` through `helm`, you can update the configuration to write the network policy logs. +If you have installed the Amazon VPC CNI plugin for Kubernetes through `helm`, you can update the configuration to write the network policy logs. .. Run the following command to enable network policy. + @@ -90,9 +281,9 @@ helm upgrade --set nodeAgent.enablePolicyEventLogs=true aws-vpc-cni --namespace ---- -[.noloc]`kubectl`:: +kubectl:: -If you have installed the [.noloc]`Amazon VPC CNI plugin for Kubernetes` through `kubectl`, you can update the configuration to write the network policy logs. +If you have installed the Amazon VPC CNI plugin for Kubernetes through `kubectl`, you can update the configuration to write the network policy logs. .. Open the `aws-node` `DaemonSet` in your editor. + @@ -100,7 +291,7 @@ If you have installed the [.noloc]`Amazon VPC CNI plugin for Kubernetes` through ---- kubectl edit daemonset -n kube-system aws-node ---- -.. Replace the `false` with `true` in the command argument `--enable-policy-event-logs=false` in the `args:` in the `aws-network-policy-agent` container in the VPC CNI `aws-node` daemonset manifest. +.. Replace the `false` with `true` in the command argument `--enable-policy-event-logs=false` in the `args:` in the `aws-network-policy-agent` container in the VPC CNI `aws-node` `DaemonSet` manifest. + [source,yaml,subs="verbatim,attributes"] ---- @@ -109,15 +300,15 @@ kubectl edit daemonset -n kube-system aws-node ---- -[[network-policies-cloudwatchlogs,network-policies-cloudwatchlogs.title]] -== Send network policy logs to Amazon CloudWatch Logs +[#network-policies-cloudwatchlogs] +=== Send network policy logs to Amazon CloudWatch Logs You can monitor the network policy logs using services such as Amazon CloudWatch Logs. You can use the following methods to send the network policy logs to CloudWatch Logs. For EKS clusters, the policy logs will be located under `/aws/eks/[.replaceable]``cluster-name``/cluster/` and for self-managed K8S clusters, the logs will be placed under `/aws/k8s-cluster/cluster/`. -[[network-policies-cwl-agent,network-policies-cwl-agent.title]] -=== Send network policy logs with [.noloc]`Amazon VPC CNI plugin for Kubernetes` +[#network-policies-cwl-agent] +==== Send network policy logs with Amazon VPC CNI plugin for Kubernetes If you enable network policy, a second container is add to the `aws-node` pods for a _node agent_. This node agent can send the network policy logs to CloudWatch Logs. @@ -128,34 +319,19 @@ Only the network policy logs are sent by the node agent. Other logs made by the ==== -[[cni-network-policy-cwl-agent-prereqs,cni-network-policy-cwl-agent-prereqs.title]] -==== Prerequisites +[#cni-network-policy-cwl-agent-prereqs] +===== Prerequisites * Add the following permissions as a stanza or separate policy to the IAM role that you are using for the VPC CNI. + [source,json,subs="verbatim,attributes"] ---- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "VisualEditor0", - "Effect": "Allow", - "Action": [ - "logs:DescribeLogGroups", - "logs:CreateLogGroup", - "logs:CreateLogStream", - "logs:PutLogEvents" - ], - "Resource": "*" - } - ] -} +include::iam_snippet:ced6be33-8b93-4011-b4bd-bc98c987cca5[] ---- -[[cni-network-policy-cwl-agent-addon,cni-network-policy-cwl-agent-addon.title]] -==== Amazon EKS add-on +[#cni-network-policy-cwl-agent-addon] +===== Amazon EKS add-on *{aws-management-console}*:: @@ -163,7 +339,7 @@ Only the network policy logs are sent by the node agent. Other logs made by the .. In the left navigation pane, select *Clusters*, and then select the name of the cluster that you want to configure the Amazon VPC CNI add-on for. .. Choose the *Add-ons* tab. .. Select the box in the top right of the add-on box and then choose *Edit*. -.. On the *Configure [.replaceable]`name of addon`* page: +.. On the *Configure [.replaceable]`Amazon VPC CNI`* page: + ... Select a `v1.14.0-eksbuild.3` or later version in the *Version* dropdown list. ... Expand the *Optional configuration settings*. @@ -182,11 +358,11 @@ Only the network policy logs are sent by the node agent. Other logs made by the + The following screenshot shows an example of this scenario. -+ + image::images/console-cni-config-network-policy-logs-cwl.png[{aws-management-console} showing the VPC CNI add-on with network policy and CloudWatch Logs in the optional configuration.,scaledwidth=80%] -{aws} CLI:: +*{aws} CLI*:: .. Run the following {aws} CLI command. Replace `my-cluster` with the name of your cluster and replace the IAM role ARN with the role that you are using. + [source,shell,subs="verbatim,attributes"] @@ -197,12 +373,12 @@ aws eks update-addon --cluster-name my-cluster --addon-name vpc-cni --addon-vers ---- -[[cni-network-policy-cwl-agent-selfmanaged,cni-network-policy-cwl-agent-selfmanaged.title]] -==== Self-managed add-on +[#cni-network-policy-cwl-agent-selfmanaged] +===== Self-managed add-on -Helm:: +*Helm*:: -If you have installed the [.noloc]`Amazon VPC CNI plugin for Kubernetes` through `helm`, you can update the configuration to send network policy logs to CloudWatch Logs. +If you have installed the Amazon VPC CNI plugin for Kubernetes through `helm`, you can update the configuration to send network policy logs to CloudWatch Logs. .. Run the following command to enable network policy logs and send them to CloudWatch Logs. + @@ -212,14 +388,14 @@ helm upgrade --set nodeAgent.enablePolicyEventLogs=true --set nodeAgent.enableCl ---- -[.noloc]`kubectl`:: +*kubectl*:: .. Open the `aws-node` `DaemonSet` in your editor. + [source,bash,subs="verbatim,attributes"] ---- kubectl edit daemonset -n kube-system aws-node ---- -.. Replace the `false` with `true` in two command arguments `--enable-policy-event-logs=false` and `--enable-cloudwatch-logs=false` in the `args:` in the `aws-network-policy-agent` container in the VPC CNI `aws-node` daemonset manifest. +.. Replace the `false` with `true` in two command arguments `--enable-policy-event-logs=false` and `--enable-cloudwatch-logs=false` in the `args:` in the `aws-network-policy-agent` container in the VPC CNI `aws-node` `DaemonSet` manifest. + [source,yaml,subs="verbatim,attributes"] ---- @@ -229,10 +405,10 @@ kubectl edit daemonset -n kube-system aws-node ---- -[[network-policies-cwl-fluentbit,network-policies-cwl-fluentbit.title]] -=== Send network policy logs with a [.noloc]`Fluent Bit` daemonset +[#network-policies-cwl-fluentbit] +==== Send network policy logs with a Fluent Bit `DaemonSet` -If you are using [.noloc]`Fluent Bit` in a daemonset to send logs from your nodes, you can add configuration to include the network policy logs from network policies. You can use the following example configuration: +If you are using Fluent Bit in a `DaemonSet` to send logs from your nodes, you can add configuration to include the network policy logs from network policies. You can use the following example configuration: [source,toml,subs="verbatim,attributes"] ---- @@ -248,10 +424,10 @@ If you are using [.noloc]`Fluent Bit` in a daemonset to send logs from your node ---- -[[network-policies-ebpf-sdk,network-policies-ebpf-sdk.title]] -== Included [.noloc]`eBPF` SDK +[#network-policies-ebpf-sdk] +== Included eBPF SDK -The [.noloc]`Amazon VPC CNI plugin for Kubernetes` installs [.noloc]`eBPF` SDK collection of tools on the nodes. You can use the [.noloc]`eBPF` SDK tools to identify issues with network policies. For example, the following command lists the programs that are running on the node. +The Amazon VPC CNI plugin for Kubernetes installs eBPF SDK collection of tools on the nodes. You can use the eBPF SDK tools to identify issues with network policies. For example, the following command lists the programs that are running on the node. [source,bash,subs="verbatim,attributes"] ---- @@ -259,3 +435,100 @@ sudo /opt/cni/bin/aws-eks-na-cli ebpf progs ---- To run this command, you can use any method to connect to the node. + +[#network-policies-troubleshooting-known-issues] +== Known issues and solutions + +The following sections describe known issues with the Amazon VPC CNI network policy feature and their solutions. + +[#network-policies-troubleshooting-policy-event-logs] +=== Network policy logs generated despite enable-policy-event-logs set to false + +*Issue*: EKS VPC CNI is generating network policy logs even when the `enable-policy-event-logs` setting is set to `false`. + +*Solution*: The `enable-policy-event-logs` setting only disables the policy "decision" logs, but it won't disable all Network Policy agent logging. This behavior is documented in the link:https://github.com/aws/aws-network-policy-agent/[aws-network-policy-agent README] on GitHub. To completely disable logging, you might need to adjust other logging configurations. + +[#network-policies-troubleshooting-map-cleanup] +=== Network policy map cleanup issues + +*Issue*: Problems with network `policyendpoint` still existing and not being cleaned up after pods are deleted. + +*Solution*: This issue was caused by a problem with the VPC CNI add-on version 1.19.3-eksbuild.1. Update to a newer version of the VPC CNI add-on to resolve this issue. + +[#network-policies-troubleshooting-policyendpoint] +=== Network policies aren't applied + +*Issue*: Network policy feature is enabled in the Amazon VPC CNI plugin, but network policies are not being applied correctly. + +If you make a network policy `kind: NetworkPolicy` and it doesn't effect the pod, check that the policyendpoint object was created in the same namespace as the pod. +If there aren't `policyendpoint` objects in the namespaces, the network policy controller (part of the EKS cluster) was unable to create network policy rules for the network policy agent (part of the VPC CNI) to apply. + +*Solution*: The solution is to fix the permissions of the VPC CNI (`ClusterRole` : `aws-node`) and the network policy controller (`ClusterRole` : `eks:network-policy-controller`) and to allow these actions in any policy enforcement tool such as Kyverno. Ensure that Kyverno policies are not blocking the creation of `policyendpoint` objects. See previous section for the permissions necessary permissions in <>. + +[#network-policies-troubleshooting-strict-mode-fallback] +=== Pods don't return to default deny state after policy deletion in strict mode + +*Issue*: When network policies are enabled in strict mode, pods start with a default deny policy. After policies are applied, traffic is allowed to the specified endpoints. However, when policies are deleted, the pod doesn't return to the default deny state and instead goes to a default allow state. + +*Solution*: This issue was fixed in the VPC CNI release 1.19.3, which included the network policy agent 1.2.0 release. After the fix, with strict mode enabled, once policies are removed, the pod will fall back to the default deny state as expected. + +[#network-policies-troubleshooting-sgfp-latency] +=== Security Groups for Pods startup latency + +*Issue*: When using the Security Groups for Pods feature in EKS, there is increased pod startup latency. + +*Solution*: The latency is due to rate limiting in the resource controller from API throttling on the `CreateNetworkInterface` API, which the VPC resource controller uses to create branch ENIs for the pods. Check your account's API limits for this operation and consider requesting a limit increase if needed. + +[#network-policies-troubleshooting-insufficient-pod-eni] +=== FailedScheduling due to insufficient vpc.amazonaws.com/pod-eni + +*Issue*: Pods fail to schedule with the error: `FailedScheduling 2m53s (x28 over 137m) default-scheduler 0/5 nodes are available: 5 Insufficient vpc.amazonaws.com/pod-eni. preemption: 0/5 nodes are available: 5 No preemption victims found for incoming pod.` + +*Solution*: As with the previous issue, assigning Security Groups to pods increases pod scheduling latency and it can increase beyond the CNI threshold for time to add each ENI, causing failures to start pods. This is expected behavior when using Security Groups for Pods. Consider the scheduling implications when designing your workload architecture. + +[#network-policies-troubleshooting-systemd-udev] +=== IPAM connectivity issues and segmentation faults + +*Issue*: Multiple errors occur including IPAM connectivity issues, throttling requests, and segmentation faults: + +* `Checking for IPAM connectivity ...` +* `Throttling request took 1.047064274s` +* `Retrying waiting for IPAM-D` +* `panic: runtime error: invalid memory address or nil pointer dereference` + +*Solution*: This issue occurs if you install `systemd-udev` on AL2023, as the file is re-written with a breaking policy. This can happen when updating to a different `releasever` that has an updated package or manually updating the package itself. Avoid installing or updating `systemd-udev` on AL2023 nodes. + +[#network-policies-troubleshooting-device-not-found] +=== Failed to find device by name error + +*Issue*: Error message: `{"level":"error","ts":"2025-02-05T20:27:18.669Z","caller":"ebpf/bpf_client.go:578","msg":"failed to find device by name eni9ea69618bf0: %!w(netlink.LinkNotFoundError={0xc000115310})"}` + +*Solution*: This issue has been identified and fixed in the latest versions of the Amazon VPC CNI network policy agent (v1.2.0). Update to the latest version of the VPC CNI to resolve this issue. + +[#network-policies-troubleshooting-cve-multus] +=== CVE vulnerabilities in Multus CNI image + +*Issue*: Enhanced EKS ImageScan CVE Report identifies vulnerabilities in the Multus CNI image version v4.1.4-eksbuild.2_thick. + +*Solution*: Update to the new version of the Multus CNI image and the new Network Policy Controller image, which have no vulnerabilities. The scanner can be updated to address the vulnerabilities found in the previous version. + +[#network-policies-troubleshooting-flow-info-deny] +=== Flow Info DENY verdicts in logs + +*Issue*: Network policy logs show DENY verdicts: `{"level":"info","ts":"2024-11-25T13:34:24.808Z","logger":"ebpf-client","caller":"events/events.go:193","msg":"Flow Info: ","Src IP":"","Src Port":9096,"Dest IP":"","Dest Port":56830,"Proto":"TCP","Verdict":"DENY"}` + +*Solution*: This issue has been resolved in the new version of the Network Policy Controller. Update to the latest EKS platform version to resolve logging issues. + +[#network-policies-troubleshooting-calico-migration] +=== Pod-to-pod communication issues after migrating from Calico + +*Issue*: After upgrading an EKS cluster to version 1.30 and switching from Calico to Amazon VPC CNI for network policy, pod-to-pod communication fails when network policies are applied. Communication is restored when network policies are deleted. + +*Solution*: The network policy agent in the VPC CNI can't have as many ports specified as Calico does. Instead, use port ranges in the network policies. The maximum number of unique combinations of ports for each protocol in each `ingress:` or `egress:` selector in a network policy is 24. Use port ranges to reduce the number of unique ports and avoid this limitation. + +[#network-policies-troubleshooting-standalone-pods] +=== Network policy agent doesn't support standalone pods + +*Issue*: Network policies applied to standalone pods may have inconsistent behavior. + +*Solution*: The Network Policy agent currently only supports pods that are deployed as part of a deployment/replicaset. If network policies are applied to standalone pods, there might be some inconsistencies in the behavior. This is documented at the top of this page, in the <>, and in the link:https://github.com/aws/aws-network-policy-agent/issues/327[aws-network-policy-agent GitHub issue #327] on GitHub. Deploy pods as part of a deployment or replicaset for consistent network policy behavior. diff --git a/latest/ug/networking/network-policy-disable.adoc b/latest/ug/networking/network-policy-disable.adoc new file mode 100644 index 000000000..36fb68468 --- /dev/null +++ b/latest/ug/networking/network-policy-disable.adoc @@ -0,0 +1,39 @@ +include::../attributes.txt[] + +[.topic] +[#network-policy-disable] += Disable Kubernetes network policies for Amazon EKS Pod network traffic +:info_titleabbrev: Disable + +[abstract] +-- +Learn how to disable Kubernetes network policies for Amazon EKS Pod network traffic. +-- + +Disable Kubernetes network policies to stop restricting Amazon EKS Pod network traffic + +. List all Kubernetes network policies. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get netpol -A +---- +. Delete each Kubernetes network policy. You must delete all network policies before disabling network policies. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl delete netpol +---- +. Open the aws-node DaemonSet in your editor. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl edit daemonset -n kube-system aws-node +---- +. Replace the `true` with `false` in the command argument `--enable-network-policy=true` in the `args:` in the `aws-network-policy-agent` container in the VPC CNI `aws-node` daemonset manifest. ++ +[source,yaml,subs="verbatim,attributes"] +---- + - args: + - --enable-network-policy=true +---- \ No newline at end of file diff --git a/latest/ug/networking/network-policy-stars-demo.adoc b/latest/ug/networking/network-policy-stars-demo.adoc index 95778b625..40188ce13 100644 --- a/latest/ug/networking/network-policy-stars-demo.adoc +++ b/latest/ug/networking/network-policy-stars-demo.adoc @@ -1,11 +1,10 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[network-policy-stars-demo,network-policy-stars-demo.title]] +[#network-policy-stars-demo] = Stars demo of network policy for Amazon EKS :info_titleabbrev: Stars policy demo -include::../attributes.txt[] - [abstract] -- This demo creates a front-end, back-end, and client service on your Amazon EKS cluster. The demo also creates a management graphical user interface that shows the available ingress and egress paths between each service. @@ -19,13 +18,13 @@ Before you create any network policies, all services can communicate bidirection + [source,bash,subs="verbatim,attributes"] ---- -kubectl apply -f https://eksworkshop.com/beginner/120_network-policies/calico/stars_policy_demo/create_resources.files/namespace.yaml -kubectl apply -f https://eksworkshop.com/beginner/120_network-policies/calico/stars_policy_demo/create_resources.files/management-ui.yaml -kubectl apply -f https://eksworkshop.com/beginner/120_network-policies/calico/stars_policy_demo/create_resources.files/backend.yaml -kubectl apply -f https://eksworkshop.com/beginner/120_network-policies/calico/stars_policy_demo/create_resources.files/frontend.yaml -kubectl apply -f https://eksworkshop.com/beginner/120_network-policies/calico/stars_policy_demo/create_resources.files/client.yaml +kubectl apply -f https://raw.githubusercontent.com/aws-samples/eks-workshop/2f9d29ed3f82ed6b083649e975a0e574fb8a4058/content/beginner/120_network-policies/calico/stars_policy_demo/create_resources.files/namespace.yaml +kubectl apply -f https://raw.githubusercontent.com/aws-samples/eks-workshop/2f9d29ed3f82ed6b083649e975a0e574fb8a4058/content/beginner/120_network-policies/calico/stars_policy_demo/create_resources.files/management-ui.yaml +kubectl apply -f https://raw.githubusercontent.com/aws-samples/eks-workshop/2f9d29ed3f82ed6b083649e975a0e574fb8a4058/content/beginner/120_network-policies/calico/stars_policy_demo/create_resources.files/backend.yaml +kubectl apply -f https://raw.githubusercontent.com/aws-samples/eks-workshop/2f9d29ed3f82ed6b083649e975a0e574fb8a4058/content/beginner/120_network-policies/calico/stars_policy_demo/create_resources.files/frontend.yaml +kubectl apply -f https://raw.githubusercontent.com/aws-samples/eks-workshop/2f9d29ed3f82ed6b083649e975a0e574fb8a4058/content/beginner/120_network-policies/calico/stars_policy_demo/create_resources.files/client.yaml ---- -. View all [.noloc]`Pods` on the cluster. +. View all Pods on the cluster. + [source,bash,subs="verbatim,attributes"] ---- @@ -53,7 +52,7 @@ stars frontend-cscnf 1/1 Running 0 ---- kubectl get service/management-ui -n management-ui ---- -. Open the a browser to the location from the previous step. You should see the management user interface. The *C* node is the client service, the *F* node is the front-end service, and the *B* node is the back-end service. Each node has full communication access to all other nodes, as indicated by the bold, colored lines. +. Open the a browser to the location from the previous step. You should see the management user interface. The *C* node is the client service, the *F* node is the front-end service, and the *B* node is the back-end service. Each node has full communication access to all other nodes, as indicated by the bold, colored lines. + image::images/stars-default.png[Open network policy,scaledwidth=100%] . Apply the following network policy in both the `stars` and `client` namespaces to isolate the services from each other: @@ -73,8 +72,8 @@ You can use the following commands to apply the policy to both namespaces: + [source,bash,subs="verbatim,attributes"] ---- -kubectl apply -n stars -f https://eksworkshop.com/beginner/120_network-policies/calico/stars_policy_demo/apply_network_policies.files/default-deny.yaml -kubectl apply -n client -f https://eksworkshop.com/beginner/120_network-policies/calico/stars_policy_demo/apply_network_policies.files/default-deny.yaml +kubectl apply -n stars -f https://raw.githubusercontent.com/aws-samples/eks-workshop/2f9d29ed3f82ed6b083649e975a0e574fb8a4058/content/beginner/120_network-policies/calico/stars_policy_demo/apply_network_policies.files/default-deny.yaml +kubectl apply -n client -f https://raw.githubusercontent.com/aws-samples/eks-workshop/2f9d29ed3f82ed6b083649e975a0e574fb8a4058/content/beginner/120_network-policies/calico/stars_policy_demo/apply_network_policies.files/default-deny.yaml ---- . Refresh your browser. You see that the management user interface can no longer reach any of the nodes, so they don't show up in the user interface. . Apply the following different network policies to allow the management user interface to access the services. Apply this policy to allow the UI: @@ -119,8 +118,8 @@ You can use the following commands to apply both policies: + [source,bash,subs="verbatim,attributes"] ---- -kubectl apply -f https://eksworkshop.com/beginner/120_network-policies/calico/stars_policy_demo/apply_network_policies.files/allow-ui.yaml -kubectl apply -f https://eksworkshop.com/beginner/120_network-policies/calico/stars_policy_demo/apply_network_policies.files/allow-ui-client.yaml +kubectl apply -f https://raw.githubusercontent.com/aws-samples/eks-workshop/2f9d29ed3f82ed6b083649e975a0e574fb8a4058/content/beginner/120_network-policies/calico/stars_policy_demo/apply_network_policies.files/allow-ui.yaml +kubectl apply -f https://raw.githubusercontent.com/aws-samples/eks-workshop/2f9d29ed3f82ed6b083649e975a0e574fb8a4058/content/beginner/120_network-policies/calico/stars_policy_demo/apply_network_policies.files/allow-ui-client.yaml ---- . Refresh your browser. You see that the management user interface can reach the nodes again, but the nodes cannot communicate with each other. + @@ -179,11 +178,11 @@ image::images/stars-final.png[Final network policy,scaledwidth=100%] + [source,bash,subs="verbatim,attributes"] ---- -kubectl delete -f https://eksworkshop.com/beginner/120_network-policies/calico/stars_policy_demo/create_resources.files/client.yaml -kubectl delete -f https://eksworkshop.com/beginner/120_network-policies/calico/stars_policy_demo/create_resources.files/frontend.yaml -kubectl delete -f https://eksworkshop.com/beginner/120_network-policies/calico/stars_policy_demo/create_resources.files/backend.yaml -kubectl delete -f https://eksworkshop.com/beginner/120_network-policies/calico/stars_policy_demo/create_resources.files/management-ui.yaml -kubectl delete -f https://eksworkshop.com/beginner/120_network-policies/calico/stars_policy_demo/create_resources.files/namespace.yaml +kubectl delete -f https://raw.githubusercontent.com/aws-samples/eks-workshop/2f9d29ed3f82ed6b083649e975a0e574fb8a4058/content/beginner/120_network-policies/calico/stars_policy_demo/create_resources.files/client.yaml +kubectl delete -f https://raw.githubusercontent.com/aws-samples/eks-workshop/2f9d29ed3f82ed6b083649e975a0e574fb8a4058/content/beginner/120_network-policies/calico/stars_policy_demo/create_resources.files/frontend.yaml +kubectl delete -f https://raw.githubusercontent.com/aws-samples/eks-workshop/2f9d29ed3f82ed6b083649e975a0e574fb8a4058/content/beginner/120_network-policies/calico/stars_policy_demo/create_resources.files/backend.yaml +kubectl delete -f https://raw.githubusercontent.com/aws-samples/eks-workshop/2f9d29ed3f82ed6b083649e975a0e574fb8a4058/content/beginner/120_network-policies/calico/stars_policy_demo/create_resources.files/management-ui.yaml +kubectl delete -f https://raw.githubusercontent.com/aws-samples/eks-workshop/2f9d29ed3f82ed6b083649e975a0e574fb8a4058/content/beginner/120_network-policies/calico/stars_policy_demo/create_resources.files/namespace.yaml ---- + Even after deleting the resources, there can still be network policy endpoints on the nodes that might interfere in unexpected ways with networking in your cluster. The only sure way to remove these rules is to reboot the nodes or terminate all of the nodes and recycle them. To terminate all nodes, either set the Auto Scaling Group desired count to 0, then back up to the desired number, or just terminate the nodes. diff --git a/latest/ug/networking/network-reqs.adoc b/latest/ug/networking/network-reqs.adoc index fdebdfaec..98870728a 100644 --- a/latest/ug/networking/network-reqs.adoc +++ b/latest/ug/networking/network-reqs.adoc @@ -1,32 +1,25 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[network-reqs,network-reqs.title]] +[#network-reqs] = View Amazon EKS networking requirements for VPC and subnets -:info_doctype: section -:info_title: View Amazon EKS networking requirements for VPC and subnets :info_titleabbrev: VPC and subnet requirements -:info_abstract: Learn how to configure the VPC and subnets to meet networking \ - requirements for creating Amazon EKS clusters with sufficient IP addresses, subnet \ - types, and availability zones. Understand IP family usage by component and shared \ - subnet considerations. - -include::../attributes.txt[] [abstract] -- Learn how to configure the VPC and subnets to meet networking requirements for creating Amazon EKS clusters with sufficient IP addresses, subnet types, and availability zones. Understand IP family usage by component and shared subnet considerations. -- -When you create a cluster, you specify a link:vpc/latest/userguide/configure-your-vpc.html[VPC,type="documentation"] and at least two subnets that are in different Availability Zones. This topic provides an overview of Amazon EKS specific requirements and considerations for the VPC and subnets that you use with your cluster. If you don't have a VPC to use with Amazon EKS, see <>. If you're creating a local or extended cluster on {aws} Outposts, see <> instead of this topic. The content in this topic applies for Amazon EKS clusters with hybrid nodes. For additional networking requirements for hybrid nodes, see <>. +When you create a cluster, you specify a link:vpc/latest/userguide/configure-your-vpc.html[VPC,type="documentation"] and at least two subnets that are in different Availability Zones. This topic provides an overview of Amazon EKS specific requirements and considerations for the VPC and subnets that you use with your cluster. If you don't have a VPC to use with Amazon EKS, see <>. If you're creating a local or extended cluster on {aws} Outposts, see <> instead of this topic. The content in this topic applies for Amazon EKS clusters with hybrid nodes. For additional networking requirements for hybrid nodes, see <>. -[[network-requirements-vpc,network-requirements-vpc.title]] +[#network-requirements-vpc] == VPC requirements and considerations When you create a cluster, the VPC that you specify must meet the following requirements and considerations: -* The VPC must have a sufficient number of IP addresses available for the cluster, any nodes, and other [.noloc]`Kubernetes` resources that you want to create. If the VPC that you want to use doesn't have a sufficient number of IP addresses, try to increase the number of available IP addresses. +* The VPC must have a sufficient number of IP addresses available for the cluster, any nodes, and other Kubernetes resources that you want to create. If the VPC that you want to use doesn't have a sufficient number of IP addresses, try to increase the number of available IP addresses. + You can do this by updating the cluster configuration to change which subnets and security groups the cluster uses. You can update from the {aws-management-console}, the latest version of the {aws} CLI, {aws} CloudFormation, and `eksctl` version `v0.164.0-rc.0` or later. You might need to do this to provide subnets with more available IP addresses to successfully upgrade a cluster version. + @@ -39,35 +32,40 @@ For example, assume that you made a cluster and specified four subnets. In the o ==== + -If you need more IP addresses than the CIDR blocks in the VPC have, you can add additional CIDR blocks by link:vpc/latest/userguide/working-with-vpcs.html#add-ipv4-cidr[associating additional Classless Inter-Domain Routing (CIDR) blocks,type="documentation"] with your VPC. You can associate private ([.noloc]`RFC 1918`) and public (non-[.noloc]`RFC 1918`) CIDR blocks to your VPC either before or after you create your cluster. It can take a cluster up to five hours for a CIDR block that you associated with a VPC to be recognized. +If you need more IP addresses than the CIDR blocks in the VPC have, you can add additional CIDR blocks by link:vpc/latest/userguide/working-with-vpcs.html#add-ipv4-cidr[associating additional Classless Inter-Domain Routing (CIDR) blocks,type="documentation"] with your VPC. You can associate private (RFC 1918) and public (non-RFC 1918) CIDR blocks to your VPC either before or after you create your cluster. + -You can conserve IP address utilization by using a transit gateway with a shared services VPC. For more information, see link:vpc/latest/tgw/transit-gateway-isolated-shared.html[Isolated VPCs with shared services,type="documentation"] and link:containers/eks-vpc-routable-ip-address-conservation[Amazon EKS VPC routable IP address conservation patterns in a hybrid network,type="blog"]. -* If you want [.noloc]`Kubernetes` to assign `IPv6` addresses to [.noloc]`Pods` and services, associate an `IPv6` CIDR block with your VPC. For more information, see link:vpc/latest/userguide/working-with-vpcs.html#vpc-associate-ipv6-cidr[Associate an IPv6 CIDR block with your VPC,type="documentation"] in the Amazon VPC User Guide. You cannot use `IPv6` addresses with Pods and services running on hybrid nodes and you cannot use hybrid nodes with clusters configured with the `IPv6` IP address family. +You can add nodes that use the new CIDR block immediately after you add it. However, because the control plane recognizes the new CIDR block only after the reconciliation is complete, it can take a cluster up to one hour for a CIDR block that you associated with a VPC to be recognized. Then you can run the `kubectl attach`, `kubectl cp`, `kubectl exec`, `kubectl logs`, and `kubectl port-forward` commands (these commands use the `kubelet API`) for nodes and pods in the new CIDR block. Also, if you have Pods that operate as a webhook backend, then you must wait for the control plane reconciliation to complete. + +* Avoid IP address range overlaps when you connect your EKS cluster to other VPCs through Transit Gateway, VPC peering, or other networking configurations. CIDR conflicts occur when your EKS cluster's service CIDR overlaps with the CIDR of a connected VPC. In these scenarios, ServiceIP addresses take priority over resources in connected VPCs with the same IP address, although traffic routing can become unpredictable and applications may fail to connect to intended resources. ++ +To avoid CIDR conflicts, ensure your EKS service CIDR doesn't overlap with any connected VPC CIDRs and maintain a centralized record of all CIDR assignments. If you encounter CIDR overlaps, you can use a transit gateway with a shared services VPC. For more information, see link:vpc/latest/tgw/transit-gateway-isolated-shared.html[Isolated VPCs with shared services,type="documentation"] and link:containers/eks-vpc-routable-ip-address-conservation[Amazon EKS VPC routable IP address conservation patterns in a hybrid network,type="blog"]. Also, refer to the Communication across VPCs section of the link:eks/latest/best-practices/subnets.html[VPC and Subnet Considerations,type="documentation"] page in the EKS Best Practices Guide. + +* If you want Kubernetes to assign `IPv6` addresses to Pods and services, associate an `IPv6` CIDR block with your VPC. For more information, see link:vpc/latest/userguide/working-with-vpcs.html#vpc-associate-ipv6-cidr[Associate an IPv6 CIDR block with your VPC,type="documentation"] in the Amazon VPC User Guide. You cannot use `IPv6` addresses with Pods and services running on hybrid nodes and you cannot use hybrid nodes with clusters configured with the `IPv6` IP address family. * The VPC must have `DNS` hostname and `DNS` resolution support. Otherwise, nodes can't register to your cluster. For more information, see link:vpc/latest/userguide/vpc-dns.html[DNS attributes for your VPC,type="documentation"] in the Amazon VPC User Guide. * The VPC might require VPC endpoints using {aws} PrivateLink. For more information, see <>. -If you created a cluster with [.noloc]`Kubernetes` `1.14` or earlier, Amazon EKS added the following tag to your VPC: +If you created a cluster with Kubernetes `1.14` or earlier, Amazon EKS added the following tag to your VPC: -[cols="1,1", options="header"] +[%header,cols="2"] |=== |Key |Value -|``kubernetes.io/cluster/[.replaceable]`my-cluster``` +|`kubernetes.io/cluster/[.replaceable]``my-cluster``` |`owned` |=== This tag was only used by Amazon EKS. You can remove the tag without impacting your services. It's not used with clusters that are version `1.15` or later. -[[network-requirements-subnets,network-requirements-subnets.title]] +[#network-requirements-subnets] == Subnet requirements and considerations -When you create a cluster, Amazon EKS creates 2–4 link:AWSEC2/latest/UserGuide/using-eni.html[elastic network interfaces,type="documentation"] in the subnets that you specify. These network interfaces enable communication between your cluster and your VPC. These network interfaces also enable [.noloc]`Kubernetes` features such as `kubectl exec` and `kubectl logs`. Each Amazon EKS created network interface has the text `Amazon EKS [.replaceable]``cluster-name``` in its description. +When you create a cluster, Amazon EKS creates 2–4 link:AWSEC2/latest/UserGuide/using-eni.html[elastic network interfaces,type="documentation"] in the subnets that you specify. These network interfaces enable communication between your cluster and your VPC. These network interfaces also enable Kubernetes features that use the `kubelet API`. The connections to the `kubelet` API are used in the `kubectl attach`, `kubectl cp`, `kubectl exec`, `kubectl logs`, and `kubectl port-forward` commands. Each Amazon EKS created network interface has the text `Amazon EKS [.replaceable]``cluster-name``` in its description. -Amazon EKS can create its network interfaces in any subnet that you specify when you create a cluster. You can change which subnets Amazon EKS creates its network interfaces in after your cluster is created. When you update the [.noloc]`Kubernetes` version of a cluster, Amazon EKS deletes the original network interfaces that it created, and creates new network interfaces. These network interfaces might be created in the same subnets as the original network interfaces or in different subnets than the original network interfaces. To control which subnets network interfaces are created in, you can limit the number of subnets you specify to only two when you create a cluster or update the subnets after creating the cluster. +Amazon EKS can create its network interfaces in any subnet that you specify when you create a cluster. You can change which subnets Amazon EKS creates its network interfaces in after your cluster is created. When you update the Kubernetes version of a cluster, Amazon EKS deletes the original network interfaces that it created, and creates new network interfaces. These network interfaces might be created in the same subnets as the original network interfaces or in different subnets than the original network interfaces. To control which subnets network interfaces are created in, you can limit the number of subnets you specify to only two when you create a cluster or update the subnets after creating the cluster. -[[cluster-subnets,cluster-subnets.title]] +[#cluster-subnets] === Subnet requirements for clusters The link:vpc/latest/userguide/configure-subnets.html#subnet-types[subnets,type="documentation"] that you specify when you create or update a cluster must meet the following requirements: @@ -76,11 +74,11 @@ The link:vpc/latest/userguide/configure-subnets.html#subnet-types[subnets,type=" * The subnets must each have at least six IP addresses for use by Amazon EKS. However, we recommend at least 16 IP addresses. * The subnets must be in at least two different Availability Zones. -* The subnets can't reside in {aws} Outposts or {aws} Wavelength. However, if you have them in your VPC, you can deploy self-managed nodes and [.noloc]`Kubernetes` resources to these types of subnets. For more information about self-managed nodes, see <>. -* The subnets can be a public or private. However, we recommend that you specify private subnets, if possible. A public subnet is a subnet with a route table that includes a route to an link:vpc/latest/userguide/VPC_Internet_Gateway.html[internet gateway,type="documentation"], whereas a private subnet is a subnet with a route table that doesn't include a route to an internet gateway. +* The subnets can't reside in {aws} Outposts or {aws} Wavelength. However, if you have them in your VPC, you can deploy self-managed nodes and Kubernetes resources to these types of subnets. For more information about self-managed nodes, see <>. +* The subnets can be a public or private. However, we recommend that you specify private subnets, if possible. A public subnet is a subnet with a route table that includes a route to an link:vpc/latest/userguide/VPC_Internet_Gateway.html[internet gateway,type="documentation"], whereas a private subnet is a subnet with a route table that doesn't include a route to an internet gateway. * The subnets can't reside in the following Availability Zones: + -[cols="1,1,1", options="header"] +[%header,cols="3"] |=== |{aws} Region |Region name @@ -104,11 +102,11 @@ The link:vpc/latest/userguide/configure-subnets.html#subnet-types[subnets,type=" [[network-requirements-ip-table]] === IP address family usage by component -The following table contains the IP address family used by each component of Amazon EKS. You can use a network address translation (NAT) or other compatibility system to connect to these components from source IP addresses in families with the [.noloc]`"No"` value for a table entry. +The following table contains the IP address family used by each component of Amazon EKS. You can use a network address translation (NAT) or other compatibility system to connect to these components from source IP addresses in families with the "No" value for a table entry. -Functionality can differ depending on the [.noloc]`IP family` (`ipFamily`) setting of the cluster. This setting changes the type of IP addresses used for the [.noloc]`CIDR` block that [.noloc]`Kubernetes` assigns to [.noloc]`Services`. A cluster with the setting value of [.noloc]`IPv4` is referred to as an _IPv4 cluster_, and a cluster with the setting value of [.noloc]`IPv6` is referred to as an _IPv6 cluster_. +Functionality can differ depending on the IP family (`ipFamily`) setting of the cluster. This setting changes the type of IP addresses used for the CIDR block that Kubernetes assigns to Services. A cluster with the setting value of IPv4 is referred to as an _IPv4 cluster_, and a cluster with the setting value of IPv6 is referred to as an _IPv6 cluster_. -[cols="1,1,1,1", options="header"] +[%header,cols="4"] |=== |Component |IPv4 addresses @@ -136,27 +134,27 @@ Functionality can differ depending on the [.noloc]`IP family` (`ipFamily`) setti |Yes^1^ |Yes^1^ -|`IPv4` [.noloc]`Kubernetes` cluster public endpoint^2^ +|`IPv4` Kubernetes cluster public endpoint^2^ |Yes |No |No -|`IPv4` [.noloc]`Kubernetes` cluster private endpoint^2^ +|`IPv4` Kubernetes cluster private endpoint^2^ |Yes |No |No -|`IPv6` [.noloc]`Kubernetes` cluster public endpoint^2^ +|`IPv6` Kubernetes cluster public endpoint^2^ |Yes^1,4^ |Yes^1,4^ |Yes^4^ -|`IPv6` [.noloc]`Kubernetes` cluster private endpoint^2^ +|`IPv6` Kubernetes cluster private endpoint^2^ |Yes^1,4^ |Yes^1,4^ |Yes^4^ -|[.noloc]`Kubernetes` cluster subnets +|Kubernetes cluster subnets |Yes^2^ |No |Yes^2^ @@ -166,17 +164,17 @@ Functionality can differ depending on the [.noloc]`IP family` (`ipFamily`) setti |No |Yes^2^ -|Cluster [.noloc]`CIDR` range for [.noloc]`Service` IP addresses +|Cluster CIDR range for Service IP addresses |Yes^2^ |Yes^2^ |No -|[.noloc]`Pod` IP addresses from the VPC CNI +|Pod IP addresses from the VPC CNI |Yes^2^ |Yes^2^ |No -|IRSA [.noloc]`OIDC` Issuer URLs +|IRSA OIDC Issuer URLs |Yes^1,3^ |Yes^1,3^ |Yes^1,3^ @@ -187,14 +185,14 @@ Functionality can differ depending on the [.noloc]`IP family` (`ipFamily`) setti ^1^ The endpoint is dual stack with both `IPv4` and `IPv6` addresses. Your applications outside of {aws}, your nodes for the cluster, and your pods inside the cluster can reach this endpoint by either `IPv4` or `IPv6`. -^2^ You choose between an `IPv4` cluster and `IPv6` cluster in the [.noloc]`IP family` (`ipFamily`) setting of the cluster when you create a cluster and this can't be changed. Instead, you must choose a different setting when you create another cluster and migrate your workloads. +^2^ You choose between an `IPv4` cluster and `IPv6` cluster in the IP family (`ipFamily`) setting of the cluster when you create a cluster and this can't be changed. Instead, you must choose a different setting when you create another cluster and migrate your workloads. ^3^ The dual-stack endpoint was introduced in August 2024. To use the dual-stack endpoints with the {aws} CLI, see the link:sdkref/latest/guide/feature-endpoints.html[Dual-stack and FIPS endpoints,type="documentation"] configuration in the _{aws} SDKs and Tools Reference Guide_. The following lists the new endpoints: *EKS API public endpoint*:: `eks.[.replaceable]``region``.api.aws` -*IRSA [.noloc]`OIDC` Issuer URLs*:: +*IRSA OIDC Issuer URLs*:: `oidc-eks.[.replaceable]``region``.api.aws` ^4^ The dual-stack cluster endpoint was introduced in October 2024. EKS creates the following endpoint for new clusters that are made after this date and that select `IPv6` in the IP family (ipFamily) setting of the cluster: @@ -204,23 +202,23 @@ Functionality can differ depending on the [.noloc]`IP family` (`ipFamily`) setti ==== -[[node-subnet-reqs,node-subnet-reqs.title]] +[#node-subnet-reqs] === Subnet requirements for nodes -You can deploy nodes and [.noloc]`Kubernetes` resources to the same subnets that you specify when you create your cluster. However, this isn't necessary. This is because you can also deploy nodes and [.noloc]`Kubernetes` resources to subnets that you didn't specify when you created the cluster. If you deploy nodes to different subnets, Amazon EKS doesn't create cluster network interfaces in those subnets. Any subnet that you deploy nodes and [.noloc]`Kubernetes` resources to must meet the following requirements: +You can deploy nodes and Kubernetes resources to the same subnets that you specify when you create your cluster. However, this isn't necessary. This is because you can also deploy nodes and Kubernetes resources to subnets that you didn't specify when you created the cluster. If you deploy nodes to different subnets, Amazon EKS doesn't create cluster network interfaces in those subnets. Any subnet that you deploy nodes and Kubernetes resources to must meet the following requirements: -* The subnets must have enough available IP addresses to deploy all of your nodes and [.noloc]`Kubernetes` resources to. -* If you want [.noloc]`Kubernetes` to assign `IPv6` addresses to [.noloc]`Pods` and services, then you must have one `IPv6` CIDR block and one `IPv4` CIDR block that are associated with your subnet. For more information, see link:vpc/latest/userguide/working-with-subnets.html#subnet-associate-ipv6-cidr[Associate an IPv6 CIDR block with your subnet,type="documentation"] in the Amazon VPC User Guide. The route tables that are associated with the subnets must include routes to `IPv4` and `IPv6` addresses. For more information, see link:vpc/latest/userguide/VPC_Route_Tables.html#route-table-routes[Routes,type="documentation"] in the Amazon VPC User Guide. Pods are assigned only an `IPv6` address. However the network interfaces that Amazon EKS creates for your cluster and your nodes are assigned an `IPv4` and an `IPv6` address. -* If you need inbound access from the internet to your [.noloc]`Pods`, make sure to have at least one public subnet with enough available IP addresses to deploy load balancers and ingresses to. You can deploy load balancers to public subnets. Load balancers can load balance to [.noloc]`Pods` in private or public subnets. We recommend deploying your nodes to private subnets, if possible. -* If you plan to deploy nodes to a public subnet, the subnet must auto-assign `IPv4` public addresses or `IPv6` addresses. If you deploy nodes to a private subnet that has an associated `IPv6` CIDR block, the private subnet must also auto-assign `IPv6` addresses. If you used the {aws} CloudFormation template provided by Amazon EKS to deploy your VPC after March 26, 2020, this setting is enabled. If you used the templates to deploy your VPC before this date or you use your own VPC, you must enable this setting manually. For the template, see <>. For more information, see link:vpc/latest/userguide/working-with-subnets.html#subnet-public-ip[Modify the public IPv4 addressing attribute for your subnet,type="documentation"] and link:vpc/latest/userguide/working-with-subnets.html#subnet-ipv6[Modify the IPv6 addressing attribute for your subnet,type="documentation"] in the link:vpc/latest/userguide/[Amazon VPC User Guide,type="documentation"]. -* If the subnet that you deploy a node to is a private subnet and its route table doesn't include a route to a network address translation link:vpc/latest/userguide/vpc-nat.html[(NAT) device,type="documentation"] (`IPv4`) or an link:vpc/latest/userguide/egress-only-internet-gateway.html[egress-only gateway,type="documentation"] (`IPv6`), add VPC endpoints using {aws} PrivateLink to your VPC. VPC endpoints are needed for all the {aws} services that your nodes and [.noloc]`Pods` need to communicate with. Examples include Amazon ECR, Elastic Load Balancing, Amazon CloudWatch, {aws} Security Token Service, and Amazon Simple Storage Service (Amazon S3). The endpoint must include the subnet that the nodes are in. Not all {aws} services support VPC endpoints. For more information, see link:vpc/latest/privatelink/what-is-privatelink.html[What is {aws} PrivateLink?,type="documentation"] and link:vpc/latest/privatelink/aws-services-privatelink-support.html[{aws} services that integrate with {aws} PrivateLink,type="documentation"]. For a list of more Amazon EKS requirements, see <>. +* The subnets must have enough available IP addresses to deploy all of your nodes and Kubernetes resources to. +* If you want Kubernetes to assign `IPv6` addresses to Pods and services, then you must have one `IPv6` CIDR block and one `IPv4` CIDR block that are associated with your subnet. For more information, see link:vpc/latest/userguide/working-with-subnets.html#subnet-associate-ipv6-cidr[Associate an IPv6 CIDR block with your subnet,type="documentation"] in the Amazon VPC User Guide. The route tables that are associated with the subnets must include routes to `IPv4` and `IPv6` addresses. For more information, see link:vpc/latest/userguide/VPC_Route_Tables.html#route-table-routes[Routes,type="documentation"] in the Amazon VPC User Guide. Pods are assigned only an `IPv6` address. However the network interfaces that Amazon EKS creates for your cluster and your nodes are assigned an `IPv4` and an `IPv6` address. +* If you need inbound access from the internet to your Pods, make sure to have at least one public subnet with enough available IP addresses to deploy load balancers and ingresses to. You can deploy load balancers to public subnets. Load balancers can load balance to Pods in private or public subnets. We recommend deploying your nodes to private subnets, if possible. +* If you plan to deploy nodes to a public subnet, the subnet must auto-assign `IPv4` public addresses or `IPv6` addresses. If you deploy nodes to a private subnet that has an associated `IPv6` CIDR block, the private subnet must also auto-assign `IPv6` addresses. If you used the {aws} CloudFormation template provided by Amazon EKS to deploy your VPC after March 26, 2020, this setting is enabled. If you used the templates to deploy your VPC before this date or you use your own VPC, you must enable this setting manually. For the template, see <>. For more information, see link:vpc/latest/userguide/working-with-subnets.html#subnet-public-ip[Modify the public IPv4 addressing attribute for your subnet,type="documentation"] and link:vpc/latest/userguide/working-with-subnets.html#subnet-ipv6[Modify the IPv6 addressing attribute for your subnet,type="documentation"] in the link:vpc/latest/userguide/[Amazon VPC User Guide,type="documentation"]. +* If the subnet that you deploy a node to is a private subnet and its route table doesn't include a route to a network address translation link:vpc/latest/userguide/vpc-nat.html[(NAT) device,type="documentation"] (`IPv4`) or an link:vpc/latest/userguide/egress-only-internet-gateway.html[egress-only gateway,type="documentation"] (`IPv6`), add VPC endpoints using {aws} PrivateLink to your VPC. VPC endpoints are needed for all the {aws} services that your nodes and Pods need to communicate with. Examples include Amazon ECR, Elastic Load Balancing, Amazon CloudWatch, {aws} Security Token Service, and Amazon Simple Storage Service (Amazon S3). The endpoint must include the subnet that the nodes are in. Not all {aws} services support VPC endpoints. For more information, see link:vpc/latest/privatelink/what-is-privatelink.html[What is {aws} PrivateLink?,type="documentation"] and link:vpc/latest/privatelink/aws-services-privatelink-support.html[{aws} services that integrate with {aws} PrivateLink,type="documentation"]. For a list of more Amazon EKS requirements, see <>. * If you want to deploy load balancers to a subnet, the subnet must have the following tag: + ** Private subnets + -[cols="1,1", options="header"] +[%header,cols="2"] |=== |Key |Value @@ -231,7 +229,7 @@ You can deploy nodes and [.noloc]`Kubernetes` resources to the same subnets that |=== ** Public subnets + -[cols="1,1", options="header"] +[%header,cols="2"] |=== |Key |Value @@ -241,19 +239,19 @@ You can deploy nodes and [.noloc]`Kubernetes` resources to the same subnets that |`1` |=== -When a [.noloc]`Kubernetes` cluster that's version `1.18` and earlier was created, Amazon EKS added the following tag to all of the subnets that were specified. +When a Kubernetes cluster that's version `1.18` and earlier was created, Amazon EKS added the following tag to all of the subnets that were specified. -[cols="1,1", options="header"] +[%header,cols="2"] |=== |Key |Value -|``kubernetes.io/cluster/[.replaceable]`my-cluster``` +|`kubernetes.io/cluster/[.replaceable]``my-cluster``` |`shared` |=== -When you create a new [.noloc]`Kubernetes` cluster now, Amazon EKS doesn't add the tag to your subnets. If the tag was on subnets that were used by a cluster that was previously a version earlier than `1.19`, the tag wasn't automatically removed from the subnets when the cluster was updated to a newer version. Version `2.1.1` or earlier of the {aws} Load Balancer Controller requires this tag. If you are using a newer version of the Load Balancer Controller, you can remove the tag without interrupting your services. For more information about the controller, see <>. +When you create a new Kubernetes cluster now, Amazon EKS doesn't add the tag to your subnets. If the tag was on subnets that were used by a cluster that was previously a version earlier than `1.19`, the tag wasn't automatically removed from the subnets when the cluster was updated to a newer version. Version `2.1.1` or earlier of the {aws} Load Balancer Controller requires this tag. If you are using a newer version of the Load Balancer Controller, you can remove the tag without interrupting your services. For more information about the controller, see <>. If you deployed a VPC by using `eksctl` or any of the Amazon EKS {aws} CloudFormation VPC templates, the following applies: @@ -266,14 +264,14 @@ This change impacts new node groups that are deployed to public subnets in the f -* *link:eks/latest/userguide/create-managed-node-group.html[Managed node groups,type="documentation"]* – If the node group is deployed to a public subnet on or after April 22, 2020, automatic assignment of public IP addresses must be enabled for the public subnet. For more information, see link:vpc/latest/userguide/vpc-ip-addressing.html#subnet-public-ip[Modifying the public IPv4 addressing attribute for your subnet,type="documentation"]. -* *link:eks/latest/userguide/launch-workers.html[Linux,type="documentation"], link:eks/latest/userguide/launch-windows-workers.html[Windows,type="documentation"], or link:eks/latest/userguide/arm-ami.html[Arm,type="documentation"] self-managed node groups* – If the node group is deployed to a public subnet on or after March 26, 2020, automatic assignment of public IP addresses must be enabled for the public subnet. Otherwise, the nodes must be launched with a public IP address instead. For more information, see link:vpc/latest/userguide/vpc-ip-addressing.html#subnet-public-ip[Modifying the public IPv4 addressing attribute for your subnet,type="documentation"] or link:vpc/latest/userguide/vpc-ip-addressing.html#vpc-public-ip[Assigning a public IPv4 address during instance launch,type="documentation"]. +* *<>* – If the node group is deployed to a public subnet on or after April 22, 2020, automatic assignment of public IP addresses must be enabled for the public subnet. For more information, see link:vpc/latest/userguide/vpc-ip-addressing.html#subnet-public-ip[Modifying the public IPv4 addressing attribute for your subnet,type="documentation"]. +* *<>, <>, or <> self-managed node groups* – If the node group is deployed to a public subnet on or after March 26, 2020, automatic assignment of public IP addresses must be enabled for the public subnet. Otherwise, the nodes must be launched with a public IP address instead. For more information, see link:vpc/latest/userguide/vpc-ip-addressing.html#subnet-public-ip[Modifying the public IPv4 addressing attribute for your subnet,type="documentation"] or link:vpc/latest/userguide/vpc-ip-addressing.html#vpc-public-ip[Assigning a public IPv4 address during instance launch,type="documentation"]. -[[network-requirements-shared,network-requirements-shared.title]] +[#network-requirements-shared] == Shared subnet requirements and considerations -You can use _VPC sharing_ to share subnets with other {aws} accounts within the same {aws} Organizations. You can create Amazon EKS clusters in shared subnets, with the following considerations: +You can use _VPC sharing_ to share subnets with other {aws} accounts within the same {aws} Organizations. You can create Amazon EKS clusters in shared subnets, with the following considerations: @@ -284,7 +282,7 @@ You can use _VPC sharing_ to share subnets with other {aws} accounts within the + ** Cluster IAM role and Node IAM roles must be created in that account. For more information, see <> and <>. ** All nodes must be made by the same participant, including managed node groups. -* The shared VPC owner cannot view, update or delete a cluster that a participant creates in the shared subnet. This is in addition to the VPC resources that each account has different access to. For more information, see link:vpc/latest/userguide/vpc-sharing.html#vpc-share-limitations[Responsibilities and permissions for owners and participants,type="documentation"] in the _Amazon VPC User Guide_. -* If you use the _custom networking_ feature of the [.noloc]`Amazon VPC CNI plugin for Kubernetes`, you need to use the Availability Zone ID mappings listed in the owner account to create each `ENIConfig`. For more information, see <>. +* The shared VPC owner cannot view, update or delete a cluster that a participant creates in the shared subnet. This is in addition to the VPC resources that each account has different access to. For more information, see link:vpc/latest/userguide/vpc-sharing.html#vpc-share-limitations[Responsibilities and permissions for owners and participants,type="documentation"] in the _Amazon VPC User Guide_. +* If you use the _custom networking_ feature of the Amazon VPC CNI plugin for Kubernetes, you need to use the Availability Zone ID mappings listed in the owner account to create each `ENIConfig`. For more information, see <>. -For more information about VPC subnet sharing, see link:vpc/latest/userguide/vpc-sharing.html#vpc-share-limitations[Share your VPC with other accounts,type="documentation"] in the _Amazon VPC User Guide_. +For more information about VPC subnet sharing, see link:vpc/latest/userguide/vpc-sharing.html#vpc-share-limitations[Share your VPC with other accounts,type="documentation"] in the _Amazon VPC User Guide_. diff --git a/latest/ug/networking/pod-multiple-network-interfaces.adoc b/latest/ug/networking/pod-multiple-network-interfaces.adoc new file mode 100644 index 000000000..c9e9ddd6e --- /dev/null +++ b/latest/ug/networking/pod-multiple-network-interfaces.adoc @@ -0,0 +1,188 @@ +include::../attributes.txt[] + +[.topic] +[#pod-multiple-network-interfaces] += Attach multiple network interfaces to Pods +:info_titleabbrev: Multiple interfaces + +[abstract] +-- +Learn how to use VPC CNI to attach multiple network interfaces to a Pod in Amazon EKS for advanced networking scenarios with high bandwidth. You can configure a workload and the VPC CNI assigned IP addresses from every NIC on the EC2 instance to each pod. The application can make concurrent connections to use the bandwidth from each NIC. Every network interface is configured in the same subnet and security groups as the node. +-- + +By default, the Amazon VPC CNI plugin assigns one IP address to each pod. This IP address is attached to an _elastic network interface_ that handles all incoming and outgoing traffic for the pod. To increase the bandwidth and packet per second rate performance, you can use the _Multi-NIC feature_ of the VPC CNI to configure a multi-homed pod. A multi-homed pod is a single Kubernetes pod that uses multiple network interfaces (and multiple IP addresses). By running a multi-homed pod, you can spread its application traffic across multiple network interfaces by using concurrent connections. This is especially useful for Artificial Intelligence (AI), Machine Learning (ML), and High Performance Computing (HPC) use cases. + +The following diagram shows a multi-homed pod running on a worker node with multiple network interface cards (NICs) in use. + +image::images/multi-homed-pod.png[A multi-homed pod with two network interfaces attached one network interface with ENA and one network interface with ENA and EFA,scaledwidth=100%] + +[#pod-multi-nic-background] +== Background + +On Amazon EC2, an _elastic network interface_ is a logical networking component in a VPC that represents a virtual network card. For many EC2 instance types, the network interfaces share a single network interface card (NIC) in hardware. This single NIC has a maximum bandwidth and packet per second rate. + +If the multi-NIC feature is enabled, the VPC CNI doesn't assign IP addresses in bulk, which it does by default. Instead, the VPC CNI assigns one IP address to a network interface on each network card on-demand when a new pod starts. This behavior reduces the rate of IP address exhaustion, which is increased by using multi-homed pods. Because the VPC CNI is assigning IP address on-demand, pods might take longer to start on instances with the multi-NIC feature enabled. + +[#pod-multi-nic-considerations] +== Considerations + +* Ensure that your Kubernetes cluster is running VPC CNI version `1.20.0` and later. The multi-NIC feature is only available in version `1.20.0` of the VPC CNI or later. +* Enable the `ENABLE_MULTI_NIC` environment variable in the VPC CNI plugin. You can run the following command to set the variable and start a deployment of the DaemonSet. +** `kubectl set env daemonset aws-node -n kube-system ENABLE_MULTI_NIC=true` +* Ensure that you create worker nodes that have multiple network interface cards (NICs). For a list of EC2 instances that have multiple network interface cards, see link:AWSEC2/latest/UserGuide/using-eni.html#network-cards[Network cards,type="documentation"] in the *Amazon EC2 User Guide*. +* If the multi-NIC feature is enabled, the VPC CNI doesn't assign IP addresses in bulk, which it does by default. Because the VPC CNI is assigning IP address on-demand, pods might take longer to start on instances with the multi-NIC feature enabled. For more information, see the previous section <>. +* With the multi-NIC feature enabled, pods don't have multiple network interfaces by default. You must configure each workload to use multi-NIC. Add the `k8s.amazonaws.com/nicConfig: multi-nic-attachment` annotation to workloads that should have multiple network interfaces. + +[#pod-multi-nic-considerations-ipv6] +=== `IPv6` Considerations + +* *Custom IAM policy* - For `IPv6` clusters, create and use the following custom IAM policy for the VPC CNI. This policy is specific to multi-NIC. For more general information about using the VPC CNI with `IPv6` clusters, see <>. ++ +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:9b526100-356f-4a63-9d35-95318929846b[] +---- +* `IPv6` *transition mechanism not available* - If you use the multi-NIC feature, the VPC CNI doesn't assign an `IPv4` address to pods on an `IPv6` cluster. Otherwise, the VPC CNI assigns a host-local `IPv4` address to each pod so that a pod can communicate with external `IPv4` resources in another Amazon VPC or the internet. + +[#pod-multi-NIC-usage] +== Usage + +After the multi-NIC feature is enabled in the VPC CNI and the `aws-node` pods have restarted, you can configure each workload to be multi-homed. The following example of a YAML configuration with the required annotation: + +[source,yaml,subs="verbatim,attributes"] +---- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: orders-deployment + namespace: ecommerce + labels: + app: orders +spec: + replicas: 3 + selector: + matchLabels: + app: orders + template: + metadata: + annotations: + k8s.amazonaws.com/nicConfig: multi-nic-attachment + labels: + app: orders + spec: +... +---- + + +[#pod-muti-nic-faqs] +== Frequently Asked Questions + +[#pod-muti-nic-faqs-nic] +=== *1. What is a network interface card (NIC)?* + +A network interface card (NIC), also simply called a network card, is a physical device that enables network connectivity for the underlying cloud compute hardware. In modern EC2 servers, this refers to the Nitro network card. An Elastic Network Interface (ENI) is a virtual representation of this underlying network card. + +Some EC2 instance types have multiple NICs for greater bandwidth and packet rate performance. For such instances, you can assign secondary ENIs to the additional network cards. For example, ENI #1 can function as the interface for the NIC attached to network card index 0, whereas ENI #2 can function as the interface for the NIC attached to a separate network card index. + +[#pod-muti-nic-faqs-pod] +=== *2. What is a multi-homed pod?* + +A multi-homed pod is a single Kubernetes pod with multiple network interfaces (and by implication multiple IP addresses). Each pod network interface is associated with an link:AWSEC2/latest/UserGuide/using-eni.html[Elastic Network Interface (ENI), type="documentation"], and these ENIs are logical representations of separate NICs on the underlying worker node. With multiple network interfaces, a multi-homed pod has additional data transfer capacity, which also raises its data transfer rate. + +[IMPORTANT] +==== +The VPC CNI can only configure multi-homed pods on instance types that have multiple NICs. +==== + +[#pod-muti-nic-faqs-why] +=== *3. Why should I use this feature?* + +If you need to scale network performance in your Kubernetes-based workloads, you can use the multi-NIC feature to run multi-homed pods that interface with all the underlying NICs that have an ENA device attached to it. Leveraging additional network cards raises the bandwidth capacity and packet rate performance in your applications by distributing application traffic across multiple concurrent connections. This is especially useful for Artificial Intelligence (AI), Machine Learning (ML), and High Performance Computing (HPC) use cases. + +[#pod-muti-nic-faqs-how-to-enable] +=== *4. How do I use this feature?* + + +. First, you must ensure that your Kubernetes cluster is using VPC CNI version 1.20 or later. For the steps to update the VPC CNI as an EKS add-on, see <>. +. Then, you have to enable multi-NIC support in the VPC CNI by using the `ENABLE_MULTI_NIC` environment variable. +. Then, you must ensure that you make and join nodes that have multiple network cards. For a list of EC2 instance types that have multiple network cards, see link:AWSEC2/latest/UserGuide/using-eni.html#network-cards[Network cards, type="documentation"] in the _Amazon EC2 User Guide_. +. Finally, you configure each workload to use either multiple network interfaces (multi-homed pods) or use a single network interface. + + +[#pod-muti-nic-faqs-how-to-workloads] +=== *5. How do I configure my workloads to use multiple NICs on a supported worker node?* + +To use multi-homed pods, you need to add the following annotation: `k8s.amazonaws.com/nicConfig: multi-nic-attachment`. This will attach an ENI from every NIC in the underlying instance to the pod (one to many mapping between a pod and the NICs). + +If this annotation is missing, the VPC CNI assumes that your pod only requires 1 network interface and assigns it an IP from an ENI on any available NIC. + +[#pod-muti-nic-faqs-adapters] +=== *6. What network interface adapters are supported with this feature?* + +You can use any network interface adapter if you have at least one ENA attached to the underlying network card for IP traffic. For more information about ENA, see link:AWSEC2/latest/UserGuide/enhanced-networking-ena.html[Elastic Network Adapter (ENA), type="documentation"] in the _Amazon EC2 User Guide_. + +Supported network device configurations: + +* **ENA** interfaces provide all of the traditional IP networking and routing features that are required to support IP networking for a VPC. For more information, see link:AWSEC2/latest/UserGuide/enhanced-networking-ena.html[Enable enhanced networking with ENA on your EC2 instances, type="documentation"]. +* **EFA** **(EFA** **with ENA)** interfaces provide both the ENA device for IP networking and the EFA device for low-latency, high-throughput communication. + +[IMPORTANT] +==== +If a network card only has an **EFA-only** adapter attached to it, the VPC CNI will skip it when provisioning network connectivity for a multi-homed pod. However, if you combine an **EFA-only** adapter with an **ENA** adapter on a network card, then the VPC CNI will manage ENIs on this device as well. To use EFA-only interfaces with EKS clusters, see <>. +==== + +[#pod-muti-nic-faqs-node-ena] +=== *7. Can I see if a node in my cluster has ENA support?* + +Yes, you can use the {aws} CLI or EC2 API to retrieve network information about an EC2 instance in your cluster. This provides details on whether or not the instance has ENA support. In the following example, replace `` with the EC2 instance ID of a node. + +{aws} CLI example: + +[source,bash,subs="verbatim,attributes"] +---- +aws ec2 describe-instances --instance-ids --query "Reservations[].Instances[].EnaSupport" +---- + + +Example output: + +``` +[ true ] +``` + +[#pod-muti-nic-faqs-list-ips] +=== *8. Can I see the different IP addresses associated with a pod?* + +No, not easily. However, you can use `nsenter` from the node to run common network tools such as `ip route show` and see the additional IP addresses and interfaces. + +[#pod-muti-nic-faqs-number-of-enis] +=== *9. Can I control the number of network interfaces for my pods?* + +No. When your workload is configured to use multiple NICs on a supported instance, a single pod automatically has an IP address from every network card on the instance. Alternatively, single-homed pods will have one network interface attached to one NIC on the instance. + +[IMPORTANT] +==== +Network cards that _only_ have an **EFA-only** device attached to it are skipped by the VPC CNI. +==== + +[#pod-muti-nic-faqs-specify-nic] +=== *10. Can I configure my pods to use a specific NIC?* + +No, this isn't supported. If a pod has the relevant annotation, then the VPC CNI automatically configures it to use every NIC with an ENA adapter on the worker node. + +[#pod-muti-nic-faqs-modes] +=== *11. Does this feature work with the other VPC CNI networking features?* + +Yes, the multi-NIC feature in the VPC CNI works with both _custom networking_ and _enhanced subnet discovery_. However, the multi-homed pods don't use the custom subnets or security groups. Instead, the VPC CNI assigns IP addresses and network interfaces to the multi-homed pods with the same subnet and security group configuration as the node. For more information about custom networking, see <>. + +The multi-NIC feature in the VPC CNI doesn't work with and can't be combined with _security groups for pods_. + +[#pod-muti-nic-faqs-netpol] +=== *12. Can I use network policies with this feature?* + +Yes, you can use Kubernetes network policies with multi-NIC. Kubernetes network policies restrict network traffic to and from your pods. For more information about applying network policies with the VPC CNI, see <>. + +[#pod-muti-nic-faqs-auto-mode] +=== *13. Is multi-NIC support enabled in EKS Auto Mode?* + +Multi-NIC isn't supported for EKS Auto Mode clusters. diff --git a/latest/ug/networking/pod-multus.adoc b/latest/ug/networking/pod-multus.adoc new file mode 100644 index 000000000..f7b3f5c21 --- /dev/null +++ b/latest/ug/networking/pod-multus.adoc @@ -0,0 +1,31 @@ +include::../attributes.txt[] + +[.topic] +[#pod-multus] += Attach multiple network interfaces to Pods with Multus +:info_titleabbrev: Multus + +[abstract] +-- +Learn how to use Multus CNI to attach multiple network interfaces to a Pod in Amazon EKS for advanced networking scenarios, while leveraging the Amazon VPC CNI plugin for primary networking. +-- + +Multus CNI is a container network interface (CNI) plugin for Amazon EKS that enables attaching multiple network interfaces to a Pod. For more information, see the https://github.com/k8snetworkplumbingwg/multus-cni[Multus-CNI] documentation on GitHub. + +In Amazon EKS, each Pod has one network interface assigned by the Amazon VPC CNI plugin. With Multus, you can create a multi-homed Pod that has multiple interfaces. This is accomplished by Multus acting as a "meta-plugin"; a CNI plugin that can call multiple other CNI plugins. {aws} support for Multus comes configured with the Amazon VPC CNI plugin as the default delegate plugin. + +* Amazon EKS won't be building and publishing single root I/O virtualization (SR-IOV) and Data Plane Development Kit (DPDK) CNI plugins. However, you can achieve packet acceleration by connecting directly to Amazon EC2 Elastic Network Adapters (ENA) through Multus managed host-device and `ipvlan` plugins. +* Amazon EKS is supporting Multus, which provides a generic process that enables simple chaining of additional CNI plugins. Multus and the process of chaining is supported, but {aws} won't provide support for all compatible CNI plugins that can be chained, or issues that may arise in those CNI plugins that are unrelated to the chaining configuration. +* Amazon EKS is providing support and life cycle management for the Multus plugin, but isn't responsible for any IP address or additional management associated with the additional network interfaces. The IP address and management of the default network interface utilizing the Amazon VPC CNI plugin remains unchanged. +* Only the Amazon VPC CNI plugin is officially supported as the default delegate plugin. You need to modify the published Multus installation manifest to reconfigure the default delegate plugin to an alternate CNI if you choose not to use the Amazon VPC CNI plugin for primary networking. +* Multus is only supported when using the Amazon VPC CNI as the primary CNI. We do not support the Amazon VPC CNI when used for higher order interfaces, secondary or otherwise. +* To prevent the Amazon VPC CNI plugin from trying to manage additional network interfaces assigned to Pods, add the following tag to the network interface: ++ +*key*:: +: `node.k8s.amazonaws.com/no_manage` ++ +*value*:: +: `true` +* Multus is compatible with network policies, but the policy has to be enriched to include ports and IP addresses that may be part of additional network interfaces attached to Pods. + +For an implementation walk through, see the https://github.com/aws-samples/eks-install-guide-for-multus/blob/main/README.md[Multus Setup Guide] on GitHub. \ No newline at end of file diff --git a/latest/ug/networking/pod-networking-use-cases.adoc b/latest/ug/networking/pod-networking-use-cases.adoc new file mode 100644 index 000000000..56d3d0ef5 --- /dev/null +++ b/latest/ug/networking/pod-networking-use-cases.adoc @@ -0,0 +1,56 @@ +include::../attributes.txt[] + +[.topic] +[#pod-networking-use-cases] += Learn about VPC CNI modes and configuration +:info_titleabbrev: Modes and configuration + +[abstract] +-- +Discover how Amazon VPC CNI plugin for Kubernetes provides pod networking capabilities and settings for different Amazon EKS node types and use cases, including security groups, Kubernetes network policies, custom networking, IPv4, and IPv6 support. +-- + +The Amazon VPC CNI plugin for Kubernetes provides networking for Pods. Use the following table to learn more about the available networking features. + +[%header,cols="2"] +|=== +|Networking feature +|Learn more + + +|Configure your cluster to assign IPv6 addresses to clusters, Pods, and services +|<> + +|Use IPv4 Source Network Address Translation for Pods +|<> + +|Restrict network traffic to and from your Pods +|<> + +|Customize the secondary network interface in nodes +|<> + +|Increase IP addresses for your node +|<> + +|Use security groups for Pod network traffic +|<> + +|Use multiple network interfaces for Pods +|<> +|=== + +include::cni-ipv6.adoc[leveloffset=+1] + +include::external-snat.adoc[leveloffset=+1] + +include::cni-network-policy.adoc[leveloffset=+1] + +include::cni-custom-network.adoc[leveloffset=+1] + +include::cni-increase-ip-addresses.adoc[leveloffset=+1] + +include::security-groups-for-pods.adoc[leveloffset=+1] + +include::pod-multiple-network-interfaces.adoc[leveloffset=+1] + diff --git a/latest/ug/networking/sec-group-reqs.adoc b/latest/ug/networking/sec-group-reqs.adoc index 7eeed60a2..2128f6ff3 100644 --- a/latest/ug/networking/sec-group-reqs.adoc +++ b/latest/ug/networking/sec-group-reqs.adoc @@ -1,30 +1,23 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[sec-group-reqs,sec-group-reqs.title]] +[#sec-group-reqs] = View Amazon EKS security group requirements for clusters -:info_doctype: section -:info_title: View Amazon EKS security group requirements for clusters :info_titleabbrev: Security group requirements -:info_abstract: Learn how to manage security groups for Amazon EKS clusters, including default \ - rules, restricting traffic, and required outbound access for nodes to function \ - properly with your cluster. Understand key security group considerations for secure \ - operation of your Kubernetes cluster on {aws}. - -include::../attributes.txt[] [abstract] -- -Learn how to manage security groups for Amazon EKS clusters, including default rules, restricting traffic, and required outbound access for nodes to function properly with your cluster. Understand key security group considerations for secure operation of your [.noloc]`Kubernetes` cluster on {aws}. +Learn how to manage security groups for Amazon EKS clusters, including default rules, restricting traffic, and required outbound access for nodes to function properly with your cluster. Understand key security group considerations for secure operation of your Kubernetes cluster on {aws}. -- This topic describes the security group requirements of an Amazon EKS cluster. -[[security-group-default-rules,security-group-default-rules.title]] +[#security-group-default-rules] == Default cluster security group When you create a cluster, Amazon EKS creates a security group that's named `eks-cluster-sg-[.replaceable]``my-cluster``-[.replaceable]``uniqueID```. This security group has the following default rules: -[cols="1,1,1,1,1", options="header"] +[%header,cols="5"] |=== |Rule type |Protocol @@ -44,18 +37,26 @@ When you create a cluster, Amazon EKS creates a security group that's named `eks |All | |0.0.0.0/0(`IPv4`) or ::/0 (`IPv6`) + +|Outbound +|All +|All +| +|Self (for EFA traffic) |=== +The default security group includes an outbound rule that allows Elastic Fabric Adapter (EFA) traffic with the destination of the same security group. This enables EFA traffic within the cluster, which is beneficial for AI/ML and High Performance Computing (HPC) workloads. For more information, see link:AWSEC2/latest/UserGuide/efa.html["Elastic Fabric Adapter for AI/ML and HPC workloads on Amazon EC2",type="documentation"] in the _Amazon Elastic Compute Cloud User Guide_. + [IMPORTANT] ==== -If your cluster doesn't need the outbound rule, you can remove it. If you remove it, you must still have the minimum rules listed in <>. If you remove the inbound rule, Amazon EKS recreates it whenever the cluster is updated. +If your cluster doesn't need the outbound rule, you can remove it. If you remove it, you must still have the minimum rules listed in <>. If you remove the inbound rule, Amazon EKS recreates it whenever the cluster is updated. ==== Amazon EKS adds the following tags to the security group. If you remove the tags, Amazon EKS adds them back to the security group whenever your cluster is updated. -[cols="1,1", options="header"] +[%header,cols="2"] |=== |Key |Value @@ -88,12 +89,12 @@ aws eks describe-cluster --name my-cluster --query cluster.resourcesVpcConfig.cl ---- -[[security-group-restricting-cluster-traffic,security-group-restricting-cluster-traffic.title]] +[#security-group-restricting-cluster-traffic] == Restricting cluster traffic -If you need to limit the open ports between the cluster and nodes, you can remove the <> and add the following minimum rules that are required for the cluster. If you remove the <>, Amazon EKS recreates it whenever the cluster is updated. +If you need to limit the open ports between the cluster and nodes, you can remove the <> and add the following minimum rules that are required for the cluster. If you remove the <>, Amazon EKS recreates it whenever the cluster is updated. -[cols="1,1,1,1", options="header"] +[%header,cols="4"] |=== |Rule type |Protocol @@ -123,14 +124,14 @@ You must also add rules for the following traffic: * Any protocol and ports that you expect your nodes to use for inter-node communication. * Outbound internet access so that nodes can access the Amazon EKS APIs for cluster introspection and node registration at launch time. If your nodes don't have internet access, review <> for additional considerations. -* Node access to pull container images from Amazon ECR or other container registries APIs that they need to pull images from, such as [.noloc]`DockerHub`. For more information, see link:general/latest/gr/aws-ip-ranges.html[{aws} IP address ranges,type="documentation"] in the {aws} General Reference. +* Node access to pull container images from Amazon ECR or other container registries APIs that they need to pull images from, such as DockerHub. For more information, see link:general/latest/gr/aws-ip-ranges.html[{aws} IP address ranges,type="documentation"] in the {aws} General Reference. * Node access to Amazon S3. * Separate rules are required for `IPv4` and `IPv6` addresses. * If you are using hybrid nodes, you must add an additional security group to your cluster to allow communication with your on-premises nodes and pods. For more information, see <>. -If you're considering limiting the rules, we recommend that you thoroughly test all of your [.noloc]`Pods` before you apply your changed rules to a production cluster. +If you're considering limiting the rules, we recommend that you thoroughly test all of your Pods before you apply your changed rules to a production cluster. -If you originally deployed a cluster with [.noloc]`Kubernetes` `1.14` and a platform version of `eks.3` or earlier, then consider the following: +If you originally deployed a cluster with Kubernetes `1.14` and a platform version of `eks.3` or earlier, then consider the following: @@ -149,4 +150,4 @@ Amazon EKS supports shared security groups. === Considerations for Amazon EKS -* EKS has the same requirements of shared or multi-VPC security groups as standard security groups. +* EKS has the same requirements of shared or multi-VPC security groups as standard security groups. diff --git a/latest/ug/networking/security-groups-for-pods.adoc b/latest/ug/networking/security-groups-for-pods.adoc new file mode 100644 index 000000000..38838a0ff --- /dev/null +++ b/latest/ug/networking/security-groups-for-pods.adoc @@ -0,0 +1,57 @@ +include::../attributes.txt[] + +[.topic] +[#security-groups-for-pods] += Assign security groups to individual Pods +:info_titleabbrev: Security groups for Pods + +[abstract] +-- +Learn how to configure security groups for Pods on Amazon EKS, integrating Amazon EC2 security groups with Kubernetes Pods to define network traffic rules. Discover the considerations, setup process, and deploy a sample application with assigned security groups. +-- + +*Applies to*: Linux nodes with Amazon EC2 instances + +*Applies to*: Private subnets + +Security groups for Pods integrate Amazon EC2 security groups with Kubernetes Pods. You can use Amazon EC2 security groups to define rules that allow inbound and outbound network traffic to and from Pods that you deploy to nodes running on many Amazon EC2 instance types and Fargate. For a detailed explanation of this capability, see the link:containers/introducing-security-groups-for-pods[Introducing security groups for Pods,type="blog"] blog post. + +[#security-groups-for-pods-compatability] +== Compatibility with Amazon VPC CNI plugin for Kubernetes features + +You can use security groups for Pods with the following features: + + + +* IPv4 Source Network Address Translation - For more information, see <>. +* IPv6 addresses to clusters, Pods, and services - For more information, see <>. +* Restricting traffic using Kubernetes network policies - For more information, see <>. + + +[#sg-pods-considerations] +== Considerations + +Before deploying security groups for Pods, consider the following limitations and conditions: + + + +* Security groups for Pods can't be used with Windows nodes or EKS Auto Mode. +* Security groups for Pods can be used with clusters configured for the `IPv6` family that contain Amazon EC2 nodes by using version 1.16.0 or later of the Amazon VPC CNI plugin. You can use security groups for Pods with clusters configure `IPv6` family that contain only Fargate nodes by using version 1.7.7 or later of the Amazon VPC CNI plugin. For more information, see <> +* Security groups for Pods are supported by most link:AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances[Nitro-based,type="documentation"] Amazon EC2 instance families, though not by all generations of a family. For example, the `m5`, `c5`, `r5`, `m6g`, `c6g`, and `r6g` instance family and generations are supported. No instance types in the `t` family are supported. For a complete list of supported instance types, see the https://github.com/aws/amazon-vpc-resource-controller-k8s/blob/v1.5.0/pkg/aws/vpc/limits.go[limits.go] file on GitHub. Your nodes must be one of the listed instance types that have `IsTrunkingCompatible: true` in that file. +* If you're using custom networking and security groups for Pods together, the security group specified by security groups for Pods is used instead of the security group specified in the `ENIConfig`. +* If you're using version `1.10.2` or earlier of the Amazon VPC CNI plugin and you include the `terminationGracePeriodSeconds` setting in your Pod spec, the value for the setting can't be zero. +* If you're using version `1.10` or earlier of the Amazon VPC CNI plugin, or version `1.11` with `POD_SECURITY_GROUP_ENFORCING_MODE`=``strict``, which is the default setting, then Kubernetes services of type `NodePort` and `LoadBalancer` using instance targets with an `externalTrafficPolicy` set to `Local` aren't supported with Pods that you assign security groups to. For more information about using a load balancer with instance targets, see <>. +* If you're using version `1.10` or earlier of the Amazon VPC CNI plugin or version `1.11` with `POD_SECURITY_GROUP_ENFORCING_MODE`=``strict``, which is the default setting, source NAT is disabled for outbound traffic from Pods with assigned security groups so that outbound security group rules are applied. To access the internet, Pods with assigned security groups must be launched on nodes that are deployed in a private subnet configured with a NAT gateway or instance. Pods with assigned security groups deployed to public subnets are not able to access the internet. ++ +If you're using version `1.11` or later of the plugin with `POD_SECURITY_GROUP_ENFORCING_MODE`=``standard``, then Pod traffic destined for outside of the VPC is translated to the IP address of the instance's primary network interface. For this traffic, the rules in the security groups for the primary network interface are used, rather than the rules in the Pod's security groups. +* To use Calico network policy with Pods that have associated security groups, you must use version `1.11.0` or later of the Amazon VPC CNI plugin and set `POD_SECURITY_GROUP_ENFORCING_MODE`=``standard``. Otherwise, traffic flow to and from Pods with associated security groups are not subjected to Calico network policy enforcement and are limited to Amazon EC2 security group enforcement only. To update your Amazon VPC CNI version, see <> +* Pods running on Amazon EC2 nodes that use security groups in clusters that use https://kubernetes.io/docs/tasks/administer-cluster/nodelocaldns/[NodeLocal DNSCache] are only supported with version `1.11.0` or later of the Amazon VPC CNI plugin and with `POD_SECURITY_GROUP_ENFORCING_MODE`=``standard``. To update your Amazon VPC CNI plugin version, see <> +* Security groups for Pods might lead to higher Pod startup latency for Pods with high churn. This is due to rate limiting in the resource controller. +* The EC2 security group scope is at the Pod-level - For more information, see link:vpc/latest/userguide/VPC_SecurityGroups.html[Security group,type="documentation"]. ++ +If you set `POD_SECURITY_GROUP_ENFORCING_MODE=standard` and `AWS_VPC_K8S_CNI_EXTERNALSNAT=false`, traffic destined for endpoints outside the VPC use the node's security groups, not the Pod's security groups. + +include::security-groups-pods-deployment.adoc[leveloffset=+1] + +include::sg-pods-example-deployment.adoc[leveloffset=+1] + diff --git a/latest/ug/networking/security-groups-pods-deployment.adoc b/latest/ug/networking/security-groups-pods-deployment.adoc new file mode 100644 index 000000000..f22b20a1f --- /dev/null +++ b/latest/ug/networking/security-groups-pods-deployment.adoc @@ -0,0 +1,91 @@ +include::../attributes.txt[] + +[.topic] +[#security-groups-pods-deployment] += Configure the Amazon VPC CNI plugin for Kubernetes for security groups for Amazon EKS Pods +:info_titleabbrev: Configure + +If you use Pods with Amazon EC2 instances, you need to configure the Amazon VPC CNI plugin for Kubernetes for security groups + +If you use Fargate Pods only, and don't have any Amazon EC2 nodes in your cluster, see <>. + +. Check your current Amazon VPC CNI plugin for Kubernetes version with the following command: ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl describe daemonset aws-node --namespace kube-system | grep amazon-k8s-cni: | cut -d : -f 3 +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +v1.7.6 +---- ++ +If your Amazon VPC CNI plugin for Kubernetes version is earlier than `1.7.7`, then update the plugin to version `1.7.7` or later. For more information, see <> +. Add the link:iam/home#/policies/arn:aws:iam::aws:policy/AmazonEKSVPCResourceController[AmazonEKSVPCResourceController,type="console"] managed IAM policy to the <> that is associated with your Amazon EKS cluster. The policy allows the role to manage network interfaces, their private IP addresses, and their attachment and detachment to and from network instances. ++ +.. Retrieve the name of your cluster IAM role and store it in a variable. Replace [.replaceable]`my-cluster` with the name of your cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +cluster_role=$(aws eks describe-cluster --name my-cluster --query cluster.roleArn --output text | cut -d / -f 2) +---- +.. Attach the policy to the role. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam attach-role-policy --policy-arn {arn-aws}iam::aws:policy/AmazonEKSVPCResourceController --role-name $cluster_role +---- +. Enable the Amazon VPC CNI add-on to manage network interfaces for Pods by setting the `ENABLE_POD_ENI` variable to `true` in the `aws-node` DaemonSet. Once this setting is set to `true`, for each node in the cluster the add-on creates a `cninode` custom resource. The VPC resource controller creates and attaches one special network interface called a _trunk network interface_ with the description `aws-k8s-trunk-eni`. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl set env daemonset aws-node -n kube-system ENABLE_POD_ENI=true +---- ++ +NOTE: The trunk network interface is included in the maximum number of network interfaces supported by the instance type. For a list of the maximum number of network interfaces supported by each instance type, see link:AWSEC2/latest/UserGuide/using-eni.html#AvailableIpPerENI[IP addresses per network interface per instance type,type="documentation"] in the _Amazon EC2 User Guide_. If your node already has the maximum number of standard network interfaces attached to it then the VPC resource controller will reserve a space. You will have to scale down your running Pods enough for the controller to detach and delete a standard network interface, create the trunk network interface, and attach it to the instance. +. You can see which of your nodes have a `CNINode` custom resource with the following command. If `No resources found` is returned, then wait several seconds and try again. The previous step requires restarting the Amazon VPC CNI plugin for Kubernetes Pods, which takes several seconds. ++ +[source,shell,subs="verbatim,attributes"] +---- +kubectl get cninode -A + NAME FEATURES + ip-192-168-64-141.us-west-2.compute.internal [{"name":"SecurityGroupsForPods"}] + ip-192-168-7-203.us-west-2.compute.internal [{"name":"SecurityGroupsForPods"}] +---- ++ +If you are using VPC CNI versions older than `1.15`, node labels were used instead of the `CNINode` custom resource. You can see which of your nodes have the node label `aws-k8s-trunk-eni` set to `true` with the following command. If `No resources found` is returned, then wait several seconds and try again. The previous step requires restarting the Amazon VPC CNI plugin for Kubernetes Pods, which takes several seconds. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get nodes -o wide -l vpc.amazonaws.com/has-trunk-attached=true +---- ++ +Once the trunk network interface is created, Pods are assigned secondary IP addresses from the trunk or standard network interfaces. The trunk interface is automatically deleted if the node is deleted. ++ +When you deploy a security group for a Pod in a later step, the VPC resource controller creates a special network interface called a _branch network interface_ with a description of `aws-k8s-branch-eni` and associates the security groups to it. Branch network interfaces are created in addition to the standard and trunk network interfaces attached to the node. ++ +If you are using liveness or readiness probes, then you also need to disable _TCP early demux_, so that the `kubelet` can connect to Pods on branch network interfaces using TCP. To disable _TCP early demux_, run the following command: ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl patch daemonset aws-node -n kube-system \ + -p '{"spec": {"template": {"spec": {"initContainers": [{"env":[{"name":"DISABLE_TCP_EARLY_DEMUX","value":"true"}],"name":"aws-vpc-cni-init"}]}}}}' +---- ++ +NOTE: If you're using `1.11.0` or later of the Amazon VPC CNI plugin for Kubernetes add-on and set `POD_SECURITY_GROUP_ENFORCING_MODE`=``standard``, as described in the next step, then you don't need to run the previous command. +. If your cluster uses `NodeLocal DNSCache`, or you want to use Calico network policy with your Pods that have their own security groups, or you have Kubernetes services of type `NodePort` and `LoadBalancer` using instance targets with an `externalTrafficPolicy` set to `Local` for Pods that you want to assign security groups to, then you must be using version `1.11.0` or later of the Amazon VPC CNI plugin for Kubernetes add-on, and you must enable the following setting: ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl set env daemonset aws-node -n kube-system POD_SECURITY_GROUP_ENFORCING_MODE=standard +---- ++ +IMPORTANT: +** Pod security group rules aren't applied to traffic between Pods or between Pods and services, such as `kubelet` or `nodeLocalDNS`, that are on the same node. Pods using different security groups on the same node can't communicate because they are configured in different subnets, and routing is disabled between these subnets. +** Outbound traffic from Pods to addresses outside of the VPC is network address translated to the IP address of the instance's primary network interface (unless you've also set `AWS_VPC_K8S_CNI_EXTERNALSNAT=true`). For this traffic, the rules in the security groups for the primary network interface are used, rather than the rules in the Pod's security groups. +** For this setting to apply to existing Pods, you must restart the Pods or the nodes that the Pods are running on. + +. To see how to use a security group policy for your Pod, see <>. diff --git a/latest/ug/networking/sg-pods-example-deployment.adoc b/latest/ug/networking/sg-pods-example-deployment.adoc new file mode 100644 index 000000000..dee0e241f --- /dev/null +++ b/latest/ug/networking/sg-pods-example-deployment.adoc @@ -0,0 +1,209 @@ +include::../attributes.txt[] + +[.topic] +[#sg-pods-example-deployment] += Use a security group policy for an Amazon EKS Pod +:info_titleabbrev: SecurityGroupPolicy + +To use security groups for Pods, you must have an existing security group. The following steps show you how to use the security group policy for a Pod. Unless otherwise noted, complete all steps from the same terminal because variables are used in the following steps that don't persist across terminals. + +If you have a Pod with Amazon EC2 instances, you must configure the plugin before you use this procedure. For more information, see <>. + +. Create a Kubernetes namespace to deploy resources to. You can replace [.replaceable]`my-namespace` with the name of a namespace that you want to use. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl create namespace my-namespace +---- +. [[deploy-securitygrouppolicy]]Deploy an Amazon EKS `SecurityGroupPolicy` to your cluster. ++ +.. Copy the following contents to your device. You can replace [.replaceable]`podSelector` with `serviceAccountSelector` if you'd rather select Pods based on service account labels. You must specify one selector or the other. An empty `podSelector` (example: `podSelector: {}`) selects all Pods in the namespace. You can change [.replaceable]`my-role` to the name of your role. An empty `serviceAccountSelector` selects all service accounts in the namespace. You can replace [.replaceable]`my-security-group-policy` with a name for your `SecurityGroupPolicy` and [.replaceable]`my-namespace` with the namespace that you want to create the `SecurityGroupPolicy` in. ++ +You must replace [.replaceable]`my_pod_security_group_id` with the ID of an existing security group. If you don't have an existing security group, then you must create one. For more information, see link:AWSEC2/latest/UserGuide/ec2-security-groups.html[Amazon EC2 security groups for Linux instances,type="documentation"] in the link:AWSEC2/latest/UserGuide/[Amazon EC2 User Guide,type="documentation"]. You can specify 1-5 security group IDs. If you specify more than one ID, then the combination of all the rules in all the security groups are effective for the selected Pods. ++ +[source,yaml,subs="verbatim,attributes"] +---- +cat >my-security-group-policy.yaml <sample-application.yaml < +my-deployment-5df6f7687b-j9fl4 1/1 Running 0 7m51s 192.168.70.145 ip-192-168-92-33.region-code.compute.internal +my-deployment-5df6f7687b-rjxcz 1/1 Running 0 7m51s 192.168.73.207 ip-192-168-92-33.region-code.compute.internal +my-deployment-5df6f7687b-zmb42 1/1 Running 0 7m51s 192.168.63.27 ip-192-168-33-28.region-code.compute.internal +---- ++ +[NOTE] +==== +Try these tips if any Pods are stuck. + +* If any Pods are stuck in the `Waiting` state, then run `kubectl describe pod [.replaceable]``my-deployment-xxxxxxxxxx-xxxxx`` -n [.replaceable]``my-namespace```. If you see `Insufficient permissions: Unable to create Elastic Network Interface.`, confirm that you added the IAM policy to the IAM cluster role in a previous step. +* If any Pods are stuck in the `Pending` state, confirm that your node instance type is listed in https://github.com/aws/amazon-vpc-resource-controller-k8s/blob/master/pkg/aws/vpc/limits.go[limits.go] and that the product of the maximum number of branch network interfaces supported by the instance type multiplied times the number of nodes in your node group hasn't already been met. For example, an `m5.large` instance supports nine branch network interfaces. If your node group has five nodes, then a maximum of 45 branch network interfaces can be created for the node group. The 46th Pod that you attempt to deploy will sit in `Pending` state until another Pod that has associated security groups is deleted. + +==== ++ +If you run `kubectl describe pod [.replaceable]``my-deployment-xxxxxxxxxx-xxxxx`` -n [.replaceable]``my-namespace``` and see a message similar to the following message, then it can be safely ignored. This message might appear when the Amazon VPC CNI plugin for Kubernetes tries to set up host networking and fails while the network interface is being created. The plugin logs this event until the network interface is created. ++ +[source,bash,subs="verbatim,attributes"] +---- +Failed to create Pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "e24268322e55c8185721f52df6493684f6c2c3bf4fd59c9c121fd4cdc894579f" network for Pod "my-deployment-5df6f7687b-4fbjm": networkPlugin +cni failed to set up Pod "my-deployment-5df6f7687b-4fbjm-c89wx_my-namespace" network: add cmd: failed to assign an IP address to container +---- ++ +You can't exceed the maximum number of Pods that can be run on the instance type. For a list of the maximum number of Pods that you can run on each instance type, see https://github.com/awslabs/amazon-eks-ami/blob/main/templates/shared/runtime/eni-max-pods.txt[eni-max-pods.txt] on GitHub. When you delete a Pod that has associated security groups, or delete the node that the Pod is running on, the VPC resource controller deletes the branch network interface. If you delete a cluster with Pods using Pods for security groups, then the controller doesn't delete the branch network interfaces, so you'll need to delete them yourself. For information about how to delete network interfaces, see link:AWSEC2/latest/UserGuide/using-eni.html#delete_eni[Delete a network interface,type="documentation"] in the Amazon EC2 User Guide. +. In a separate terminal, shell into one of the Pods. For the remainder of this topic, this terminal is referred to as `TerminalB`. Replace [.replaceable]`5df6f7687b-4fbjm` with the ID of one of the Pods returned in your output from the previous step. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl exec -it -n my-namespace my-deployment-5df6f7687b-4fbjm -- /bin/bash +---- +. From the shell in `TerminalB`, confirm that the sample application works. ++ +[source,bash,subs="verbatim,attributes"] +---- +curl my-app +---- ++ +An example output is as follows. ++ +[source,html,subs="verbatim"] +---- + + + +Welcome to nginx! +[...] +---- ++ +You received the output because all Pods running the application are associated with the security group that you created. That group contains a rule that allows all traffic between all Pods that the security group is associated to. DNS traffic is allowed outbound from that security group to the cluster security group, which is associated with your nodes. The nodes are running the CoreDNS Pods, which your Pods did a name lookup to. +. From `TerminalA`, remove the security group rules that allow DNS communication to the cluster security group from your security group. If you didn't add the DNS rules to the cluster security group in a previous step, then replace [.replaceable]`$my_cluster_security_group_id` with the ID of the security group that you created the rules in. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws ec2 revoke-security-group-ingress --group-id $my_cluster_security_group_id --security-group-rule-ids $my_tcp_rule_id +aws ec2 revoke-security-group-ingress --group-id $my_cluster_security_group_id --security-group-rule-ids $my_udp_rule_id +---- +. From `TerminalB`, attempt to access the application again. ++ +[source,bash,subs="verbatim,attributes"] +---- +curl my-app +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +curl: (6) Could not resolve host: my-app +---- ++ +The attempt fails because the Pod is no longer able to access the CoreDNS Pods, which have the cluster security group associated to them. The cluster security group no longer has the security group rules that allow DNS communication from the security group associated to your Pod. ++ +If you attempt to access the application using the IP addresses returned for one of the Pods in a previous step, you still receive a response because all ports are allowed between Pods that have the security group associated to them and a name lookup isn't required. +. Once you've finished experimenting, you can remove the sample security group policy, application, and security group that you created. Run the following commands from `TerminalA`. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl delete namespace my-namespace +aws ec2 revoke-security-group-ingress --group-id $my_pod_security_group_id --security-group-rule-ids $my_inbound_self_rule_id +wait +sleep 45s +aws ec2 delete-security-group --group-id $my_pod_security_group_id +---- \ No newline at end of file diff --git a/latest/ug/networking/vpc-add-on-create.adoc b/latest/ug/networking/vpc-add-on-create.adoc new file mode 100644 index 000000000..9a181245a --- /dev/null +++ b/latest/ug/networking/vpc-add-on-create.adoc @@ -0,0 +1,91 @@ +include::../attributes.txt[] + +[.topic] +[#vpc-add-on-create] += Create the Amazon VPC CNI (Amazon EKS add-on) +:info_titleabbrev: Create + +Use the following steps to create the Amazon VPC CNI plugin for Kubernetes Amazon EKS add-on. + +Before you begin, review the considerations. For more information, see <>. + + +[#vpc-add-on-create-prerequisites] +== Prerequisites + +The following are prerequisites for the Amazon VPC CNI plugin for Kubernetes Amazon EKS add-on. + +* An existing Amazon EKS cluster. To deploy one, see <>. +* An existing {aws} Identity and Access Management (IAM) OpenID Connect (OIDC) provider for your cluster. To determine whether you already have one, or to create one, see <>. +* An IAM role with the link:aws-managed-policy/latest/reference/AmazonEKS_CNI_Policy.html[AmazonEKS_CNI_Policy,type="documentation"] IAM policy (if your cluster uses the `IPv4` family) or an IPv6 policy (if your cluster uses the `IPv6` family) attached to it. For more information about the VPC CNI role, see <>. For information about the IPv6 policy, see <>. + +[IMPORTANT] +==== + +Amazon VPC CNI plugin for Kubernetes versions `v1.16.0` to `v1.16.1` implement CNI specification version `v1.0.0`. For more information about `v1.0.0` of the CNI spec, see https://github.com/containernetworking/cni/blob/spec-v1.0.0/SPEC.md[Container Network Interface (CNI) Specification] on GitHub. + +==== + + +[#vpc-add-on-create-procedure] +== Procedure + +After you complete the prerequisites, use the following steps to create the add-on. + +. See which version of the add-on is installed on your cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl describe daemonset aws-node --namespace kube-system | grep amazon-k8s-cni: | cut -d : -f 3 +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +v1.16.4-eksbuild.2 +---- +. See which type of the add-on is installed on your cluster. Depending on the tool that you created your cluster with, you might not currently have the Amazon EKS add-on type installed on your cluster. Replace [.replaceable]`my-cluster` with the name of your cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks describe-addon --cluster-name my-cluster --addon-name vpc-cni --query addon.addonVersion --output text +---- ++ +If a version number is returned, you have the Amazon EKS type of the add-on installed on your cluster and don't need to complete the remaining steps in this procedure. If an error is returned, you don't have the Amazon EKS type of the add-on installed on your cluster. Complete the remaining steps of this procedure to install it. +. Save the configuration of your currently installed add-on. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get daemonset aws-node -n kube-system -o yaml > aws-k8s-cni-old.yaml +---- +. Create the add-on using the {aws} CLI. If you want to use the {aws-management-console} or `eksctl` to create the add-on, see <> and specify `vpc-cni` for the add-on name. Copy the command that follows to your device. Make the following modifications to the command, as needed, and then run the modified command. ++ +** Replace [.replaceable]`my-cluster` with the name of your cluster. +** Replace [.replaceable]`v1.20.3-eksbuild.1` with the latest version listed in the latest version table for your cluster version. For the latest version table, see <>. +** Replace [.replaceable]`111122223333` with your account ID and [.replaceable]`AmazonEKSVPCCNIRole` with the name of an <> that you've created. Specifying a role requires that you have an IAM OpenID Connect (OIDC) provider for your cluster. To determine whether you have one for your cluster, or to create one, see <>. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks create-addon --cluster-name my-cluster --addon-name vpc-cni --addon-version v1.20.3-eksbuild.1 \ + --service-account-role-arn {arn-aws}iam::111122223333:role/AmazonEKSVPCCNIRole +---- ++ +If you've applied custom settings to your current add-on that conflict with the default settings of the Amazon EKS add-on, creation might fail. If creation fails, you receive an error that can help you resolve the issue. Alternatively, you can add `--resolve-conflicts OVERWRITE` to the previous command. This allows the add-on to overwrite any existing custom settings. Once you've created the add-on, you can update it with your custom settings. +. Confirm that the latest version of the add-on for your cluster's Kubernetes version was added to your cluster. Replace [.replaceable]`my-cluster` with the name of your cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks describe-addon --cluster-name my-cluster --addon-name vpc-cni --query addon.addonVersion --output text +---- ++ +It might take several seconds for add-on creation to complete. ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +v1.20.3-eksbuild.1 +---- +. If you made custom settings to your original add-on, before you created the Amazon EKS add-on, use the configuration that you saved in a previous step to update the EKS add-on with your custom settings. Follow the steps in <>. +. (Optional) Install the `cni-metrics-helper` to your cluster. It scrapes elastic network interface and IP address information, aggregates it at a cluster level, and publishes the metrics to Amazon CloudWatch. For more information, see https://github.com/aws/amazon-vpc-cni-k8s/blob/master/cmd/cni-metrics-helper/README.md[cni-metrics-helper] on GitHub. \ No newline at end of file diff --git a/latest/ug/networking/vpc-add-on-self-managed-update.adoc b/latest/ug/networking/vpc-add-on-self-managed-update.adoc new file mode 100644 index 000000000..5379cea56 --- /dev/null +++ b/latest/ug/networking/vpc-add-on-self-managed-update.adoc @@ -0,0 +1,72 @@ +include::../attributes.txt[] + +[.topic] +[#vpc-add-on-self-managed-update] += Update the Amazon VPC CNI (self-managed add-on) +:info_titleabbrev: Update (self-managed) + +[IMPORTANT] +==== + +We recommend adding the Amazon EKS type of the add-on to your cluster instead of using the self-managed type of the add-on. If you're not familiar with the difference between the types, see <>. For more information about adding an Amazon EKS add-on to your cluster, see <>. If you're unable to use the Amazon EKS add-on, we encourage you to submit an issue about why you can't to the https://github.com/aws/containers-roadmap/issues[Containers roadmap GitHub repository]. + +==== +. Confirm that you don't have the Amazon EKS type of the add-on installed on your cluster. Replace [.replaceable]`my-cluster` with the name of your cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks describe-addon --cluster-name my-cluster --addon-name vpc-cni --query addon.addonVersion --output text +---- ++ +If an error message is returned, you don't have the Amazon EKS type of the add-on installed on your cluster. To self-manage the add-on, complete the remaining steps in this procedure to update the add-on. If a version number is returned, you have the Amazon EKS type of the add-on installed on your cluster. To update it, use the procedure in <>, rather than using this procedure. If you're not familiar with the differences between the add-on types, see <>. +. See which version of the container image is currently installed on your cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl describe daemonset aws-node --namespace kube-system | grep amazon-k8s-cni: | cut -d : -f 3 +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +v1.20.0-eksbuild.1 +---- ++ +Your output might not include the build number. +. Backup your current settings so you can configure the same settings once you've updated your version. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get daemonset aws-node -n kube-system -o yaml > aws-k8s-cni-old.yaml +---- +To review the available versions and familiarize yourself with the changes in the version that you want to update to, see https://github.com/aws/amazon-vpc-cni-k8s/releases[releases] on GitHub. Note that we recommend updating to the same `major`.``minor``.``patch`` version listed in the latest available versions table, even if later versions are available on GitHub. For the latest available version table, see <>. The build versions listed in the table aren't specified in the self-managed versions listed on GitHub. Update your version by completing the tasks in one of the following options: ++ +** If you don't have any custom settings for the add-on, then run the command under the `To apply this release:` heading on GitHub for the https://github.com/aws/amazon-vpc-cni-k8s/releases[release] that you're updating to. +** If you have custom settings, download the manifest file with the following command. Change [.replaceable]`https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/v1.20.0/config/master/aws-k8s-cni.yaml` to the URL for the release on GitHub that you're updating to. ++ +[source,bash,subs="verbatim,attributes"] +---- +curl -O https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/v1.20.3/config/master/aws-k8s-cni.yaml +---- ++ +If necessary, modify the manifest with the custom settings from the backup you made in a previous step and then apply the modified manifest to your cluster. If your nodes don't have access to the private Amazon EKS Amazon ECR repositories that the images are pulled from (see the lines that start with `image:` in the manifest), then you'll have to download the images, copy them to your own repository, and modify the manifest to pull the images from your repository. For more information, see <>. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl apply -f aws-k8s-cni.yaml +---- +. Confirm that the new version is now installed on your cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl describe daemonset aws-node --namespace kube-system | grep amazon-k8s-cni: | cut -d : -f 3 +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +v1.20.3 +---- +. (Optional) Install the `cni-metrics-helper` to your cluster. It scrapes elastic network interface and IP address information, aggregates it at a cluster level, and publishes the metrics to Amazon CloudWatch. For more information, see https://github.com/aws/amazon-vpc-cni-k8s/blob/master/cmd/cni-metrics-helper/README.md[cni-metrics-helper] on GitHub. \ No newline at end of file diff --git a/latest/ug/networking/vpc-add-on-update.adoc b/latest/ug/networking/vpc-add-on-update.adoc new file mode 100644 index 000000000..99350dcb0 --- /dev/null +++ b/latest/ug/networking/vpc-add-on-update.adoc @@ -0,0 +1,77 @@ +include::../attributes.txt[] + +[.topic] +[#vpc-add-on-update] += Update the Amazon VPC CNI (Amazon EKS add-on) +:info_titleabbrev: Update (EKS add-on) + +Update the Amazon EKS type of the Amazon VPC CNI plugin for Kubernetes add-on. If you haven't added the Amazon EKS type of the add-on to your cluster, you can install it by following <>. Or, update the other type of VPC CNI installation by following <>. + +. See which version of the add-on is installed on your cluster. Replace [.replaceable]`my-cluster` with your cluster name. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks describe-addon --cluster-name my-cluster --addon-name vpc-cni --query "addon.addonVersion" --output text +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +v1.20.0-eksbuild.1 +---- ++ +Compare the version with the table of latest versions at <>. If the version returned is the same as the version for your cluster's Kubernetes version in the latest version table, then you already have the latest version installed on your cluster and don't need to complete the rest of this procedure. If you receive an error, instead of a version number in your output, then you don't have the Amazon EKS type of the add-on installed on your cluster. You need to create the add-on before you can update it with this procedure. To create the Amazon EKS type of the VPC CNI add-on, you can follow <>. +. Save the configuration of your currently installed add-on. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get daemonset aws-node -n kube-system -o yaml > aws-k8s-cni-old.yaml +---- +. Update your add-on using the {aws} CLI. If you want to use the {aws-management-console} or `eksctl` to update the add-on, see <>. Copy the command that follows to your device. Make the following modifications to the command, as needed, and then run the modified command. ++ +** Replace [.replaceable]`my-cluster` with the name of your cluster. +** Replace [.replaceable]`v1.20.0-eksbuild.1` with the latest version listed in the latest version table for your cluster version. +** Replace [.replaceable]`111122223333` with your account ID and [.replaceable]`AmazonEKSVPCCNIRole` with the name of an existing IAM role that you've created. To create an IAM role for the VPC CNI, see <>. Specifying a role requires that you have an IAM OpenID Connect (OIDC) provider for your cluster. To determine whether you have one for your cluster, or to create one, see <>. +** The `--resolve-conflicts PRESERVE` option preserves existing configuration values for the add-on. If you've set custom values for add-on settings, and you don't use this option, Amazon EKS overwrites your values with its default values. If you use this option, then we recommend testing any field and value changes on a non-production cluster before updating the add-on on your production cluster. If you change this value to `OVERWRITE`, all settings are changed to Amazon EKS default values. If you've set custom values for any settings, they might be overwritten with Amazon EKS default values. If you change this value to `none`, Amazon EKS doesn't change the value of any settings, but the update might fail. If the update fails, you receive an error message to help you resolve the conflict. +** If you're not updating a configuration setting, remove `--configuration-values '{[.replaceable]``"env":{"AWS_VPC_K8S_CNI_EXTERNALSNAT":"true"}``}'` from the command. If you're updating a configuration setting, replace [.replaceable]`"env":{"AWS_VPC_K8S_CNI_EXTERNALSNAT":"true"}` with the setting that you want to set. In this example, the `AWS_VPC_K8S_CNI_EXTERNALSNAT` environment variable is set to `true`. The value that you specify must be valid for the configuration schema. If you don't know the configuration schema, run `aws eks describe-addon-configuration --addon-name vpc-cni --addon-version [.replaceable]``v1.20.0-eksbuild.1```, replacing [.replaceable]`v1.20.0-eksbuild.1` with the version number of the add-on that you want to see the configuration for. The schema is returned in the output. If you have any existing custom configuration, want to remove it all, and set the values for all settings back to Amazon EKS defaults, remove [.replaceable]`"env":{"AWS_VPC_K8S_CNI_EXTERNALSNAT":"true"}` from the command, so that you have empty `{}`. For an explanation of each setting, see https://github.com/aws/amazon-vpc-cni-k8s#cni-configuration-variables[CNI Configuration Variables] on GitHub. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks update-addon --cluster-name my-cluster --addon-name vpc-cni --addon-version v1.20.3-eksbuild.1 \ + --service-account-role-arn {arn-aws}iam::111122223333:role/AmazonEKSVPCCNIRole \ + --resolve-conflicts PRESERVE --configuration-values '{"env":{"AWS_VPC_K8S_CNI_EXTERNALSNAT":"true"}}' +---- ++ +It might take several seconds for the update to complete. +. Confirm that the add-on version was updated. Replace [.replaceable]`my-cluster` with the name of your cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks describe-addon --cluster-name my-cluster --addon-name vpc-cni +---- ++ +It might take several seconds for the update to complete. ++ +An example output is as follows. ++ +[source,json,subs="verbatim,attributes",role="nocopy"] +---- +{ + "addon": { + "addonName": "vpc-cni", + "clusterName": "my-cluster", + "status": "ACTIVE", + "addonVersion": "v1.20.3-eksbuild.1", + "health": { + "issues": [] + }, + "addonArn": "{arn-aws}eks:region:111122223333:addon/my-cluster/vpc-cni/74c33d2f-b4dc-8718-56e7-9fdfa65d14a9", + "createdAt": "2023-04-12T18:25:19.319000+00:00", + "modifiedAt": "2023-04-12T18:40:28.683000+00:00", + "serviceAccountRoleArn": "{arn-aws}iam::111122223333:role/AmazonEKSVPCCNIRole", + "tags": {}, + "configurationValues": "{\"env\":{\"AWS_VPC_K8S_CNI_EXTERNALSNAT\":\"true\"}}" + } +} +---- \ No newline at end of file diff --git a/latest/ug/nodes/al2023.adoc b/latest/ug/nodes/al2023.adoc index 0f44be2ff..110382ccd 100644 --- a/latest/ug/nodes/al2023.adoc +++ b/latest/ug/nodes/al2023.adoc @@ -1,17 +1,16 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[al2023,al2023.title]] +[#al2023] = Upgrade from Amazon Linux 2 to Amazon Linux 2023 -:info_titleabbrev: Upgrade from AL2 to AL2023 - -include::../attributes.txt[] +:info_titleabbrev: Upgrade to AL2023 [abstract] -- AL2023 is a new Linux-based operating system designed to provide a secure, stable, and high-performance environment for your cloud applications. -- -The Amazon EKS optimized AMIs are available in two families based on AL2 and AL2023. AL2023 is a new Linux-based operating system designed to provide a secure, stable, and high-performance environment for your cloud applications. It's the next generation of Amazon Linux from Amazon Web Services and is available across all supported Amazon EKS versions, including versions `1.23` and `1.24` in extended support. +The Amazon EKS optimized AMIs are available in two families based on AL2 and AL2023. AL2023 is a new Linux-based operating system designed to provide a secure, stable, and high-performance environment for your cloud applications. It's the next generation of Amazon Linux from Amazon Web Services and is available across all supported Amazon EKS versions. AL2023 offers several improvements over AL2. For a full comparison, see link:linux/al2023/ug/compare-with-al2.html[Comparing AL2 and Amazon Linux 2023,type="documentation"] in the _Amazon Linux 2023 User Guide_. Several packages have been added, upgraded, and removed from AL2. It's highly recommended to test your applications with AL2023 before upgrading. For a list of all package changes in AL2023, see link:linux/al2023/release-notes/compare-packages.html[Package changes in Amazon Linux 2023,type="documentation"] in the _Amazon Linux 2023 Release Notes_. @@ -32,18 +31,45 @@ spec: cidr: 10.100.0.0/16 ---- + -In AL2, the metadata from these parameters was discovered from the Amazon EKS `DescribeCluster` API call. With AL2023, this behavior has changed since the additional API call risks throttling during large node scale ups. This change doesn't affect you if you're using managed node groups without a launch template or if you're using [.noloc]`Karpenter`. For more information on `certificateAuthority` and service `cidr`, see ` link:eks/latest/APIReference/API_DescribeCluster.html[DescribeCluster,type="documentation"]` in the _Amazon EKS API Reference_. -* [.noloc]`Docker` isn't supported in AL2023 for all supported Amazon EKS versions. Support for [.noloc]`Docker` has ended and been removed with Amazon EKS version `1.24` or greater in AL2. For more information on deprecation, see <>. +In AL2, the metadata from these parameters was discovered from the Amazon EKS `DescribeCluster` API call. With AL2023, this behavior has changed since the additional API call risks throttling during large node scale ups. This change doesn't affect you if you're using managed node groups without a launch template or if you're using Karpenter. For more information on `certificateAuthority` and service `cidr`, see link:eks/latest/APIReference/API_DescribeCluster.html[`DescribeCluster`,type="documentation"] in the _Amazon EKS API Reference_. +* For AL2023, `nodeadm` also changes the format to apply parameters to the `kubelet` for each node using https://awslabs.github.io/amazon-eks-ami/nodeadm/doc/api/#nodeconfigspec[`NodeConfigSpec`]. In AL2, this was done with the `--kubelet-extra-args` parameter. This is commonly used to add labels and taints to nodes. An example below shows applying `maxPods` and `--node-labels` to the node. ++ +[source,yaml,subs="verbatim,attributes"] +---- +--- +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: test-cluster + apiServerEndpoint: https://example.com + certificateAuthority: Y2VydGlmaWNhdGVBdXRob3JpdHk= + cidr: 10.100.0.0/16 + kubelet: + config: + maxPods: 110 + flags: + - --node-labels=karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=test +---- ++ * Amazon VPC CNI version `1.16.2` or greater is required for AL2023. * AL2023 requires `IMDSv2` by default. `IMDSv2` has several benefits that help improve security posture. It uses a session-oriented authentication method that requires the creation of a secret token in a simple HTTP PUT request to start the session. A session's token can be valid for anywhere between 1 second and 6 hours. For more information on how to transition from `IMDSv1` to `IMDSv2`, see link:AWSEC2/latest/UserGuide/instance-metadata-transition-to-version-2.html[Transition to using Instance Metadata Service Version 2,type="documentation"] and link:security/get-the-full-benefits-of-imdsv2-and-disable-imdsv1-across-your-aws-infrastructure[Get the full benefits of IMDSv2 and disable IMDSv1 across your {aws} infrastructure,type="blog"]. If you would like to use `IMDSv1`, you can still do so by manually overriding the settings using instance metadata option launch properties. + -NOTE: For `IMDSv2`, the default hop count for managed node groups is set to 1. This means that containers won't have access to the node's credentials using IMDS. If you require container access to the node's credentials, you can still do so by manually overriding the `HttpPutResponseHopLimit` in a link:AWSCloudFormation/latest/UserGuide/aws-properties-ec2-launchtemplate-metadataoptions.html[custom Amazon EC2 launch template,type="documentation"], increasing it to 2.Alternatively, you can use <> to provide credentials instead of `IMDSv2`. +[NOTE] +==== +For `IMDSv2` with AL2023, the default hop count for managed node groups can vary: + +* When not using a launch template, the default is set to `1`. This means that containers won't have access to the node's credentials using IMDS. If you require container access to the node's credentials, you can still do so by using a link:AWSCloudFormation/latest/UserGuide/aws-properties-ec2-launchtemplate-metadataoptions.html[custom Amazon EC2 launch template,type="documentation"]. +* When using a custom AMI in a launch template, the default `HttpPutResponseHopLimit` is set to `2`. You can manually override the `HttpPutResponseHopLimit` in the launch template. + +Alternatively, you can use <> to provide credentials instead of `IMDSv2`. +==== * AL2023 features the next generation of unified control group hierarchy (`cgroupv2`). `cgroupv2` is used to implement a container runtime, and by `systemd`. While AL2023 still includes code that can make the system run using `cgroupv1`, this isn't a recommended or supported configuration. This configuration will be completely removed in a future major release of Amazon Linux. * `eksctl` version `0.176.0` or greater is required for `eksctl` to support AL2023. For previously existing managed node groups, you can either perform an in-place upgrade or a blue/green upgrade depending on how you're using a launch template: * If you're using a custom AMI with a managed node group, you can perform an in-place upgrade by swapping the AMI ID in the launch template. You should ensure that your applications and any user data transfer over to AL2023 first before performing this upgrade strategy. -* If you're using managed node groups with either the standard launch template or with a custom launch template that doesn't specify the AMI ID, you're required to upgrade using a blue/green strategy. A blue/green upgrade is typically more complex and involves creating an entirely new node group where you would specify AL2023 as the AMI type. The new node group will need to then be carefully configured to ensure that all custom data from the AL2 node group is compatible with the new OS. Once the new node group has been tested and validated with your applications, [.noloc]`Pods` can be migrated from the old node group to the new node group. Once the migration is completed, you can delete the old node group. +* If you're using managed node groups with either the standard launch template or with a custom launch template that doesn't specify the AMI ID, you're required to upgrade using a blue/green strategy. A blue/green upgrade is typically more complex and involves creating an entirely new node group where you would specify AL2023 as the AMI type. The new node group will need to then be carefully configured to ensure that all custom data from the AL2 node group is compatible with the new OS. Once the new node group has been tested and validated with your applications, Pods can be migrated from the old node group to the new node group. Once the migration is completed, you can delete the old node group. -If you're using [.noloc]`Karpenter` and want to use AL2023, you'll need to modify the `EC2NodeClass` `amiFamily` field with AL2023. By default, Drift is enabled in [.noloc]`Karpenter`. This means that once the `amiFamily` field has been changed, [.noloc]`Karpenter` will automatically update your worker nodes to the latest AMI when available. +If you're using Karpenter and want to use AL2023, you'll need to modify the `EC2NodeClass` `amiFamily` field with AL2023. By default, Drift is enabled in Karpenter. This means that once the `amiFamily` field has been changed, Karpenter will automatically update your worker nodes to the latest AMI when available. diff --git a/latest/ug/nodes/auto-get-logs.adoc b/latest/ug/nodes/auto-get-logs.adoc index be4fac095..bc36721dd 100644 --- a/latest/ug/nodes/auto-get-logs.adoc +++ b/latest/ug/nodes/auto-get-logs.adoc @@ -1,11 +1,10 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[auto-get-logs,auto-get-logs.title]] +[#auto-get-logs] = Retrieve node logs for a managed node using kubectl and S3 :info_titleabbrev: Get node logs -include::../attributes.txt[] - [abstract] -- Learn how to retrieve node logs for an Amazon EKS managed node that has the node monitoring agent. @@ -31,7 +30,7 @@ If you don't already have an S3 bucket to store the logs, create one. Use the fo [source,bash,subs="verbatim,attributes,quotes"] ---- -aws s3api create-bucket --bucket [.replaceable]`bucket-name` +aws s3api create-bucket --bucket [.replaceable]`bucket-name` --region [.replaceable]`region-code` --create-bucket-configuration LocationConstraint=[.replaceable]`region-code` ---- == Step 2: Create pre-signed S3 URL for HTTP Put @@ -45,15 +44,15 @@ The logs will be returned as a gzip tarball, with the `.tar.gz` extension. You must use the {aws} API or a SDK to create the pre-signed S3 upload URL for EKS to upload the log file. You cannot create a pre-signed S3 upload URL using the {aws} CLI. ==== -. Determine where in the bucket you want to store the logs. For example, you might use `2024-11-12/logs1.tar.gz` as the key. -. Save the following Python code to the file `presign-upload.py`. Replace `` and ``. The key should end with `.tar.gz`. +. Determine where in the bucket you want to store the logs. For example, you might use [.replaceable]`2024-11-12/logs1.tar.gz` as the key. +. Save the following Python code to the file [.replaceable]`presign-upload.py`. Replace [.replaceable]`` and [.replaceable]``. The key should end with `.tar.gz`. + [source,python,subs="verbatim,attributes"] ---- import boto3; print(boto3.client('s3').generate_presigned_url( ClientMethod='put_object', - Params={'Bucket': '', 'Key': ''}, - ExpiresIn=1000 + Params={'Bucket': '[.replaceable]``', 'Key': '[.replaceable]``'}, + ExpiresIn=[.replaceable]`1000` )) ---- . Run the script with @@ -79,7 +78,7 @@ resource's name, and providing a HTTP PUT URL destination. apiVersion: eks.amazonaws.com/v1alpha1 kind: NodeDiagnostic metadata: - name: [.replaceable]`node-name` + name: [.replaceable]`` spec: logCapture: destination: [.replaceable]`http-put-destination` @@ -102,7 +101,7 @@ You can check on the Status of the collection by describing the [source,bash,subs="verbatim,attributes,quotes"] ---- -kubectl describe nodediagnostics.eks.amazonaws.com/[.replaceable]`node-name` +kubectl describe nodediagnostics.eks.amazonaws.com/[.replaceable]`` ---- == Step 4: Download logs from S3 @@ -112,7 +111,7 @@ Wait approximately one minute before attempting to download the logs. Then, use [source,bash,subs="verbatim,attributes,quotes"] ---- # Once NodeDiagnostic shows Success status, download the logs -aws s3 cp s3://[.replaceable]`bucket-name`/[.replaceable]`key` ./node-logs.tar.gz +aws s3 cp s3://[.replaceable]``/[.replaceable]`key` ./[.replaceable]``.tar.gz ---- == Step 5: Clean up NodeDiagnostic resource @@ -124,5 +123,5 @@ artifacts [source,bash,subs="verbatim,attributes,quotes"] ---- # Delete the NodeDiagnostic resource -kubectl delete nodediagnostics.eks.amazonaws.com/[.replaceable]`node-name` +kubectl delete nodediagnostics.eks.amazonaws.com/[.replaceable]`` ---- diff --git a/latest/ug/nodes/bottlerocket-compliance-support.adoc b/latest/ug/nodes/bottlerocket-compliance-support.adoc index 2977db31c..55936ae29 100644 --- a/latest/ug/nodes/bottlerocket-compliance-support.adoc +++ b/latest/ug/nodes/bottlerocket-compliance-support.adoc @@ -1,18 +1,18 @@ -//!!NODE_ROOT
include::../attributes.txt[] + [.topic] -[[bottlerocket-compliance-support,bottlerocket-compliance-support.title]] -= Meet compliance requirements with [.noloc]`Bottlerocket` +[#bottlerocket-compliance-support] += Meet compliance requirements with Bottlerocket :info_titleabbrev: Compliance support [abstract] -- -[.noloc]`Bottlerocket` complies with recommendations defined by various organizations. +Bottlerocket complies with recommendations defined by various organizations. -- -[.noloc]`Bottlerocket` complies with recommendations defined by various organizations: +Bottlerocket complies with recommendations defined by various organizations: -* There is a https://www.cisecurity.org/benchmark/bottlerocket[CIS Benchmark] defined for [.noloc]`Bottlerocket`. In a default configuration, [.noloc]`Bottlerocket` image has most of the controls required by CIS Level 1 configuration profile. You can implement the controls required for a CIS Level 2 configuration profile. For more information, see link:containers/validating-amazon-eks-optimized-bottlerocket-ami-against-the-cis-benchmark[Validating Amazon EKS optimized Bottlerocket AMI against the CIS Benchmark,type="blog"] on the {aws} blog. -* The optimized feature set and reduced attack surface means that [.noloc]`Bottlerocket` instances require less configuration to satisfy PCI DSS requirements. The https://www.cisecurity.org/benchmark/bottlerocket[CIS Benchmark for Bottlerocket] is an excellent resource for hardening guidance, and supports your requirements for secure configuration standards under PCI DSS requirement 2.2. You can also leverage https://opensearch.org/blog/technical-post/2022/07/bottlerocket-k8s-fluent-bit/[Fluent Bit] to support your requirements for operating system level audit logging under PCI DSS requirement 10.2. {aws} publishes new (patched) [.noloc]`Bottlerocket` instances periodically to help you meet PCI DSS requirement 6.2 (for v3.2.1) and requirement 6.3.3 (for v4.0). -* [.noloc]`Bottlerocket` is an HIPAA-eligible feature authorized for use with regulated workloads for both Amazon EC2 and Amazon EKS. For more information, see the link:pdfs/whitepapers/latest/architecting-hipaa-security-and-compliance-on-amazon-eks/architecting-hipaa-security-and-compliance-on-amazon-eks.pdf[Architecting for HIPAA Security and Compliance on Amazon EKS,type="documentation"] whitepaper. -* [.noloc]`Bottlerocket` AMIs are available that are preconfigured to use FIPS 140-3 validated cryptographic modules. This includes the Amazon Linux 2023 Kernel Crypto API Cryptographic Module and the {aws}-LC Cryptographic Module. For more information on selecting FIPS-enabled variants, see <>. +* There is a https://www.cisecurity.org/benchmark/bottlerocket[CIS Benchmark] defined for Bottlerocket. In a default configuration, Bottlerocket image has most of the controls required by CIS Level 1 configuration profile. You can implement the controls required for a CIS Level 2 configuration profile. For more information, see link:containers/validating-amazon-eks-optimized-bottlerocket-ami-against-the-cis-benchmark[Validating Amazon EKS optimized Bottlerocket AMI against the CIS Benchmark,type="blog"] on the {aws} blog. +* The optimized feature set and reduced attack surface means that Bottlerocket instances require less configuration to satisfy PCI DSS requirements. The https://www.cisecurity.org/benchmark/bottlerocket[CIS Benchmark for Bottlerocket] is an excellent resource for hardening guidance, and supports your requirements for secure configuration standards under PCI DSS requirement 2.2. You can also leverage https://opensearch.org/blog/technical-post/2022/07/bottlerocket-k8s-fluent-bit/[Fluent Bit] to support your requirements for operating system level audit logging under PCI DSS requirement 10.2. {aws} publishes new (patched) Bottlerocket instances periodically to help you meet PCI DSS requirement 6.2 (for v3.2.1) and requirement 6.3.3 (for v4.0). +* Bottlerocket is an HIPAA-eligible feature authorized for use with regulated workloads for both Amazon EC2 and Amazon EKS. For more information, see link:compliance/hipaa-eligible-services-reference/[HIPAA Eligible Services Reference,type="marketing"]. +* Bottlerocket AMIs are available that are preconfigured to use FIPS 140-3 validated cryptographic modules. This includes the Amazon Linux 2023 Kernel Crypto API Cryptographic Module and the {aws}-LC Cryptographic Module. For more information, see <>. \ No newline at end of file diff --git a/latest/ug/nodes/bottlerocket-fips-amis.adoc b/latest/ug/nodes/bottlerocket-fips-amis.adoc new file mode 100644 index 000000000..aa741b3e6 --- /dev/null +++ b/latest/ug/nodes/bottlerocket-fips-amis.adoc @@ -0,0 +1,75 @@ +include::../attributes.txt[] + +[.topic] +[#bottlerocket-fips-amis] += Make your worker nodes FIPS ready with Bottlerocket FIPS AMIs +:info_titleabbrev: Bottlerocket FIPS AMIs + +[abstract] +-- +Bottlerocket makes it easier to adhere to FIPS by offering AMIs with a FIPS kernel. +-- + +The Federal Information Processing Standard (FIPS) Publication 140-3 is a United States and Canadian government standard that specifies the security requirements for cryptographic modules that protect sensitive information. Bottlerocket makes it easier to adhere to FIPS by offering AMIs with a FIPS kernel. + +These AMIs are preconfigured to use FIPS 140-3 validated cryptographic modules. This includes the Amazon Linux 2023 Kernel Crypto API Cryptographic Module and the {aws}-LC Cryptographic Module. + +Using Bottlerocket FIPS AMIs makes your worker nodes "FIPS ready" but not automatically "FIPS-compliant". For more information, see link:compliance/fips/[Federal Information Processing Standard (FIPS) 140-3,type="marketing"]. + +== Considerations + +* If your cluster uses isolated subnets, the Amazon ECR FIPS endpoint may not be accessible. This can cause the node bootstrap to fail. Make sure that your network configuration allows access to the necessary FIPS endpoints. For more information, see link:vpc/latest/privatelink/use-resource-endpoint.html[Access a resource through a resource VPC endpoint,type="documentation"] in the _{aws} PrivateLink Guide_. +* If your cluster uses a subnet with <>, image pulls will fail because Amazon ECR FIPS endpoints are not available through PrivateLink. + +== Create a managed node group with a Bottlerocket FIPS AMI + +The Bottlerocket FIPS AMI comes in two variants to support your workloads: + +* `BOTTLEROCKET_x86_64_FIPS` +* `BOTTLEROCKET_ARM_64_FIPS` + +To create a managed node group with a Bottlerocket FIPS AMI, choose the applicable AMI type during the creation process. For more information, see <>. + +For more information on selecting FIPS-enabled variants, see <>. + +== Disable the FIPS endpoint for non-supported {aws} Regions + +Bottlerocket FIPS AMIs are supported directly in the United States, including {aws} GovCloud (US) Regions. For {aws} Regions where the AMIs are available but not supported directly, you can still use the AMIs by creating a managed node group with a launch template. + +The Bottlerocket FIPS AMI relies on the Amazon ECR FIPS endpoint during bootstrap, which are not generally available outside of the United States. To use the AMI for its FIPS kernel in {aws} Regions that don't have the Amazon ECR FIPS endpoint available, do these steps to disable the FIPS endpoint: + +. Create a new configuration file with the following content or incorporate the content into your existing configuration file. + +[source,bash,subs="verbatim,attributes"] +---- +[default] +use_fips_endpoint=false +---- + +. Encode the file content as Base64 format. +. In your launch template's `UserData`, add the following encoded string using TOML format: + +[source,bash,subs="verbatim,attributes"] +---- +[settings.aws] +config = "" +---- + +For other settings, see Bottlerocket's link:https://github.com/bottlerocket-os/bottlerocket?tab=readme-ov-file#description-of-settings[Description of settings] on GitHub. + +Here is an example of `UserData` in a launch template: + +[source,bash,subs="verbatim,attributes"] +---- +[settings] +motd = "Hello from eksctl!" +[settings.aws] +config = "W2RlZmF1bHRdCnVzZV9maXBzX2VuZHBvaW50PWZhbHNlCg==" # Base64-encoded string. +[settings.kubernetes] +api-server = "" +cluster-certificate = "" +cluster-name = "" +... +---- + +For more information on creating a launch template with user data, see <>. diff --git a/latest/ug/nodes/choosing-instance-type.adoc b/latest/ug/nodes/choosing-instance-type.adoc index 9b1fa8463..438f321b1 100644 --- a/latest/ug/nodes/choosing-instance-type.adoc +++ b/latest/ug/nodes/choosing-instance-type.adoc @@ -1,15 +1,9 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[choosing-instance-type,choosing-instance-type.title]] +[#choosing-instance-type] = Choose an optimal Amazon EC2 node instance type -:info_doctype: section -:info_title: Choose an optimal Amazon EC2 node instance type :info_titleabbrev: Amazon EC2 instance types -:keywords: choose, select, instance, type, family, group, max-pods, max pods, maximum pods -:info_abstract: Each Amazon EC2 instance type offers different compute, memory, storage, and network \ - capabilities. [abstract] -- @@ -20,56 +14,57 @@ Amazon EC2 provides a wide selection of instance types for worker nodes. Each in -* All Amazon EKS AMIs don't currently support the `g5g` and `mac` families. -* [.noloc]`Arm` and non-accelerated Amazon EKS AMIs don't support the `g3`, `g4`, `inf`, and `p` families. +* All Amazon EKS AMIs don't currently support the `mac` family. +* Arm and non-accelerated Amazon EKS AMIs don't support the `g3`, `g4`, `inf`, and `p` families. * Accelerated Amazon EKS AMIs don't support the `a`, `c`, `hpc`, `m`, and `t` families. -* For Arm-based instances, Amazon Linux 2023 (AL2023) only supports instance types that use [.noloc]`Graviton2` or later processors. AL2023 doesn't support `A1` instances. +* For Arm-based instances, Amazon Linux 2023 (AL2023) only supports instance types that use Graviton2 or later processors. AL2023 doesn't support `A1` instances. When choosing between instance types that are supported by Amazon EKS, consider the following capabilities of each type. *Number of instances in a node group*:: -In general, fewer, larger instances are better, especially if you have a lot of [.noloc]`Daemonsets`. Each instance requires API calls to the API server, so the more instances you have, the more load on the API server. +In general, fewer, larger instances are better, especially if you have a lot of Daemonsets. Each instance requires API calls to the API server, so the more instances you have, the more load on the API server. *Operating system*:: -Review the supported instance types for link:AWSEC2/latest/UserGuide/instance-types.html[Linux,type="documentation"], link:AWSEC2/latest/WindowsGuide/instance-types.html[Windows,type="documentation"], and link:bottlerocket/faqs/[Bottlerocket,type="marketing"]. Before creating [.noloc]`Windows` instances, review <>. +Review the supported instance types for link:AWSEC2/latest/UserGuide/instance-types.html[Linux,type="documentation"], link:AWSEC2/latest/WindowsGuide/instance-types.html[Windows,type="documentation"], and link:bottlerocket/faqs/[Bottlerocket,type="marketing"]. Before creating Windows instances, review <>. *Hardware architecture*:: -Do you need [.noloc]`x86` or [.noloc]`Arm`? Before deploying [.noloc]`Arm` instances, review <>. Do you need instances built on the [.noloc]`Nitro System` ( link:AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances[Linux,type="documentation"] or link:AWSEC2/latest/WindowsGuide/instance-types.html#ec2-nitro-instances[Windows,type="documentation"]) or that have link:AWSEC2/latest/WindowsGuide/accelerated-computing-instances.html[Accelerated,type="documentation"] capabilities? If you need accelerated capabilities, you can only use [.noloc]`Linux` with Amazon EKS. +Do you need x86 or Arm? Before deploying Arm instances, review <>. Do you need instances built on the Nitro System ( link:AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances[Linux,type="documentation"] or link:AWSEC2/latest/WindowsGuide/instance-types.html#ec2-nitro-instances[Windows,type="documentation"]) or that have link:AWSEC2/latest/WindowsGuide/accelerated-computing-instances.html[Accelerated,type="documentation"] capabilities? If you need accelerated capabilities, you can only use Linux with Amazon EKS. -*Maximum number of [.noloc]`Pods`*:: -Since each [.noloc]`Pod` is assigned its own IP address, the number of IP addresses supported by an instance type is a factor in determining the number of [.noloc]`Pods` that can run on the instance. To manually determine how many [.noloc]`Pods` an instance type supports, see <>. -+`NOTE: If you're using an Amazon EKS optimized Amazon Linux 2 AMI that's `v20220406` or newer, you can use a new instance type without upgrading to the latest AMI. For these AMIs, the AMI auto-calculates the necessary `max-pods` value if it isn't listed in the https://github.com/awslabs/amazon-eks-ami/blob/main/templates/shared/runtime/eni-max-pods.txt[eni-max-pods.txt] file. Instance types that are currently in preview may not be supported by Amazon EKS by default. Values for max-pods` for such types still need to be added to `eni-max-pods.txt` in our AMI. +*Maximum number of Pods*:: +Since each Pod is assigned its own IP address, the number of IP addresses supported by an instance type is a factor in determining the number of Pods that can run on the instance. To manually determine how many Pods an instance type supports, see <>. + -link:ec2/nitro/[{aws} Nitro System,type="marketing"] instance types optionally support significantly more IP addresses than non-Nitro System instance types. However, not all IP addresses assigned for an instance are available to [.noloc]`Pods`. To assign a significantly larger number of IP addresses to your instances, you must have version `1.9.0` or later of the Amazon VPC CNI add-on installed in your cluster and configured appropriately. For more information, see <>. To assign the largest number of IP addresses to your instances, you must have version `1.10.1` or later of the Amazon VPC CNI add-on installed in your cluster and deploy the cluster with the `IPv6` family. +NOTE: If you're using an Amazon EKS optimized Amazon Linux 2 AMI that's `v20220406` or newer, you can use a new instance type without upgrading to the latest AMI. For these AMIs, the AMI auto-calculates the necessary `max-pods` value if it isn't listed in the https://github.com/awslabs/amazon-eks-ami/blob/main/templates/shared/runtime/eni-max-pods.txt[eni-max-pods.txt] file. Instance types that are currently in preview may not be supported by Amazon EKS by default. Values for `max-pods` for such types still need to be added to `eni-max-pods.txt` in our AMI. ++ +link:ec2/nitro/[{aws} Nitro System,type="marketing"] instance types optionally support significantly more IP addresses than non-Nitro System instance types. However, not all IP addresses assigned for an instance are available to Pods. To assign a significantly larger number of IP addresses to your instances, you must have version `1.9.0` or later of the Amazon VPC CNI add-on installed in your cluster and configured appropriately. For more information, see <>. To assign the largest number of IP addresses to your instances, you must have version `1.10.1` or later of the Amazon VPC CNI add-on installed in your cluster and deploy the cluster with the `IPv6` family. *IP family*:: -You can use any supported instance type when using the `IPv4` family for a cluster, which allows your cluster to assign private `IPv4` addresses to your [.noloc]`Pods` and Services. But if you want to use the `IPv6` family for your cluster, then you must use link:ec2/nitro/[{aws} Nitro System,type="marketing"] instance types or bare metal instance types. Only `IPv4` is supported for [.noloc]`Windows` instances. Your cluster must be running version `1.10.1` or later of the Amazon VPC CNI add-on. For more information about using `IPv6`, see <>. +You can use any supported instance type when using the `IPv4` family for a cluster, which allows your cluster to assign private `IPv4` addresses to your Pods and Services. But if you want to use the `IPv6` family for your cluster, then you must use link:ec2/nitro/[{aws} Nitro System,type="marketing"] instance types or bare metal instance types. Only `IPv4` is supported for Windows instances. Your cluster must be running version `1.10.1` or later of the Amazon VPC CNI add-on. For more information about using `IPv6`, see <>. *Version of the Amazon VPC CNI add-on that you're running*:: -The latest version of the https://github.com/aws/amazon-vpc-cni-k8s[Amazon VPC CNI plugin for Kubernetes] supports https://github.com/aws/amazon-vpc-cni-k8s/blob/master/pkg/vpc/vpc_ip_resource_limit.go[these instance types]. You may need to update your Amazon VPC CNI add-on version to take advantage of the latest supported instance types. For more information, see <>. The latest version supports the latest features for use with Amazon EKS. Earlier versions don't support all features. You can view features supported by different versions in the https://github.com/aws/amazon-vpc-cni-k8s/blob/master/CHANGELOG.md[Changelog] on [.noloc]`GitHub`. +The latest version of the https://github.com/aws/amazon-vpc-cni-k8s[Amazon VPC CNI plugin for Kubernetes] supports https://github.com/aws/amazon-vpc-cni-k8s/blob/master/pkg/vpc/vpc_ip_resource_limit.go[these instance types]. You may need to update your Amazon VPC CNI add-on version to take advantage of the latest supported instance types. For more information, see <>. The latest version supports the latest features for use with Amazon EKS. Earlier versions don't support all features. You can view features supported by different versions in the https://github.com/aws/amazon-vpc-cni-k8s/blob/master/CHANGELOG.md[Changelog] on GitHub. *{aws} Region that you're creating your nodes in*:: Not all instance types are available in all {aws} Regions. -*Whether you're using security groups for [.noloc]`Pods`*:: -If you're using security groups for [.noloc]`Pods`, only specific instance types are supported. For more information, see <>. +*Whether you're using security groups for Pods*:: +If you're using security groups for Pods, only specific instance types are supported. For more information, see <>. -[[determine-max-pods,determine-max-pods.title]] -== Amazon EKS recommended maximum [.noloc]`Pods` for each Amazon EC2 instance type +[#determine-max-pods] +== Amazon EKS recommended maximum Pods for each Amazon EC2 instance type -Since each [.noloc]`Pod` is assigned its own IP address, the number of IP addresses supported by an instance type is a factor in determining the number of [.noloc]`Pods` that can run on the instance. Amazon EKS provides a script that you can download and run to determine the Amazon EKS recommended maximum number of [.noloc]`Pods` to run on each instance type. The script uses hardware attributes of each instance, and configuration options, to determine the maximum [.noloc]`Pods` number. You can use the number returned in these steps to enable capabilities such as <> and <>. If you're using a managed node group with multiple instance types, use a value that would work for all instance types. +Since each Pod is assigned its own IP address, the number of IP addresses supported by an instance type is a factor in determining the number of Pods that can run on the instance. Amazon EKS provides a script that you can download and run to determine the Amazon EKS recommended maximum number of Pods to run on each instance type. The script uses hardware attributes of each instance, and configuration options, to determine the maximum Pods number. You can use the number returned in these steps to enable capabilities such as <> and <>. If you're using a managed node group with multiple instance types, use a value that would work for all instance types. -. Download a script that you can use to calculate the maximum number of [.noloc]`Pods` for each instance type. +. Download a script that you can use to calculate the maximum number of Pods for each instance type. + [source,bash,subs="verbatim,attributes"] ---- @@ -95,7 +90,7 @@ An example output is as follows. 29 ---- + -You can add the following options to the script to see the maximum [.noloc]`Pods` supported when using optional capabilities. +You can add the following options to the script to see the maximum Pods supported when using optional capabilities. + ** `--cni-custom-networking-enabled` – Use this option when you want to assign IP addresses from a different subnet than your instance's. For more information, see <>. Adding this option to the previous script with the same example values yields `20`. ** `--cni-prefix-delegation-enabled` – Use this option when you want to assign significantly more IP addresses to each elastic network interface. This capability requires an Amazon Linux instance that run on the Nitro System and version `1.9.0` or later of the Amazon VPC CNI add-on. For more information, see <>. Adding this option to the previous script with the same example values yields `110`. @@ -105,7 +100,7 @@ You can also run the script with the `--help` option to see all available option [NOTE] ==== -The max [.noloc]`Pods` calculator script limits the return value to `110` based on https://github.com/kubernetes/community/blob/master/sig-scalability/configs-and-limits/thresholds.md[Kubernetes scalability thresholds] and recommended settings. If your instance type has greater than 30 vCPUs, this limit jumps to `250`, a number based on internal Amazon EKS scalability team testing. For more information, see the link:containers/amazon-vpc-cni-increases-pods-per-node-limits[Amazon VPC CNI plugin increases pods per node limits,type="blog"] blog post. +The max Pods calculator script limits the return value to `110` based on https://github.com/kubernetes/community/blob/master/sig-scalability/configs-and-limits/thresholds.md[Kubernetes scalability thresholds] and recommended settings. If your instance type has greater than 30 vCPUs, this limit jumps to `250`, a number based on internal Amazon EKS scalability team testing. For more information, see the link:containers/amazon-vpc-cni-increases-pods-per-node-limits[Amazon VPC CNI plugin increases pods per node limits,type="blog"] blog post. ==== @@ -114,4 +109,4 @@ The max [.noloc]`Pods` calculator script limits the return value to `110` based EKS Auto Mode limits the number of pods on nodes to the lower of: * 110 pods hard cap -* The result of the max pods calculation described above. +* The result of the max pods calculation described above. \ No newline at end of file diff --git a/latest/ug/nodes/create-managed-node-group.adoc b/latest/ug/nodes/create-managed-node-group.adoc index a508021a2..353c79f3f 100644 --- a/latest/ug/nodes/create-managed-node-group.adoc +++ b/latest/ug/nodes/create-managed-node-group.adoc @@ -1,17 +1,16 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[create-managed-node-group,create-managed-node-group.title]] +[#create-managed-node-group] = Create a managed node group for your cluster :info_titleabbrev: Create -include::../attributes.txt[] - [abstract] -- This topic describes how you can launch Amazon EKS managed node groups of nodes that register with your Amazon EKS cluster. -- -This topic describes how you can launch Amazon EKS managed node groups of nodes that register with your Amazon EKS cluster. After the nodes join the cluster, you can deploy [.noloc]`Kubernetes` applications to them. +This topic describes how you can launch Amazon EKS managed node groups of nodes that register with your Amazon EKS cluster. After the nodes join the cluster, you can deploy Kubernetes applications to them. If this is your first time launching an Amazon EKS managed node group, we recommend that you instead follow one of our guides in <>. These guides provide walkthroughs for creating an Amazon EKS cluster with nodes. @@ -26,9 +25,9 @@ If this is your first time launching an Amazon EKS managed node group, we recomm * An existing Amazon EKS cluster. To deploy one, see <>. * An existing IAM role for the nodes to use. To create one, see <>. If this role doesn't have either of the policies for the VPC CNI, the separate role that follows is required for the VPC CNI pods. -* (Optional, but recommended) The [.noloc]`Amazon VPC CNI plugin for Kubernetes` add-on configured with its own IAM role that has the necessary IAM policy attached to it. For more information, see <>. +* (Optional, but recommended) The Amazon VPC CNI plugin for Kubernetes add-on configured with its own IAM role that has the necessary IAM policy attached to it. For more information, see <>. * Familiarity with the considerations listed in <>. Depending on the instance type you choose, there may be additional prerequisites for your cluster and VPC. -* To add a [.noloc]`Windows` managed node group, you must first enable [.noloc]`Windows` support for your cluster. For more information, see <>. +* To add a Windows managed node group, you must first enable Windows support for your cluster. For more information, see <>. You can create a managed node group with either of the following: @@ -48,7 +47,7 @@ eksctl version For instructions on how to install or upgrade `eksctl`, see https://eksctl.io/installation[Installation] in the `eksctl` documentation. -. (Optional) If the *AmazonEKS_CNI_Policy* managed IAM policy is attached to your <>, we recommend assigning it to an IAM role that you associate to the [.noloc]`Kubernetes` `aws-node` service account instead. For more information, see <>. +. (Optional) If the *AmazonEKS_CNI_Policy* managed IAM policy is attached to your <>, we recommend assigning it to an IAM role that you associate to the Kubernetes `aws-node` service account instead. For more information, see <>. . Create a managed node group with or without using a custom launch template. Manually specifying a launch template allows for greater customization of a node group. For example, it can allow deploying a custom AMI or providing arguments to the `boostrap.sh` script in an Amazon EKS optimized AMI. For a complete list of every available option and default, enter the following command. + [source,bash,subs="verbatim,attributes"] @@ -69,19 +68,19 @@ If you don't use a custom launch template when first creating a managed node gro Replace [.replaceable]`ami-family` with an allowed keyword. For more information, see https://eksctl.io/usage/custom-ami-support/#setting-the-node-ami-family[Setting the node AMI Family] in the `eksctl` documentation. Replace [.replaceable]`my-key` with the name of your Amazon EC2 key pair or public key. This key is used to SSH into your nodes after they launch. -NOTE: For [.noloc]`Windows`, this command doesn't enable SSH. Instead, it associates your Amazon EC2 key pair with the instance and allows you to RDP into the instance. +NOTE: For Windows, this command doesn't enable SSH. Instead, it associates your Amazon EC2 key pair with the instance and allows you to RDP into the instance. -If you don't already have an Amazon EC2 key pair, you can create one in the {aws-management-console}. For [.noloc]`Linux` information, see link:AWSEC2/latest/UserGuide/ec2-key-pairs.html[Amazon EC2 key pairs and Linux instances,type="documentation"] in the _Amazon EC2 User Guide_. For [.noloc]`Windows` information, see link:AWSEC2/latest/WindowsGuide/ec2-key-pairs.html[Amazon EC2 key pairs and Windows instances,type="documentation"] in the _Amazon EC2 User Guide_. +If you don't already have an Amazon EC2 key pair, you can create one in the {aws-management-console}. For Linux information, see link:AWSEC2/latest/UserGuide/ec2-key-pairs.html[Amazon EC2 key pairs and Linux instances,type="documentation"] in the _Amazon EC2 User Guide_. For Windows information, see link:AWSEC2/latest/WindowsGuide/ec2-key-pairs.html[Amazon EC2 key pairs and Windows instances,type="documentation"] in the _Amazon EC2 User Guide_. -We recommend blocking [.noloc]`Pod` access to IMDS if the following conditions are true: +We recommend blocking Pod access to IMDS if the following conditions are true: -* You plan to assign IAM roles to all of your [.noloc]`Kubernetes` service accounts so that [.noloc]`Pods` only have the minimum permissions that they need. +* You plan to assign IAM roles to all of your Kubernetes service accounts so that Pods only have the minimum permissions that they need. -* No [.noloc]`Pods` in the cluster require access to the Amazon EC2 instance metadata service (IMDS) for other reasons, such as retrieving the current {aws} Region. +* No Pods in the cluster require access to the Amazon EC2 instance metadata service (IMDS) for other reasons, such as retrieving the current {aws} Region. For more information, see https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node[Restrict access to the instance profile assigned to the worker node]. -If you want to block [.noloc]`Pod` access to IMDS, then add the `--disable-pod-imds` option to the following command. +If you want to block Pod access to IMDS, then add the `--disable-pod-imds` option to the following command. [source,bash,subs="verbatim,attributes"] ---- @@ -98,25 +97,25 @@ eksctl create nodegroup \ --ssh-public-key my-key ---- -Your instances can optionally assign a significantly higher number of IP addresses to [.noloc]`Pods`, assign IP addresses to [.noloc]`Pods` from a different CIDR block than the instance's, and be deployed to a cluster without internet access. For more information, see <>, <>, and <> for additional options to add to the previous command. +Your instances can optionally assign a significantly higher number of IP addresses to Pods, assign IP addresses to Pods from a different CIDR block than the instance's, and be deployed to a cluster without internet access. For more information, see <>, <>, and <> for additional options to add to the previous command. -Managed node groups calculates and applies a single value for the maximum number of [.noloc]`Pods` that can run on each node of your node group, based on instance type. If you create a node group with different instance types, the smallest value calculated across all instance types is applied as the maximum number of [.noloc]`Pods` that can run on every instance type in the node group. Managed node groups calculates the value using the script referenced in <>. +Managed node groups calculates and applies a single value for the maximum number of Pods that can run on each node of your node group, based on instance type. If you create a node group with different instance types, the smallest value calculated across all instance types is applied as the maximum number of Pods that can run on every instance type in the node group. Managed node groups calculates the value using the script referenced in <>. *With a launch template* The launch template must already exist and must meet the requirements specified in <>. -We recommend blocking [.noloc]`Pod` access to IMDS if the following conditions are true: +We recommend blocking Pod access to IMDS if the following conditions are true: -* You plan to assign IAM roles to all of your [.noloc]`Kubernetes` service accounts so that [.noloc]`Pods` only have the minimum permissions that they need. +* You plan to assign IAM roles to all of your Kubernetes service accounts so that Pods only have the minimum permissions that they need. -* No [.noloc]`Pods` in the cluster require access to the Amazon EC2 instance metadata service (IMDS) for other reasons, such as retrieving the current {aws} Region. +* No Pods in the cluster require access to the Amazon EC2 instance metadata service (IMDS) for other reasons, such as retrieving the current {aws} Region. For more information, see https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node[Restrict access to the instance profile assigned to the worker node]. -If you want to block [.noloc]`Pod` access to IMDS, then specify the necessary settings in the launch template. +If you want to block Pod access to IMDS, then specify the necessary settings in the launch template. [loweralpha] -.. Copy the following contents to your device. Replace the [.replaceable]`example values` and then run the modified command to create the `eks-nodegroup.yaml` file. Several settings that you specify when deploying without a launch template are moved into the launch template. If you don't specify a `version`, the template's default version is used. +.. Copy the following contents to your device. Replace the example values and then run the modified command to create the `eks-nodegroup.yaml` file. Several settings that you specify when deploying without a launch template are moved into the launch template. If you don't specify a `version`, the template's default version is used. + [source,yaml,subs="verbatim,attributes"] ---- @@ -134,11 +133,11 @@ managedNodeGroups: EOF ---- + -For a complete list of `eksctl` config file settings, see https://eksctl.io/usage/schema/[Config file schema] in the `eksctl` documentation. Your instances can optionally assign a significantly higher number of IP addresses to [.noloc]`Pods`, assign IP addresses to [.noloc]`Pods` from a different CIDR block than the instance's, use the `containerd` runtime, and be deployed to a cluster without outbound internet access. For more information, see <>, <>, <>, and <> for additional options to add to the config file. +For a complete list of `eksctl` config file settings, see https://eksctl.io/usage/schema/[Config file schema] in the `eksctl` documentation. Your instances can optionally assign a significantly higher number of IP addresses to Pods, assign IP addresses to Pods from a different CIDR block than the instance's, and be deployed to a cluster without outbound internet access. For more information, see <>, <>, and <> for additional options to add to the config file. + -If you didn't specify an AMI ID in your launch template, managed node groups calculates and applies a single value for the maximum number of [.noloc]`Pods` that can run on each node of your node group, based on instance type. If you create a node group with different instance types, the smallest value calculated across all instance types is applied as the maximum number of [.noloc]`Pods` that can run on every instance type in the node group. Managed node groups calculates the value using the script referenced in <>. +If you didn't specify an AMI ID in your launch template, managed node groups calculates and applies a single value for the maximum number of Pods that can run on each node of your node group, based on instance type. If you create a node group with different instance types, the smallest value calculated across all instance types is applied as the maximum number of Pods that can run on every instance type in the node group. Managed node groups calculates the value using the script referenced in <>. + -If you specified an AMI ID in your launch template, specify the maximum number of [.noloc]`Pods` that can run on each node of your node group if you're using <> or want to <>. For more information, see <>. +If you specified an AMI ID in your launch template, specify the maximum number of Pods that can run on each node of your node group if you're using <> or want to <>. For more information, see <>. .. Deploy the nodegroup with the following command. + @@ -156,7 +155,7 @@ eksctl create nodegroup --config-file eks-nodegroup.yaml . Choose the name of the cluster that you want to create a managed node group in. . Select the *Compute* tab. . Choose *Add node group*. -. On the *Configure node group* page, fill out the parameters accordingly, and then choose *Next*. +. On the *Configure node group* page, fill out the parameters accordingly, and then choose *Next*. + ** *Name* – Enter a unique name for your managed node group. The node group name can't be longer than 63 characters. It must start with letter or digit, but can also include hyphens and underscores for the remaining characters. ** *Node IAM role* – Choose the node instance role to use with your node group. For more information, see <>. @@ -168,48 +167,48 @@ eksctl create nodegroup --config-file eks-nodegroup.yaml **** We recommend using a role that's not currently in use by any self-managed node group. Otherwise, you plan to use with a new self-managed node group. For more information, see <>. ==== -*** *Use launch template* – (Optional) Choose if you want to use an existing launch template. Select a *Launch Template Name*. Then, select a *Launch template version*. If you don't select a version, then Amazon EKS uses the template's default version. Launch templates allow for more customization of your node group, such as allowing you to deploy a custom AMI, assign a significantly higher number of IP addresses to [.noloc]`Pods`, assign IP addresses to [.noloc]`Pods` from a different CIDR block than the instance's, enable the `containerd` runtime for your instances, and deploying nodes to a cluster without outbound internet access. For more information, see <>, <>, <>, and <>. +*** *Use launch template* – (Optional) Choose if you want to use an existing launch template. Select a *Launch Template Name*. Then, select a *Launch template version*. If you don't select a version, then Amazon EKS uses the template's default version. Launch templates allow for more customization of your node group, such as allowing you to deploy a custom AMI, assign a significantly higher number of IP addresses to Pods, assign IP addresses to Pods from a different CIDR block than the instance's, and deploying nodes to a cluster without outbound internet access. For more information, see <>, <>, and <>. + The launch template must meet the requirements in <>. If you don't use your own launch template, the Amazon EKS API creates a default Amazon EC2 launch template in your account and deploys the node group using the default launch template. + -If you implement <>, assign necessary permissions directly to every [.noloc]`Pod` that requires access to {aws} services, and no [.noloc]`Pods` in your cluster require access to IMDS for other reasons, such as retrieving the current {aws} Region, then you can also disable access to IMDS for [.noloc]`Pods` that don't use host networking in a launch template. For more information, see https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node[Restrict access to the instance profile assigned to the worker node]. -*** *[.noloc]`Kubernetes` labels* – (Optional) You can choose to apply [.noloc]`Kubernetes` labels to the nodes in your managed node group. -*** *[.noloc]`Kubernetes` taints* – (Optional) You can choose to apply [.noloc]`Kubernetes` taints to the nodes in your managed node group. The available options in the *Effect* menu are `*NoSchedule*`, `*NoExecute*`, and `*PreferNoSchedule*`. For more information, see <>. +If you implement <>, assign necessary permissions directly to every Pod that requires access to {aws} services, and no Pods in your cluster require access to IMDS for other reasons, such as retrieving the current {aws} Region, then you can also disable access to IMDS for Pods that don't use host networking in a launch template. For more information, see https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node[Restrict access to the instance profile assigned to the worker node]. +*** *Kubernetes labels* – (Optional) You can choose to apply Kubernetes labels to the nodes in your managed node group. +*** *Kubernetes taints* – (Optional) You can choose to apply Kubernetes taints to the nodes in your managed node group. The available options in the *Effect* menu are `*NoSchedule*`, `*NoExecute*`, and `*PreferNoSchedule*`. For more information, see <>. *** *Tags* – (Optional) You can choose to tag your Amazon EKS managed node group. These tags don't propagate to other resources in the node group, such as Auto Scaling groups or instances. For more information, see <>. -. On the *Set compute and scaling configuration* page, fill out the parameters accordingly, and then choose *Next*. +. On the *Set compute and scaling configuration* page, fill out the parameters accordingly, and then choose *Next*. + -*** *AMI type* – Select an AMI type.If you are deploying Arm instances, be sure to review the considerations in <> before deploying. +*** *AMI type* – Select an AMI type. If you are deploying Arm instances, be sure to review the considerations in <> before deploying. + If you specified a launch template on the previous page, and specified an AMI in the launch template, then you can't select a value. The value from the template is displayed. The AMI specified in the template must meet the requirements in <>. -*** *Capacity type* – Select a capacity type. For more information about choosing a capacity type, see <>. You can't mix different capacity types within the same node group. If you want to use both capacity types, create separate node groups, each with their own capacity and instance types. See <> for information on provisioning and scaling GPU-accelerated worker nodes. +*** *Capacity type* – Select a capacity type. For more information about choosing a capacity type, see <>. You can't mix different capacity types within the same node group. If you want to use both capacity types, create separate node groups, each with their own capacity and instance types. See link:eks/latest/userguide/capacity-blocks-mng.html[Reserve GPUs for managed node groups,type="documentation"] for information on provisioning and scaling GPU-accelerated worker nodes. *** *Instance types* – By default, one or more instance type is specified. To remove a default instance type, select the `X` on the right side of the instance type. Choose the instance types to use in your managed node group. For more information, see <>. + -The console displays a set of commonly used instance types. If you need to create a managed node group with an instance type that's not displayed, then use `eksctl`, the {aws} CLI, {aws} CloudFormation, or an SDK to create the node group. If you specified a launch template on the previous page, then you can't select a value because the instance type must be specified in the launch template. The value from the launch template is displayed. If you selected *Spot* for *Capacity type*, then we recommend specifying multiple instance types to enhance availability. +The console displays a set of commonly used instance types. If you need to create a managed node group with an instance type that's not displayed, then use `eksctl`, the {aws} CLI, {aws} CloudFormation, or an SDK to create the node group. If you specified a launch template on the previous page, then you can't select a value because the instance type must be specified in the launch template. The value from the launch template is displayed. If you selected *Spot* for *Capacity type*, then we recommend specifying multiple instance types to enhance availability. *** *Disk size* – Enter the disk size (in GiB) to use for your node's root volume. + If you specified a launch template on the previous page, then you can't select a value because it must be specified in the launch template. *** *Desired size* – Specify the current number of nodes that the managed node group should maintain at launch. + -NOTE: Amazon EKS doesn't automatically scale your node group in or out. However, you can configure the [.noloc]`Kubernetes` Cluster Autoscaler to do this for you. For more information, see https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md[Cluster Autoscaler on {aws}]. +NOTE: Amazon EKS doesn't automatically scale your node group in or out. However, you can configure the Kubernetes Cluster Autoscaler to do this for you. For more information, see https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md[Cluster Autoscaler on {aws}]. *** *Minimum size* – Specify the minimum number of nodes that the managed node group can scale in to. *** *Maximum size* – Specify the maximum number of nodes that the managed node group can scale out to. -*** *Node group update configuration* – (Optional) You can select the number or percentage of nodes to be updated in parallel. These nodes will be unavailable during the update. For *Maximum unavailable*, select one of the following options and specify a *Value*: +*** *Node group update configuration* – (Optional) You can select the number or percentage of nodes to be updated in parallel. These nodes will be unavailable during the update. For *Maximum unavailable*, select one of the following options and specify a *Value*: + **** *Number* – Select and specify the number of nodes in your node group that can be updated in parallel. **** *Percentage* – Select and specify the percentage of nodes in your node group that can be updated in parallel. This is useful if you have a large number of nodes in your node group. *** *Node auto repair configuration* – (Optional) If you activate the *Enable node auto repair* checkbox, Amazon EKS will automatically replace nodes when detected issues occur. For more information, see <>. -. On the *Specify networking* page, fill out the parameters accordingly, and then choose *Next*. +. On the *Specify networking* page, fill out the parameters accordingly, and then choose *Next*. + *** *Subnets* – Choose the subnets to launch your managed nodes into. + [IMPORTANT] ==== -If you are running a stateful application across multiple Availability Zones that is backed by Amazon EBS volumes and using the [.noloc]`Kubernetes` https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md[Cluster Autoscaler], you should configure multiple node groups, each scoped to a single Availability Zone. In addition, you should enable the `--balance-similar-node-groups` feature. +If you are running a stateful application across multiple Availability Zones that is backed by Amazon EBS volumes and using the Kubernetes https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md[Cluster Autoscaler], you should configure multiple node groups, each scoped to a single Availability Zone. In addition, you should enable the `--balance-similar-node-groups` feature. ==== + [IMPORTANT] ==== -**** If you choose a public subnet, and your cluster has only the public API server endpoint enabled, then the subnet must have `MapPublicIPOnLaunch` set to `true` for the instances to successfully join a cluster. If the subnet was created using `eksctl` or the <> on or after March 26, 2020, then this setting is already set to `true`. If the subnets were created with `eksctl` or the {aws} CloudFormation templates before March 26, 2020, then you need to change the setting manually. For more information, see link:vpc/latest/userguide/vpc-ip-addressing.html#subnet-public-ip[Modifying the public IPv4 addressing attribute for your subnet,type="documentation"]. +**** If you choose a public subnet, and your cluster has only the public API server endpoint enabled, then the subnet must have `MapPublicIPOnLaunch` set to `true` for the instances to successfully join a cluster. If the subnet was created using `eksctl` or the <> on or after March 26, 2020, then this setting is already set to `true`. If the subnets were created with `eksctl` or the {aws} CloudFormation templates before March 26, 2020, then you need to change the setting manually. For more information, see link:vpc/latest/userguide/vpc-ip-addressing.html#subnet-public-ip[Modifying the public IPv4 addressing attribute for your subnet,type="documentation"]. **** If you use a launch template and specify multiple network interfaces, Amazon EC2 won't auto-assign a public `IPv4` address, even if `MapPublicIpOnLaunch` is set to `true`. For nodes to join the cluster in this scenario, you must either enable the cluster's private API server endpoint, or launch nodes in a private subnet with outbound internet access provided through an alternative method, such as a NAT Gateway. For more information, see link:AWSEC2/latest/UserGuide/using-instance-addressing.html[Amazon EC2 instance IP addressing,type="documentation"] in the _Amazon EC2 User Guide_. ==== @@ -217,10 +216,10 @@ If you are running a stateful application across multiple Availability Zones tha + If you chose to use a launch template, then this option isn't shown. To enable remote access to your nodes, specify a key pair in the launch template and ensure that the proper port is open to the nodes in the security groups that you specify in the launch template. For more information, see <>. + -NOTE: For [.noloc]`Windows`, this command doesn't enable SSH. Instead, it associates your Amazon EC2 key pair with the instance and allows you to RDP into the instance. -*** For *SSH key pair* (Optional), choose an Amazon EC2 SSH key to use. For [.noloc]`Linux` information, see link:AWSEC2/latest/UserGuide/ec2-key-pairs.html[Amazon EC2 key pairs and Linux instances,type="documentation"] in the _Amazon EC2 User Guide_. For [.noloc]`Windows` information, see link:AWSEC2/latest/WindowsGuide/ec2-key-pairs.html[Amazon EC2 key pairs and Windows instances,type="documentation"] in the _Amazon EC2 User Guide_. If you chose to use a launch template, then you can't select one. When an Amazon EC2 SSH key is provided for node groups using [.noloc]`Bottlerocket` AMIs, the administrative container is also enabled. For more information, see https://github.com/bottlerocket-os/bottlerocket#admin-container[Admin container] on [.noloc]`GitHub`. +NOTE: For Windows, this command doesn't enable SSH. Instead, it associates your Amazon EC2 key pair with the instance and allows you to RDP into the instance. +*** For *SSH key pair* (Optional), choose an Amazon EC2 SSH key to use. For Linux information, see link:AWSEC2/latest/UserGuide/ec2-key-pairs.html[Amazon EC2 key pairs and Linux instances,type="documentation"] in the _Amazon EC2 User Guide_. For Windows information, see link:AWSEC2/latest/WindowsGuide/ec2-key-pairs.html[Amazon EC2 key pairs and Windows instances,type="documentation"] in the _Amazon EC2 User Guide_. If you chose to use a launch template, then you can't select one. When an Amazon EC2 SSH key is provided for node groups using Bottlerocket AMIs, the administrative container is also enabled. For more information, see https://github.com/bottlerocket-os/bottlerocket#admin-container[Admin container] on GitHub. *** For *Allow SSH remote access from*, if you want to limit access to specific instances, then select the security groups that are associated to those instances. If you don't select specific security groups, then SSH access is allowed from anywhere on the internet (`0.0.0.0/0`). -. On the *Review and create* page, review your managed node group configuration and choose *Create*. +. On the *Review and create* page, review your managed node group configuration and choose *Create*. + If nodes fail to join the cluster, then see <> in the Troubleshooting chapter. . Watch the status of your nodes and wait for them to reach the `Ready` status. @@ -229,7 +228,7 @@ If nodes fail to join the cluster, then see <> in the Troubles ---- kubectl get nodes --watch ---- -. (GPU nodes only) If you chose a GPU instance type and an Amazon EKS optimized accelerated AMI, then you must apply the https://github.com/NVIDIA/k8s-device-plugin[NVIDIA device plugin for Kubernetes] as a [.noloc]`DaemonSet` on your cluster. Replace [.replaceable]`vX.X.X` with your desired https://github.com/NVIDIA/k8s-device-plugin/releases[NVIDIA/k8s-device-plugin] version before running the following command. +. (GPU nodes only) If you chose a GPU instance type and an Amazon EKS optimized accelerated AMI, then you must apply the https://github.com/NVIDIA/k8s-device-plugin[NVIDIA device plugin for Kubernetes] as a DaemonSet on your cluster. Replace [.replaceable]`vX.X.X` with your desired https://github.com/NVIDIA/k8s-device-plugin/releases[NVIDIA/k8s-device-plugin] version before running the following command. + [source,bash,subs="verbatim,attributes"] ---- @@ -237,16 +236,16 @@ kubectl apply -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/vX.X ---- == Install Kubernetes add-ons -Now that you have a working Amazon EKS cluster with nodes, you're ready to start installing [.noloc]`Kubernetes` add-ons and deploying applications to your cluster. The following documentation topics help you to extend the functionality of your cluster. +Now that you have a working Amazon EKS cluster with nodes, you're ready to start installing Kubernetes add-ons and deploying applications to your cluster. The following documentation topics help you to extend the functionality of your cluster. -* The link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] that created the cluster is the only principal that can make calls to the [.noloc]`Kubernetes` API server with `kubectl` or the {aws-management-console}. If you want other IAM principals to have access to your cluster, then you need to add them. For more information, see <> and <>. -* We recommend blocking [.noloc]`Pod` access to IMDS if the following conditions are true: +* The link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] that created the cluster is the only principal that can make calls to the Kubernetes API server with `kubectl` or the {aws-management-console}. If you want other IAM principals to have access to your cluster, then you need to add them. For more information, see <> and <>. +* We recommend blocking Pod access to IMDS if the following conditions are true: + -** You plan to assign IAM roles to all of your [.noloc]`Kubernetes` service accounts so that [.noloc]`Pods` only have the minimum permissions that they need. -** No [.noloc]`Pods` in the cluster require access to the Amazon EC2 instance metadata service (IMDS) for other reasons, such as retrieving the current {aws} Region. +** You plan to assign IAM roles to all of your Kubernetes service accounts so that Pods only have the minimum permissions that they need. +** No Pods in the cluster require access to the Amazon EC2 instance metadata service (IMDS) for other reasons, such as retrieving the current {aws} Region. + For more information, see https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node[Restrict access to the instance profile assigned to the worker node]. -* Configure the [.noloc]`Kubernetes` https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md[Cluster Autoscaler] to automatically adjust the number of nodes in your node groups. +* Configure the Kubernetes https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md[Cluster Autoscaler] to automatically adjust the number of nodes in your node groups. * Deploy a <> to your cluster. -* <> with important tools for managing your cluster. +* <> with important tools for managing your cluster. \ No newline at end of file diff --git a/latest/ug/nodes/delete-fargate-profile.adoc b/latest/ug/nodes/delete-fargate-profile.adoc index cac678fe7..55a63a52e 100644 --- a/latest/ug/nodes/delete-fargate-profile.adoc +++ b/latest/ug/nodes/delete-fargate-profile.adoc @@ -1,16 +1,16 @@ -//!!NODE_ROOT
include::../attributes.txt[] + [.topic] -[[delete-fargate-profile,delete-fargate-profile.title]] +[#delete-fargate-profile] = Delete a Fargate profile :info_titleabbrev: Delete profiles [abstract] -- -When you delete a Fargate profile, any [.noloc]`Pods` that were scheduled onto Fargate with the profile are deleted. +When you delete a Fargate profile, any Pods that were scheduled onto Fargate with the profile are deleted. -- -This topic describes how to delete a Fargate profile. When you delete a Fargate profile, any [.noloc]`Pods` that were scheduled onto Fargate with the profile are deleted. If those [.noloc]`Pods` match another Fargate profile, then they're scheduled on Fargate with that profile. If they no longer match any Fargate profiles, then they aren't scheduled onto Fargate and might remain as pending. +This topic describes how to delete a Fargate profile. When you delete a Fargate profile, any Pods that were scheduled onto Fargate with the profile are deleted. If those Pods match another Fargate profile, then they're scheduled on Fargate with that profile. If they no longer match any Fargate profiles, then they aren't scheduled onto Fargate and might remain as pending. Only one Fargate profile in a cluster can be in the `DELETING` status at a time. Wait for a Fargate profile to finish deleting before you can delete any other profiles in that cluster. @@ -39,7 +39,7 @@ eksctl delete fargateprofile --name my-profile --cluster my-cluster . In the left navigation pane, choose *Clusters*. In the list of clusters, choose the cluster that you want to delete the Fargate profile from. . Choose the *Compute* tab. . Choose the Fargate profile to delete, and then choose *Delete*. -. On the *Delete Fargate profile* page, enter the name of the profile, and then choose *Delete*. +. On the *Delete Fargate profile* page, enter the name of the profile, and then choose *Delete*. == {aws} CLI [[awscli_delete_a_fargate_profile]] @@ -51,4 +51,4 @@ Use the following command to delete a profile from a cluster. Replace every [.re [source,bash,subs="verbatim,attributes"] ---- aws eks delete-fargate-profile --fargate-profile-name my-profile --cluster-name my-cluster ----- +---- \ No newline at end of file diff --git a/latest/ug/nodes/delete-managed-node-group.adoc b/latest/ug/nodes/delete-managed-node-group.adoc index b957db2c4..d116cfa4b 100644 --- a/latest/ug/nodes/delete-managed-node-group.adoc +++ b/latest/ug/nodes/delete-managed-node-group.adoc @@ -1,7 +1,7 @@ -//!!NODE_ROOT
include::../attributes.txt[] + [.topic] -[[delete-managed-node-group,delete-managed-node-group.title]] +[#delete-managed-node-group] = Delete a managed node group from your cluster :info_titleabbrev: Delete @@ -12,7 +12,7 @@ This topic describes how you can delete an Amazon EKS managed node group. This topic describes how you can delete an Amazon EKS managed node group. When you delete a managed node group, Amazon EKS first sets the minimum, maximum, and desired size of your Auto Scaling group to zero. This then causes your node group to scale down. -Before each instance is terminated, Amazon EKS sends a signal to drain the [.noloc]`Pods` from that node. If the [.noloc]`Pods` haven't drained after a few minutes, Amazon EKS lets Auto Scaling continue the termination of the instance. After every instance is terminated, the Auto Scaling group is deleted. +Before each instance is terminated, Amazon EKS sends a signal to drain that node. During the drain process, Kubernetes does the following for each pod on the node: runs any configured `preStop` lifecycle hooks, sends `SIGTERM` signals to the containers, then waits for the `terminationGracePeriodSeconds` for graceful shutdown. If the node hasn't been drained after 5 minutes, Amazon EKS lets Auto Scaling continue the forced termination of the instance. After all instances have been terminated, the Auto Scaling group is deleted. [IMPORTANT] ==== @@ -32,14 +32,14 @@ You can delete a managed node group with: *Delete a managed node group with `eksctl`* -Enter the following command. Replace every [.replaceable]`example value` with your own values. +Enter the following command. Replace every `` with your own values. [source,bash,subs="verbatim,attributes"] ---- eksctl delete nodegroup \ - --cluster my-cluster \ - --name my-mng \ - --region region-code + --cluster \ + --name \ + --region ---- For more options, see https://eksctl.io/usage/nodegroups/#deleting-and-draining-nodegroups[Deleting and draining nodegroups] in the `eksctl` documentation. @@ -51,24 +51,24 @@ For more options, see https://eksctl.io/usage/nodegroups/#deleting-and-draining- . Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. . On the *Clusters* page, choose the cluster that contains the node group to delete. . On the selected cluster page, choose the *Compute* tab. -. In the *Node groups* section, choose the node group to delete. Then choose *Delete*. -. In the *Delete node group* confirmation dialog box, enter the name of the node group. Then choose *Delete*. +. In the *Node groups* section, choose the node group to delete. Then choose *Delete*. +. In the *Delete node group* confirmation dialog box, enter the name of the node group. Then choose *Delete*. == {aws} CLI [[awscli-delete-managed-nodegroup]] *Delete a managed node group with {aws} CLI* -. Enter the following command. Replace every [.replaceable]`example value` with your own values. +. Enter the following command. Replace every `` with your own values. + [source,bash,subs="verbatim,attributes"] ---- aws eks delete-nodegroup \ - --cluster-name my-cluster \ - --nodegroup-name my-mng \ - --region region-code + --cluster-name \ + --nodegroup-name \ + --region ---- -. Use the arrow keys on your keyboard to scroll through the response output. Press the `q` key when you're finished. +. If `cli_pager=` is set in the CLI config, use the arrow keys on your keyboard to scroll through the response output. Press the `q` key when you're finished. + -For more options, see the `link:cli/latest/reference/eks/delete-nodegroup.html[delete-nodegroup,type="documentation"]` command in the _{aws} CLI Command Reference_. +For more options, see the `link:cli/latest/reference/eks/delete-nodegroup.html[delete-nodegroup,type="documentation"]` command in the _{aws} CLI Command Reference_. \ No newline at end of file diff --git a/latest/ug/nodes/dockershim-deprecation.adoc b/latest/ug/nodes/dockershim-deprecation.adoc deleted file mode 100644 index fe88a0687..000000000 --- a/latest/ug/nodes/dockershim-deprecation.adoc +++ /dev/null @@ -1,100 +0,0 @@ -//!!NODE_ROOT
-include::../attributes.txt[] -[.topic] -[[dockershim-deprecation,dockershim-deprecation.title]] -= Migrate from `dockershim` to `containerd` -:info_titleabbrev: Dockershim deprecation - -[abstract] --- -Starting with [.noloc]`Kubernetes` version `1.24`, Amazon EKS AMIs that are officially published only include the `containerd` runtime. --- - -[.noloc]`Kubernetes` no longer supports `dockershim`. The [.noloc]`Kubernetes` team removed the runtime in [.noloc]`Kubernetes` version `1.24`. For more information, see https://kubernetes.io/blog/2022/01/07/kubernetes-is-moving-on-from-dockershim/[Kubernetes is Moving on From Dockershim: Commitments and Next Steps] on the [.noloc]`Kubernetes` Blog. - -Amazon EKS also ended support for `dockershim` starting with the [.noloc]`Kubernetes` version `1.24` release. Amazon EKS AMIs that are officially published have `containerd` as the only runtime starting with version `1.24`. This topic covers some details, but more information is available in link:containers/all-you-need-to-know-about-moving-to-containerd-on-amazon-eks[All you need to know about moving to containerd on Amazon EKS,type="blog"]. - -There's a `kubectl` plugin that you can use to see which of your [.noloc]`Kubernetes` workloads mount the [.noloc]`Docker` socket volume. For more information, see https://github.com/aws-containers/kubectl-detector-for-docker-socket[Detector for Docker Socket (DDS)] on [.noloc]`GitHub`. Amazon EKS AMIs that run [.noloc]`Kubernetes` versions that are earlier than `1.24` use [.noloc]`Docker` as the default runtime. However, these Amazon EKS AMIs have a bootstrap flag option that you can use to test out your workloads on any supported cluster using `containerd`. For more information, see <>. - -We will continue to publish AMIs for existing [.noloc]`Kubernetes` versions until the end of their support date. For more information, see <>. If you require more time to test your workloads on `containerd`, use a supported version before `1.24`. But, when you want to upgrade official Amazon EKS AMIs to version `1.24` or later, make sure to validate that your workloads run on `containerd`. - -The `containerd` runtime provides more reliable performance and security. `containerd` is the runtime that's being standardized on across Amazon EKS. Fargate and [.noloc]`Bottlerocket` already use `containerd` only. `containerd` helps to minimize the number of Amazon EKS AMI releases that are required to address `dockershim` https://cve.mitre.org/[Common Vulnerabilities and Exposures] (CVEs). Because `dockershim` already uses `containerd` internally, you might not need to make any changes. However, there are some situations where changes might or must be required: - - - -* You must make changes to applications that mount the [.noloc]`Docker` socket. For example, container images that are built with a container are impacted. Many monitoring tools also mount the [.noloc]`Docker` socket. You might need to wait for updates or re-deploy workloads for runtime monitoring. -* You might need to make changes for applications that are reliant on specific [.noloc]`Docker` settings. For example, the `HTTPS_PROXY` protocol is no longer supported. You must update applications that use this protocol. For more information, see https://docs.docker.com/engine/reference/commandline/dockerd/[dockerd] in the [.noloc]`Docker` Documentation. -* If you use the Amazon ECR credential helper to pull images, you must switch to the `kubelet` image credential provider. For more information, see https://kubernetes.io/docs/tasks/kubelet-credential-provider/kubelet-credential-provider/[Configure a kubelet image credential provider] in the [.noloc]`Kubernetes` documentation. -* Because Amazon EKS `1.24` no longer supports [.noloc]`Docker`, some flags that the https://github.com/awslabs/amazon-eks-ami/blob/main/templates/al2/runtime/bootstrap.sh[Amazon EKS bootstrap script] previously supported are no longer supported. Before moving to Amazon EKS `1.24` or later, you must remove any reference to flags that are now unsupported: -+ -** `--container-runtime dockerd` (``containerd`` is the only supported value) -** `--enable-docker-bridge` -** `--docker-config-json` -* If you already have [.noloc]`Fluentd` configured for [.noloc]`Container Insights`, then you must migrate [.noloc]`Fluentd` to [.noloc]`Fluent Bit` before changing to `containerd`. The [.noloc]`Fluentd` parsers are configured to only parse log messages in JSON format. Unlike `dockerd`, the `containerd` container runtime has log messages that aren't in JSON format. If you don't migrate to [.noloc]`Fluent Bit`, some of the configured [.noloc]`Fluentd's` parsers will generate a massive amount of errors inside the [.noloc]`Fluentd` container. For more information on migrating, see link:AmazonCloudWatch/latest/monitoring/Container-Insights-setup-logs-FluentBit.html[Set up Fluent Bit as a DaemonSet to send logs to CloudWatch Logs,type="documentation"]. -* If you use a custom AMI and you are upgrading to Amazon EKS `1.24`, then you must make sure that IP forwarding is enabled for your worker nodes. This setting wasn't needed with [.noloc]`Docker` but is required for `containerd`. It is needed to troubleshoot [.noloc]`Pod`-to-[.noloc]`Pod`, [.noloc]`Pod`-to-external, or [.noloc]`Pod`-to-[.noloc]`apiserver` network connectivity. -+ -To verify this setting on a worker node, run either of the following commands: -+ -** `sysctl net.ipv4.ip_forward` -** `cat /proc/sys/net/ipv4/ip_forward` - -+ -If the output is `0`, then run either of the following commands to activate the `net.ipv4.ip_forward` kernel variable: -+ -** `sysctl -w net.ipv4.ip_forward=1` -** `echo 1 > /proc/sys/net/ipv4/ip_forward` - -For the setting's activation on Amazon EKS AMIs for Amazon Linux 2 in the `containerd` runtime, see `https://github.com/awslabs/amazon-eks-ami/blob/main/templates/al2/provisioners/install-worker.sh[install-worker.sh]` on [.noloc]`GitHub`. - -[[containerd-bootstrap,containerd-bootstrap.title]] -== Test Amazon Linux 2 migration from [.noloc]`Docker` to `containerd` - -For [.noloc]`Kubernetes` version `1.23`, you can use an optional bootstrap flag to enable the `containerd` runtime for Amazon EKS optimized AL2 AMIs. This feature gives you a clear path to migrate to `containerd` when updating to version `1.24` or later. Amazon EKS ended support for [.noloc]`Docker` starting with the [.noloc]`Kubernetes` version `1.24` launch. The `containerd` runtime is widely adopted in the [.noloc]`Kubernetes` community and is a graduated project with the CNCF. You can test it by adding a node group to a new or existing cluster. - -You can enable the boostrap flag by creating one of the following types of node groups. - - - -*Self-managed*:: -Create the node group using the instructions in <>. Specify an Amazon EKS optimized AMI and the following text for the `BootstrapArguments` parameter. -+ -[source,bash,subs="verbatim,attributes"] ----- ---container-runtime containerd ----- - - -*Managed*:: -If you use `eksctl`, create a file named `my-nodegroup.yaml` with the following contents. Replace every [.replaceable]`example value` with your own values. The node group name can't be longer than 63 characters. It must start with letter or digit, but can also include hyphens and underscores for the remaining characters. To retrieve an optimized AMI ID for `ami-[.replaceable]``1234567890abcdef0```, see <>. -+ -[source,yaml,subs="verbatim,attributes"] ----- -apiVersion: eksctl.io/v1alpha5 -kind: ClusterConfig -metadata: - name: my-cluster - region: region-code - version: 1.23 -managedNodeGroups: - - name: my-nodegroup - ami: ami-1234567890abcdef0 - overrideBootstrapCommand: | - #!/bin/bash - /etc/eks/bootstrap.sh my-cluster --container-runtime containerd ----- -+ -NOTE: If you launch many nodes simultaneously, you may also want to specify values for the `--apiserver-endpoint`, `--b64-cluster-ca`, and `--dns-cluster-ip` bootstrap arguments to avoid errors. For more information, see <>. -+ -Run the following command to create the node group. -+ -[source,bash,subs="verbatim,attributes"] ----- -eksctl create nodegroup -f my-nodegroup.yaml ----- -+ -If you prefer to use a different tool to create your managed node group, you must deploy the node group using a launch template. In your launch template, specify an <>, then <> and provide the following user data. This user data passes arguments into the `bootstrap.sh` file. For more information about the bootstrap file, see https://github.com/awslabs/amazon-eks-ami/blob/main/templates/al2/runtime/bootstrap.sh[bootstrap.sh] on [.noloc]`GitHub`. -+ -[source,bash,subs="verbatim,attributes"] ----- -/etc/eks/bootstrap.sh my-cluster --container-runtime containerd ----- diff --git a/latest/ug/nodes/eks-ami-build-scripts.adoc b/latest/ug/nodes/eks-ami-build-scripts.adoc index ee6387c01..3f0e0fa8f 100644 --- a/latest/ug/nodes/eks-ami-build-scripts.adoc +++ b/latest/ug/nodes/eks-ami-build-scripts.adoc @@ -1,8 +1,8 @@ -//!!NODE_ROOT
include::../attributes.txt[] + [.topic] -[[eks-ami-build-scripts,eks-ami-build-scripts.title]] -= Build a custom Amazon Linux AMI with a script +[#eks-ami-build-scripts] += Build a custom Amazon Linux AMI :info_titleabbrev: Custom builds [abstract] @@ -10,16 +10,117 @@ include::../attributes.txt[] Amazon Elastic Kubernetes Service (Amazon EKS) has open-source scripts that are used to build the Amazon EKS optimized AMI. -- -Amazon Elastic Kubernetes Service (Amazon EKS) has open-source scripts that are used to build the Amazon EKS optimized AMI. These build scripts are available https://github.com/awslabs/amazon-eks-ami[on GitHub]. +[IMPORTANT] +==== +Amazon EKS will no longer publish EKS-optimized Amazon Linux 2 (AL2) AMIs after November 26th, 2025. Additionally, Kubernetes version `1.32` is the last version for which Amazon EKS will release AL2 AMIs. From version `1.33` onwards, Amazon EKS will continue to release AL2023 and Bottlerocket based AMIs. For more information, see <>. +==== + +The Amazon EKS-optimized Amazon Linux (AL) AMIs are built on top of AL2 and AL2023, specifically for use as nodes in Amazon EKS clusters. Amazon EKS provides open-source build scripts in the https://github.com/awslabs/amazon-eks-ami[Amazon EKS AMI Build Specification] repository that you can use to view the configurations for `kubelet`, the runtime, the {aws} IAM Authenticator for Kubernetes, and build your own AL-based AMI from scratch. + +This repository contains the specialized https://github.com/awslabs/amazon-eks-ami/blob/main/templates/al2/runtime/bootstrap.sh[bootstrap script for AL2] and https://awslabs.github.io/amazon-eks-ami/nodeadm/[nodeadm tool for AL2023] that runs at boot time. These scripts configure your instance's certificate data, control plane endpoint, cluster name, and more. The scripts are considered the source of truth for Amazon EKS-optimized AMI builds, so you can follow the GitHub repository to monitor changes to our AMIs. + +When building custom AMIs with the EKS-optimized AMIs as the base, it is not recommended or supported to run an operating system upgrade (ie. `dnf upgrade`) or upgrade any of the Kubernetes or GPU packages that are included in the EKS-optimized AMIs, as this risks breaking component compatibility. If you do upgrade the operating system or packages that are included in the EKS-optimized AMIs, it is recommended to thoroughly test in a development or staging environment before deploying to production. + +When building custom AMIs for GPU instances, it is recommended to build separate custom AMIs for each instance type generation and family that you will run. The EKS-optimized accelerated AMIs selectively install drivers and packages at runtime based on the underlying instance type generation and family. For more information, see the EKS AMI scripts for https://github.com/awslabs/amazon-eks-ami/blob/main/templates/al2023/provisioners/install-nvidia-driver.sh[installation] and https://github.com/awslabs/amazon-eks-ami/blob/main/templates/al2023/runtime/gpu/nvidia-kmod-load.sh[runtime]. + +== Prerequisites + +* https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html[Install the {aws} CLI] +* https://developer.hashicorp.com/packer/downloads[Install HashiCorp Packer v1.9.4+] +* https://www.gnu.org/software/make/[Install GNU Make] + +== Quickstart + +This quickstart shows you the commands to create a custom AMI in your {aws} account. To learn more about the configurations available to customize your AMI, see the template variables on the https://awslabs.github.io/amazon-eks-ami/usage/al2023/[Amazon Linux 2023] page. + +=== Prerequisites + +Install the required https://developer.hashicorp.com/packer/integrations/hashicorp/amazon[Amazon plugin]. For example: + +[source,bash] +---- +packer plugins install github.com/hashicorp/amazon +---- + +=== Step 1. Setup your environment + +Clone or fork the official Amazon EKS AMI repository. For example: + +[source,bash] +---- +git clone https://github.com/awslabs/amazon-eks-ami.git +cd amazon-eks-ami +---- + +Verify that Packer is installed: + +[source,bash] +---- +packer --version +---- + +=== Step 2. Create a custom AMI + +The following are example commands for various custom AMIs. + +*Basic NVIDIA AL2 AMI:* + +[source,bash] +---- +make k8s=1.31 os_distro=al2 \ + enable_accelerator=nvidia \ + nvidia_driver_major_version=560 \ + enable_efa=true +---- + +*Basic NVIDIA AL2023 AMI:* + +[source,bash] +---- +make k8s=1.31 os_distro=al2023 \ + enable_accelerator=nvidia \ + nvidia_driver_major_version=560 \ + enable_efa=true +---- + +*STIG-Compliant Neuron AL2023 AMI:* + +[source,bash] +---- +make k8s=1.31 os_distro=al2023 \ + enable_accelerator=neuron \ + enable_fips=true \ + source_ami_id=ami-0abcd1234efgh5678 \ + kms_key_id=alias/aws-stig +---- + +After you run these commands, Packer will do the following: +* Launch a temporary Amazon EC2 instance. +* Install Kubernetes components, drivers, and configurations. +* Create the AMI in your {aws} account. + +The expected output should look like this: + +[source,bash] +---- +==> Wait completed after 8 minutes 42 seconds -The Amazon EKS optimized Amazon Linux AMIs are built on top of Amazon Linux 2 (AL2) and Amazon Linux 2023 (AL2023), specifically for use as a node in Amazon EKS clusters. You can use this repository to view the specifics of how the Amazon EKS team configures `kubelet`, the runtime, the {aws} IAM Authenticator for [.noloc]`Kubernetes`, and build your own Amazon Linux based AMI from scratch. +==> Builds finished. The artifacts of successful builds are: +--> amazon-ebs: AMIs were created: +us-west-2: ami-0e139a4b1a7a9a3e9 -The build scripts repository includes a https://www.packer.io/[HashiCorp packer] template and build scripts to generate an AMI. These scripts are the source of truth for Amazon EKS optimized AMI builds, so you can follow the [.noloc]`GitHub` repository to monitor changes to our AMIs. For example, perhaps you want your own AMI to use the same version of [.noloc]`Docker` that the Amazon EKS team uses for the official AMI. +--> amazon-ebs: AMIs were created: +us-west-2: ami-0e139a4b1a7a9a3e9 -The [.noloc]`GitHub` repository also contains the specialized https://github.com/awslabs/amazon-eks-ami/blob/main/templates/al2/runtime/bootstrap.sh[bootstrap script] and https://awslabs.github.io/amazon-eks-ami/nodeadm/[nodeadm script] that runs at boot time to configure your instance's certificate data, control plane endpoint, cluster name, and more. +--> amazon-ebs: AMIs were created: +us-west-2: ami-0e139a4b1a7a9a3e9 +---- -Additionally, the [.noloc]`GitHub` repository contains our Amazon EKS node {aws} CloudFormation templates. These templates make it easier to spin up an instance running an Amazon EKS optimized AMI and register it with a cluster. +=== Step 3. View default values -For more information, see the repositories on [.noloc]`GitHub` at https://github.com/awslabs/amazon-eks-ami. +To view default values and additional options, run the following command: -Amazon EKS optimized AL2 contains an optional bootstrap flag to enable the `containerd` runtime. +[source,bash] +---- +make help +---- diff --git a/latest/ug/nodes/eks-ami-deprecation-faqs.adoc b/latest/ug/nodes/eks-ami-deprecation-faqs.adoc new file mode 100644 index 000000000..63d224845 --- /dev/null +++ b/latest/ug/nodes/eks-ami-deprecation-faqs.adoc @@ -0,0 +1,157 @@ +include::../attributes.txt[] + +[.topic] +[#eks-ami-deprecation-faqs] += Guide to EKS AL2 & AL2-Accelerated AMIs transition features +:info_titleabbrev: AL2 AMI deprecation + +[abstract] +-- +This document outlines the End of Support (EOS) information for Amazon EKS AL2-optimized and AL2-accelerated AMIs. +-- + +{aws} will end support for EKS AL2-optimized and AL2-accelerated AMIs, effective November 26, 2025. While you can continue using EKS AL2 AMIs after the end-of-support (EOS) date (November 26, 2025), EKS will no longer release any new Kubernetes versions or updates to AL2 AMIs, including minor releases, patches, and bug fixes after this date. We recommend upgrading to Amazon Linux 2023 (AL2023) or Bottlerocket AMIs: + +* AL2023 enables a secure-by-default approach with preconfigured security policies, SELinux in permissive mode, IMDSv2-only mode enabled by default, optimized boot times, and improved package management for enhanced security and performance, well-suited for infrastructure requiring significant customizations like direct OS-level access or extensive node changes. +* Bottlerocket enables enhanced security, faster boot times, and a smaller attack surface for improved efficiency with its purpose-built, container-optimized design, well-suited for container-native approaches with minimal node customizations. + +Alternatively, you can <> until the EOS date (November 26, 2025), or build a custom AMI with an Amazon Linux 2 base instance until the Amazon Linux 2 EOS date (June 30, 2026). For more information, please visit https://aws.amazon.com/linux/amazon-linux-2023/faqs/[AL2023 FAQs], https://aws.amazon.com/bottlerocket/faqs/[Bottlerocket FAQs] or refer to <> or <> documentation for detailed migration guidance. + + +== Migration and support FAQs + +=== How do I migrate from my AL2 to an AL2023 AMI? + +We recommend creating and implementing a migration plan that includes thorough application workload testing and documented rollback procedures, then following the step-by-step instructions in the https://docs.aws.amazon.com/eks/latest/userguide/al2023.html[Upgrade from Amazon Linux 2 to Amazon Linux 2023] in EKS official documentation. + +=== Can I build a custom AL2 AMI past the EKS end-of-support (EOS) date for EKS optimized AL2 AMIs? + +While we recommend moving to officially supported and published EKS optimized AMIs for AL2023 or Bottlerocket, you can build custom EKS AL2-optimized and AL2-accelerated AMIs until the AL2 AMI EOS date (November 26, 2025). Alternatively, you can build a custom AMI with an Amazon Linux 2 base instance until the Amazon Linux 2 EOS date (June 30, 2026). For step-by-step instructions to build a custom EKS AL2-optimized and AL2-accelerated AMI, see https://docs.aws.amazon.com/eks/latest/userguide/eks-ami-build-scripts.html[Build a custom Amazon Linux AMI] in EKS official documentation. + +=== Does the EKS Kubernetes version support policy apply to Amazon Linux distributions? + +No. The EOS date for EKS AL2-optimized and AL2-accelerated AMIs is independent of the standard and extended support timelines for Kubernetes versions by EKS. You need to migrate to AL2023 or Bottlerocket even if you are using EKS extended support. + +=== How does the shift from cgroupv1 to cgroupv2 affect my migration? + +The https://github.com/kubernetes/enhancements/blob/master/keps/sig-node/4569-cgroup-v1-maintenance-mode/README.md[Kubernetes community] moved `cgroupv1` support (used by AL2) into maintenance mode, meaning no new features will be added and only critical security and major bug fixes will be provided. +To adopt `cgroupv2` in Kubernetes, you need to ensure compatibility across the OS, kernel, container runtime, and Kubernetes components. +This requires a Linux distribution that enables `cgroupv2` by default, such as AL2023, Bottlerocket, Red Hat Enterprise Linux (RHEL) 9+, Ubuntu 22.04+, or Debian 11+. +These distributions ship with kernel versions ≥5.8, which is the minimum requirement for `cgroupv2` support in Kubernetes. To learn more, see https://kubernetes.io/docs/concepts/architecture/cgroups/[About cgroup v2]. + +=== What do I do if I need Neuron in my custom AL2 AMI? + +You cannot run your full Neuron-powered applications natively on an AL2-based AMIs. +To leverage {aws} Neuron on an AL2 AMI, you must containerize you applications using a Neuron-supported container with a non-AL2 Linux distribution (e.g., Ubuntu 22.04, Amazon Linux 2023, etc.) and then deploy those containers on an AL2-based AMI that has the Neuron Driver (`aws-neuronx-dkms`) installed. + +=== Should I switch to a bare Amazon Linux 2 base instance after the EKS AL2 AMI EOS date (November 26, 2025)? + +Switching to a bare Amazon Linux 2 base instance lacks the specific optimizations, container runtime configurations, and customizations provided by the official EKS AL2-optimized and AL2-accelerated AMIs. Instead, if you must continue using an AL2-based solution, we recommend building a custom AMI using the EKS AMI recipes at <> or https://github.com/awslabs/amazon-eks-ami[Amazon EKS AMI Build Specification]. This ensures compatibility with your existing workloads and includes AL2 kernel updates until the Amazon Linux 2 EOS date (June 30, 2026). + +=== When building a custom AL2 AMI using the EKS AMI GitHub repository after the EKS AL2 AMI EOS date (November 26, 2025), what support is available for packages from repositories like amzn2-core and amzn2extra-docker? + +The EKS AMI recipe at https://github.com/awslabs/amazon-eks-ami[Amazon EKS AMI Build Specification] pulls packages via YUM from standard Amazon Linux 2 software such as https://docs.aws.amazon.com/linux/al2/ug/managing-software.html[amzn2-core] and https://docs.aws.amazon.com/linux/al2/ug/managing-software.html[amzn2extra-docker]. After the EKS AL2 AMI EOS date (November 26, 2025), this software will continue to be supported until the broader Amazon Linux 2 EOS date (June 30, 2026). Note that support is limited to kernel updates during this period, meaning you will need to manually manage and apply other package updates, security patches, and any non-kernel dependencies to maintain security and compatibility. + +=== Why might Java applications using older versions of JDK8 on Amazon EKS with AL2023 experience Out of Memory (OOM) exceptions and pod restarts, and how can this be resolved? + +When running on Amazon EKS nodes with AL2023, Java applications relying on JDK 8 versions prior to `jdk8u372` can cause OOM exceptions and pod restarts because the JVM is not compatible with `cgroupv2`. This issue arises specifically from the JVM's inability to detect container memory limits using `cgroupv2`, the default in Amazon Linux 2023. As a result, it bases heap allocation on the node's total memory rather than the pod's defined limit. This stems from `cgroupv2` changing the storage location for memory limit data, causing older Java versions to misread available memory and assume node-level resources. A few possible options include: + +* **Upgrade JDK version**: Upgrading to `jdk8u372` or later, or to a newer JDK version with full `cgroupv2` support, can resolve this issue. For a list of compatible Java versions that fully support `cgroupv2`, see https://kubernetes.io/docs/concepts/architecture/cgroups/[About cgroup v2]. +* **Build a custom AMI**: If you must continue using an AL2-based solution, you can build a custom AL2-based AMI (until November 26, 2025) using <> or https://github.com/awslabs/amazon-eks-ami[Amazon EKS AMI Build Specification]. For example, you can build an AL2-based v1.33 AMI (until November 26, 2025). Amazon EKS will provide AL2-based AMIs until the EKS AL2 EOS date (November 26, 2025). After the EOS date (November 26, 2025), you will need to build your own AMI. +* **Enable cgroupv1**: If you must continue using `cgroupv1`, you can enable `cgroupv1` on an EKS AL2023 AMI. To enable, run `sudo grubby --update-kernel=ALL --args="systemd.unified_cgroup_hierarchy=0"` and reboot the system (e.g., EC2 instance or node running Amazon Linux 2023). This will modify the boot parameters for the system (e.g., by adding the kernel parameter 'systemd.unified_cgroup_hierarchy=0' to the GRUB configuration, which instructs systemd to use the legacy `cgroupv1` hierarchy) and enable `cgroupv1`. Note that when you run this grubby command, you are reconfiguring the kernel to launch with `cgroupv1` enabled and `cgroupv2` disabled. Only one of these cgroup versions is used for active resource management on a node. This is not the same as running `cgroupv2` with backwards compatibility for the `cgroupv1` API. + +[WARNING] +==== +We do not recommend the continued use of `cgroupv1`. Instead, we recommend migrating to `cgroupv2`. The Kubernetes community moved `cgroupv1` support (used by AL2) into maintenance mode, meaning no new features or updates will be added and only critical security and major bug fixes will be provided. The full removal of `cgroupv1` support is expected in a future release, though a specific date for this full removal has not yet been announced. If you experience issues with `cgroupv1`, {aws} will be unable to provide support and recommend that you upgrade to `cgroupv2`. +==== + + +== Compatibility and versions + +=== Supported Kubernetes versions for AL2 AMIs + +Kubernetes version 1.32 is the last version for which Amazon EKS will release AL2 (Amazon Linux 2) AMIs. For https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html[supported] Kubernetes versions up to 1.32, EKS will continue to release AL2 AMIs (AL2_ARM_64, AL2_x86_64) and AL2-accelerated AMIs (AL2_x86_64_GPU) until November 26, 2025. After this date, EKS will stop releasing AL2-optimized and AL2-accelerated AMIs for all Kubernetes versions. Note that the EOS date for EKS AL2-optimized and AL2-accelerated AMIs is independent of the standard and extended support timelines for Kubernetes versions by EKS. + +=== Supported drivers and Linux kernel versions comparison for AL2, AL2023, and Bottlerocket AMIs + +[%header,cols="4"] +|=== +|Component +|EKS AL2 AMI +|EKS AL2023 AMI +|EKS Bottlerocket AMI + +|Base OS Compatibility +|RHEL7/CentOS 7 +|Fedora/CentOS 9 +|N/A + +|https://docs.nvidia.com/deploy/cuda-compatibility/why-cuda-compatibility.html#why-cuda-compatibility[CUDA user mode driver] +|12.x +|12.x +|12.x,13.x + +|NVIDIA GPU Driver +|R570 +|R570 +|R570, R580 + +|{aws} Neuron Driver +|2.20+ +|2.20+ +|2.20+ + +|Linux Kernel +|5.10 +|6.1, 6.12 +|6.1, 6.12 +|=== + +For more information on NVIDIA driver and CUDA compatibility, see the https://docs.nvidia.com/datacenter/tesla/drivers/index.html#supported-drivers-and-cuda-toolkit-versions[NVIDIA documentation]. + +=== {aws} Neuron compatibility with AL2 AMIs + +Starting from https://awsdocs-neuron.readthedocs-hosted.com/en/latest/release-notes/prev/rn.html#neuron-2-20-0-whatsnew[{aws} Neuron release 2.20], the Neuron Runtime (`aws-neuronx-runtime-lib`) used by EKS AL-based AMIs no longer supports Amazon Linux 2 (AL2). +The Neuron Driver (`aws-neuronx-dkms`) is now the only {aws} Neuron package that supports Amazon Linux 2. +This means you cannot run your Neuron-powered applications natively on an AL2-based AMI. +To setup Neuron on AL2023 AMIs, see the https://awsdocs-neuron.readthedocs-hosted.com/en/latest/general/setup/index.html#setup-guide-index[{aws} Neuron Setup] guide. + +=== Kubernetes compatibility with AL2 AMIs + +The Kubernetes community has moved `cgroupv1` support (used by AL2) to maintenance mode. +This means no new features will be added, and only critical security and major bug fixes will be provided. +Any Kubernetes features relying on cgroupv2, such as MemoryQoS and enhanced resource isolation, are unavailable on AL2. +Furthermore, Amazon EKS Kubernetes version 1.32 was the last version to support AL2 AMIs. +To maintain compatibility with the latest Kubernetes versions, we recommend migrating to AL2023 or Bottlerocket, which enable `cgroupv2` by default. + +=== Linux version compatibility with AL2 AMIs + +Amazon Linux 2 (AL2) is supported by {aws} until its end-of-support (EOS) date on June 30, 2026. +However, as AL2 has aged, support from the broader Linux community for new applications and functionality has become more limited. +AL2 AMIs are based on https://docs.aws.amazon.com/linux/al2/ug/kernel.html[Linux kernel 5.10], while AL2023 uses https://docs.aws.amazon.com/linux/al2023/ug/compare-with-al2-kernel.html[Linux kernel 6.1]. +Unlike AL2023, AL2 has limited support from the broader Linux community. +This means many upstream Linux packages and tools need to be backported to work with AL2's older kernel version, some modern Linux features and security improvements aren't available due to the older kernel, many open source projects have deprecated or limited support for older kernel versions like 5.10. + +=== Deprecated packages not included in AL2023 + +A few of the most common packages that are not included or which changed in AL2023, include: + +* Some https://docs.aws.amazon.com/linux/al2023/release-notes/removed-AL2023.6-AL2.html[source binary packages in Amazon Linux 2] are no longer available in Amazon Linux 2023 +* Changes in how Amazon Linux supports different versions of packages (e.g., https://repost.aws/questions/QUWGU3VFJMRSGf6MDPWn4tLg/how-to-resolve-amazon-linux-extras-in-al2023[amazon-linux-extras system]) in AL2023 +* https://docs.aws.amazon.com/linux/al2023/ug/epel.html[Extra Packages for Enterprise Linux (EPEL)] are not supported in AL2023 +* https://docs.aws.amazon.com/linux/al2023/ug/deprecated-al2.html#deprecated-32bit-rpms[32-bit applications] are not supported in AL2023 + +To learn more, see https://docs.aws.amazon.com/linux/al2023/ug/compare-with-al2.html[Comparing AL2 and AL2023]. + +=== FIPS validation comparison across AL2, AL2023, and Bottlerocket + +Amazon Linux 2 (AL2), Amazon Linux 2023 (AL2023), and Bottlerocket provide support for Federal Information Processing Standards (FIPS) compliance. + +* AL2 is certified under FIPS 140-2 and AL2023 is certified under FIPS 140-3. To enable FIPS mode on AL2023, install the necessary packages on your Amazon EC2 instance and follow the configuration steps using the instructions in https://docs.aws.amazon.com/linux/al2023/ug/fips-mode.html[Enable FIPS Mode on AL2023]. To learn more, see +https://aws.amazon.com/linux/amazon-linux-2023/faqs[AL2023 FAQs]. +* Bottlerocket provides purpose-built variants specifically for FIPS which constrain the kernel and userspace components to the use of cryptographic modules that have been submitted to the FIPS 140-3 Cryptographic Module Validation Program. + +=== EKS AMI driver and versions changelog + +For a complete list of all EKS AMI components and their versions, see https://github.com/awslabs/amazon-eks-ami/releases[Amazon EKS AMI Release Notes] on GitHub. + diff --git a/latest/ug/nodes/eks-ami-versions-bottlerocket.adoc b/latest/ug/nodes/eks-ami-versions-bottlerocket.adoc index a05207d61..fcd9d9bcd 100644 --- a/latest/ug/nodes/eks-ami-versions-bottlerocket.adoc +++ b/latest/ug/nodes/eks-ami-versions-bottlerocket.adoc @@ -1,15 +1,15 @@ -//!!NODE_ROOT
include::../attributes.txt[] + [.topic] -[[eks-ami-versions-bottlerocket,eks-ami-versions-bottlerocket.title]] -= Retrieve [.noloc]`Bottlerocket` AMI version information +[#eks-ami-versions-bottlerocket] += Retrieve Bottlerocket AMI version information :info_titleabbrev: Get version information [abstract] -- -This topic gives resources for Amazon EKS optimized [.noloc]`Bottlerocket` AMIs version information. +This topic gives resources for Amazon EKS optimized Bottlerocket AMIs version information. -- -Each [.noloc]`Bottlerocket` AMI release includes various versions of https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/[kubelet], the [.noloc]`Bottlerocket` kernel, and https://containerd.io/[containerd]. Accelerated AMI variants also include various versions of the [.noloc]`NVIDIA` driver. You can find this version information in the https://bottlerocket.dev/en/os/[OS] topic of the _Bottlerocket Documentation_. From this page, navigate to the applicable _Version Information_ sub-topic. +Each Bottlerocket AMI release includes various versions of https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/[kubelet], the Bottlerocket kernel, and https://containerd.io/[containerd]. Accelerated AMI variants also include various versions of the NVIDIA driver. You can find this version information in the https://bottlerocket.dev/en/os/[OS] topic of the _Bottlerocket Documentation_. From this page, navigate to the applicable _Version Information_ sub-topic. -The _Bottlerocket Documentation_ can sometimes lag behind the versions that are available on GitHub. You can find a list of changes for the latest versions in the https://github.com/bottlerocket-os/bottlerocket/releases[releases] on [.noloc]`GitHub`. +The _Bottlerocket Documentation_ can sometimes lag behind the versions that are available on GitHub. You can find a list of changes for the latest versions in the https://github.com/bottlerocket-os/bottlerocket/releases[releases] on GitHub. \ No newline at end of file diff --git a/latest/ug/nodes/eks-ami-versions-windows.adoc b/latest/ug/nodes/eks-ami-versions-windows.adoc index a858c1a0d..74ebf9b47 100644 --- a/latest/ug/nodes/eks-ami-versions-windows.adoc +++ b/latest/ug/nodes/eks-ami-versions-windows.adoc @@ -1,27 +1,20 @@ -//!!NODE_ROOT
include::../attributes.txt[] + [.topic] -[[eks-ami-versions-windows,eks-ami-versions-windows.title]] -= Retrieve [.noloc]`Windows` AMI version information +[#eks-ami-versions-windows] += Retrieve Windows AMI version information :info_titleabbrev: Get version information [abstract] -- -This topic lists versions of the Amazon EKS optimized [.noloc]`Windows` AMIs and their corresponding versions of `kubelet`, `containerd`, and `csi-proxy`. +This topic lists versions of the Amazon EKS optimized Windows AMIs and their corresponding versions of `kubelet`, `containerd`, and `csi-proxy`. -- -[IMPORTANT] -==== - -Extended Support for Amazon EKS optimized [.noloc]`Windows` AMIs that are published by {aws} isn't available for [.noloc]`Kubernetes` version `1.23` but is available for [.noloc]`Kubernetes` version `1.24` and higher. - -==== - -This topic lists versions of the Amazon EKS optimized [.noloc]`Windows` AMIs and their corresponding versions of https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/[kubelet], https://containerd.io/[containerd], and https://github.com/kubernetes-csi/csi-proxy[csi-proxy]. +This topic lists versions of the Amazon EKS optimized Windows AMIs and their corresponding versions of https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/[kubelet], https://containerd.io/[containerd], and https://github.com/kubernetes-csi/csi-proxy[csi-proxy]. The Amazon EKS optimized AMI metadata, including the AMI ID, for each variant can be retrieved programmatically. For more information, see <>. -AMIs are versioned by [.noloc]`Kubernetes` version and the release date of the AMI in the following format: +AMIs are versioned by Kubernetes version and the release date of the AMI in the following format: [source,none,subs="verbatim,attributes"] ---- @@ -31,20 +24,174 @@ k8s_major_version.k8s_minor_version-release_date [NOTE] ==== -Amazon EKS managed node groups support the November 2022 and later releases of the [.noloc]`Windows` AMIs. +Amazon EKS managed node groups support the November 2022 and later releases of the Windows AMIs. ==== -[[eks-ami-versions-windows-2022-core,eks-ami-versions-windows-2022-core.title]] -== Amazon EKS optimized [.noloc]`Windows` Server 2022 Core AMI +To receive notifications of all source file changes to this specific documentation page, you can subscribe to the following URL with an RSS reader: + +[source,none] +---- +https://github.com/awsdocs/amazon-eks-user-guide/commits/mainline/latest/ug/nodes/eks-ami-versions-windows.adoc.atom +---- + +[#eks-ami-versions-windows-2022-core] +== Amazon EKS optimized Windows Server 2022 Core AMI + +The following tables list the current and previous versions of the Amazon EKS optimized Windows Server 2022 Core AMI. + -The following tables list the current and previous versions of the Amazon EKS optimized [.noloc]`Windows` Server 2022 Core AMI. ==== [role="tablist"] -*[.noloc]`Kubernetes` version [.noloc]`1.31`*:: +*Kubernetes version 1.34*:: ++ +[%header,cols="5"] +|=== +|AMI version +|kubelet version +|containerd version +|csi-proxy version +|Release notes + + +|`1.34-2025.10.18` +|`1.34.1` +|`2.1.4` +|`1.2.1` +| + +|`1.34-2025.09.13` +|`1.34.0` +|`2.1.4` +|`1.2.1` +| + +|=== + +*Kubernetes version 1.33*:: ++ +[%header,cols="5"] +|=== +|AMI version +|kubelet version +|containerd version +|csi-proxy version +|Release notes + +|`1.33-2025.10.18` +|`1.33.5` +|`1.7.28` +|`1.2.1` +| + +|`1.33-2025.09.13` +|`1.33.4` +|`1.7.28` +|`1.2.1` +|Changed GMSA plugin logs to Windows Events + +|`1.33-2025.08.18` +|`1.33.3` +|`1.7.27` +|`1.2.1` +| + +|`1.33-2025.07.16` +|`1.33.1` +|`1.7.27` +|`1.2.1` +| + +|`1.33-2025.06.13` +|`1.33.1` +|`1.7.27` +|`1.2.1` +| + +|`1.33-2025.05.17` +|`1.33.1` +|`1.7.27` +|`1.2.1` +| + +|=== + +*Kubernetes version 1.32*:: ++ +[%header,cols="5"] +|=== +|AMI version +|kubelet version +|containerd version +|csi-proxy version +|Release notes + +|`1.32-2025.10.18` +|`1.32.9` +|`1.7.28` +|`1.2.1` +| + +|`1.32-2025.09.13` +|`1.32.8` +|`1.7.28` +|`1.2.1` +|Changed GMSA plugin logs to Windows Events + +|`1.32-2025.08.18` +|`1.32.7` +|`1.7.27` +|`1.2.1` +| + +|`1.32-2025.07.16` +|`1.32.5` +|`1.7.27` +|`1.2.1` +| + +|`1.32-2025.06.13` +|`1.32.5` +|`1.7.27` +|`1.2.1` +|Upgraded `containerd` to `1.7.27`. + +|`1.32-2025.05.17` +|`1.32.5` +|`1.7.20` +|`1.2.1` +| + +|`1.32-2025.04.14` +|`1.32.1` +|`1.7.20` +|`1.1.3` +| + +|`1.32-2025.03.14` +|`1.32.1` +|`1.7.20` +|`1.1.3` +| + +|`1.32-2025.02.18` +|`1.32.1` +|`1.7.20` +|`1.1.3` +| + +|`1.32-2025.01.15` +|`1.32.0` +|`1.7.20` +|`1.1.3` +| + +|=== + +*Kubernetes version 1.31*:: + -[cols="1,1,1,1,1", options="header"] +[%header,cols="5"] |=== |AMI version |kubelet version @@ -52,6 +199,72 @@ The following tables list the current and previous versions of the Amazon EKS op |csi-proxy version |Release notes +|`1.31-2025.10.18` +|`1.31.13` +|`1.7.28` +|`1.2.1` +| + +|`1.31-2025.09.13` +|`1.31.12` +|`1.7.28` +|`1.2.1` +|Changed GMSA plugin logs to Windows Events + +|`1.31-2025.08.18` +|`1.31.11` +|`1.7.27` +|`1.2.1` +| + +|`1.31-2025.07.16` +|`1.31.9` +|`1.7.27` +|`1.2.1` +| + +|`1.31-2025.06.13` +|`1.31.9` +|`1.7.27` +|`1.2.1` +|Upgraded `containerd` to `1.7.27`. + +|`1.31-2025.05.17` +|`1.31.9` +|`1.7.20` +|`1.2.1` +| + +|`1.31-2025.04.14` +|`1.31.5` +|`1.7.20` +|`1.1.3` +| + +|`1.31-2025.03.14` +|`1.31.5` +|`1.7.20` +|`1.1.3` +| + +|`1.31-2025.02.15` +|`1.31.5` +|`1.7.20` +|`1.1.3` +| + +|`1.31-2025.01.15` +|`1.31.4` +|`1.7.20` +|`1.1.3` +| + +|`1.31-2025.01.01` +|`1.31.4` +|`1.7.20` +|`1.1.3` +|Includes patches for `CVE-2024-9042`. + |`1.31-2024.12.13` |`1.31.3` |`1.7.20` @@ -81,11 +294,12 @@ The following tables list the current and previous versions of the Amazon EKS op |`1.7.20` |`1.1.3` | + |=== -*[.noloc]`Kubernetes` version [.noloc]`1.30`*:: +*Kubernetes version 1.30*:: + -[cols="1,1,1,1,1", options="header"] +[%header,cols="5"] |=== |AMI version |kubelet version @@ -93,6 +307,72 @@ The following tables list the current and previous versions of the Amazon EKS op |csi-proxy version |Release notes +|`1.30-2025.10.18` +|`1.30.14` +|`1.7.28` +|`1.2.1` +| + +|`1.30-2025.09.13` +|`1.30.14` +|`1.7.28` +|`1.2.1` +|Changed GMSA plugin logs to Windows Events + +|`1.30-2025.08.18` +|`1.30.14` +|`1.7.27` +|`1.2.1` +| + +|`1.30-2025.07.16` +|`1.30.13` +|`1.7.27` +|`1.2.1` +| + +|`1.30-2025.06.13` +|`1.30.13` +|`1.7.27` +|`1.2.1` +|Upgraded `containerd` to `1.7.27`. + +|`1.30-2025.05.17` +|`1.30.13` +|`1.7.20` +|`1.2.1` +| + +|`1.30-2025.04.14` +|`1.30.9` +|`1.7.20` +|`1.1.3` +| + +|`1.30-2025.03.14` +|`1.30.9` +|`1.7.20` +|`1.1.3` +|Upgraded `containerd` to `1.7.20`. + +|`1.30-2025.02.15` +|`1.30.9` +|`1.7.14` +|`1.1.3` +| + +|`1.30-2025.01.15` +|`1.30.8` +|`1.7.14` +|`1.1.3` +| + +|`1.30-2025.01.01` +|`1.30.8` +|`1.7.14` +|`1.1.3` +|Includes patches for `CVE-2024-9042`. + |`1.30-2024.12.11` |`1.30.7` |`1.7.14` @@ -140,11 +420,12 @@ The following tables list the current and previous versions of the Amazon EKS op |`1.6.28` |`1.1.2` | + |=== -*[.noloc]`Kubernetes` version [.noloc]`1.29`*:: +*Kubernetes version 1.29*:: + -[cols="1,1,1,1,1", options="header"] +[%header,cols="5"] |=== |AMI version |kubelet version @@ -152,6 +433,72 @@ The following tables list the current and previous versions of the Amazon EKS op |csi-proxy version |Release notes +|`1.29-2025.10.18` +|`1.29.15` +|`1.7.28` +|`1.2.1` +| + +|`1.29-2025.09.13` +|`1.29.15` +|`1.7.28` +|`1.2.1` +|Changed GMSA plugin logs to Windows Events + +|`1.29-2025.08.18` +|`1.29.15` +|`1.7.27` +|`1.2.1` +| + +|`1.29-2025.07.16` +|`1.29.15` +|`1.7.27` +|`1.2.1` +| + +|`1.29-2025.06.13` +|`1.29.15` +|`1.7.27` +|`1.2.1` +|Upgraded `containerd` to `1.7.27`. + +|`1.29-2025.05.17` +|`1.29.15` +|`1.7.20` +|`1.2.1` +| + +|`1.29-2025.04.14` +|`1.29.13` +|`1.7.20` +|`1.1.3` +| + +|`1.29-2025.03.14` +|`1.29.13` +|`1.7.20` +|`1.1.3` +|Upgraded `containerd` to `1.7.20`. + +|`1.29-2025.02.15` +|`1.29.13` +|`1.7.14` +|`1.1.3` +| + +|`1.29-2025.01.15` +|`1.29.12` +|`1.7.14` +|`1.1.3` +| + +|`1.29-2025.01.01` +|`1.29.12` +|`1.7.14` +|`1.1.3` +|Includes patches for `CVE-2024-9042`. + |`1.29-2024.12.11` |`1.29.10` |`1.7.14` @@ -228,12 +575,13 @@ The following tables list the current and previous versions of the Amazon EKS op |`1.29.0` |`1.6.18` |`1.1.2` -|Excluded Standalone [.noloc]`Windows` Update https://support.microsoft.com/en-au/topic/kb5034439-windows-recovery-environment-update-for-azure-stack-hci-version-22h2-and-windows-server-2022-january-9-2024-6f9d26e6-784c-4503-a3c6-0beedda443ca[KB5034439] on [.noloc]`Windows` Server 2022 Core AMIs. The KB applies only to [.noloc]`Windows` installations with a separate [.noloc]`WinRE` partition, which aren't included with any of our Amazon EKS Optimized [.noloc]`Windows` AMIs. +|Excluded Standalone Windows Update https://support.microsoft.com/en-au/topic/kb5034439-windows-recovery-environment-update-for-azure-stack-hci-version-22h2-and-windows-server-2022-january-9-2024-6f9d26e6-784c-4503-a3c6-0beedda443ca[KB5034439] on Windows Server 2022 Core AMIs. The KB applies only to Windows installations with a separate WinRE partition, which aren't included with any of our Amazon EKS Optimized Windows AMIs. + |=== -*[.noloc]`Kubernetes` version [.noloc]`1.28`*:: +*Kubernetes version 1.28*:: + -[cols="1,1,1,1,1", options="header"] +[%header,cols="5"] |=== |AMI version |kubelet version @@ -241,6 +589,72 @@ The following tables list the current and previous versions of the Amazon EKS op |csi-proxy version |Release notes +|`1.28-2025.10.18` +|`1.28.15` +|`1.7.28` +|`1.2.1` +| + +|`1.28-2025.09.13` +|`1.28.15` +|`1.7.28` +|`1.2.1` +|Changed GMSA plugin logs to Windows Events + +|`1.28-2025.08.18` +|`1.28.15` +|`1.7.27` +|`1.2.1` +| + +|`1.28-2025.07.16` +|`1.28.15` +|`1.7.27` +|`1.2.1` +| + +|`1.28-2025.06.13` +|`1.28.15` +|`1.7.27` +|`1.2.1` +|Upgraded `containerd` to `1.7.27`. + +|`1.28-2025.05.17` +|`1.28.15` +|`1.7.20` +|`1.2.1` +| + +|`1.28-2025.04.14` +|`1.28.15` +|`1.7.20` +|`1.1.3` +| + +|`1.28-2025.03.14` +|`1.28.15` +|`1.7.20` +|`1.1.3` +|Upgraded `containerd` to `1.7.20`. + +|`1.28-2025.02.15` +|`1.28.15` +|`1.7.14` +|`1.1.3` +| + +|`1.28-2025.01.15` +|`1.28.15` +|`1.7.14` +|`1.1.3` +| + +|`1.28-2025-01-01` +|`1.28.15` +|`1.7.14` +|`1.1.3` +|Includes patches for `CVE-2024-9042`. + |`1.28-2024.12.11` |`1.28.15` |`1.7.14` @@ -311,7 +725,7 @@ The following tables list the current and previous versions of the Amazon EKS op |`1.28.5` |`1.6.18` |`1.1.2` -|Excluded Standalone [.noloc]`Windows` Update https://support.microsoft.com/en-au/topic/kb5034439-windows-recovery-environment-update-for-azure-stack-hci-version-22h2-and-windows-server-2022-january-9-2024-6f9d26e6-784c-4503-a3c6-0beedda443ca[KB5034439] on [.noloc]`Windows` Server 2022 Core AMIs. The KB applies only to [.noloc]`Windows` installations with a separate [.noloc]`WinRE` partition, which aren't included with any of our Amazon EKS Optimized [.noloc]`Windows` AMIs. +|Excluded Standalone Windows Update https://support.microsoft.com/en-au/topic/kb5034439-windows-recovery-environment-update-for-azure-stack-hci-version-22h2-and-windows-server-2022-january-9-2024-6f9d26e6-784c-4503-a3c6-0beedda443ca[KB5034439] on Windows Server 2022 Core AMIs. The KB applies only to Windows installations with a separate WinRE partition, which aren't included with any of our Amazon EKS Optimized Windows AMIs. |`1.28-2023.12.12` |`1.28.3` @@ -342,11 +756,21 @@ The following tables list the current and previous versions of the Amazon EKS op |`1.6.6` |`1.1.2` | + |=== -*[.noloc]`Kubernetes` version [.noloc]`1.27`*:: +==== + +[#eks-ami-versions-windows-2022-full] +== Amazon EKS optimized Windows Server 2022 Full AMI + +The following tables list the current and previous versions of the Amazon EKS optimized Windows Server 2022 Full AMI. + +==== +[role="tablist"] +*Kubernetes version 1.34*:: + -[cols="1,1,1,1,1", options="header"] +[%header,cols="5"] |=== |AMI version |kubelet version @@ -354,1473 +778,72 @@ The following tables list the current and previous versions of the Amazon EKS op |csi-proxy version |Release notes -|`1.27-2024.12.11` -|`1.27.16` -|`1.7.14` -|`1.1.3` + +|`1.34-2025.10.18` +|`1.34.1` +|`2.1.4` +|`1.2.1` | -|`1.27-2024.11.12` -|`1.27.16` -|`1.7.14` -|`1.1.3` +|`1.34-2025.09.13` +|`1.34.0` +|`2.1.4` +|`1.2.1` | -|`1.27-2024.10.08` -|`1.27.16` -|`1.7.14` -|`1.1.3` +|=== + +*Kubernetes version 1.33*:: ++ +[%header,cols="5"] +|=== +|AMI version +|kubelet version +|containerd version +|csi-proxy version +|Release notes + +|`1.33-2025.10.18` +|`1.33.5` +|`1.7.28` +|`1.2.1` | -|`1.27-2024.09.10` -|`1.27.15` -|`1.7.14` -|`1.1.3` +|`1.33-2025.09.13` +|`1.33.4` +|`1.7.28` +|`1.2.1` +|Changed GMSA plugin logs to Windows Events + +|`1.33-2025.08.18` +|`1.33.3` +|`1.7.27` +|`1.2.1` | -|`1.27-2024.08.13` -|`1.27.15` -|`1.7.14` -|`1.1.3` +|`1.33-2025.07.16` +|`1.33.1` +|`1.7.27` +|`1.2.1` | -|`1.27-2024.07.10` -|`1.27.15` -|`1.7.11` -|`1.1.2` -|Includes patches for `CVE-2024-5321`. +|`1.33-2025.06.13` +|`1.33.1` +|`1.7.27` +|`1.2.1` +| -|`1.27-2024.06.17` -|`1.27.12` -|`1.7.11` -|`1.1.2` -|Upgraded `containerd` to `1.7.11`. +|`1.33-2025.05.17` +|`1.33.1` +|`1.7.27` +|`1.2.1` +| -|`1.27-2024.05.14` -|`1.27.12` -|`1.6.28` -|`1.1.2` -|Upgraded `containerd` to `1.6.28`. Upgraded `kubelet` to `1.27.12`. +|=== -|`1.27-2024.04.09` -|`1.27.9` -|`1.6.25` -|`1.1.2` -|Upgraded `containerd` to `1.6.25`. Rebuilt CNI and `csi-proxy` using `golang 1.22.1`. - -|`1.27-2024.03.12` -|`1.27.9` -|`1.6.18` -|`1.1.2` -| - -|`1.27-2024.02.13` -|`1.27.9` -|`1.6.18` -|`1.1.2` -| - -|`1.27-2024.01.11` -|`1.27.9` -|`1.6.18` -|`1.1.2` -|Excluded Standalone [.noloc]`Windows` Update https://support.microsoft.com/en-au/topic/kb5034439-windows-recovery-environment-update-for-azure-stack-hci-version-22h2-and-windows-server-2022-january-9-2024-6f9d26e6-784c-4503-a3c6-0beedda443ca[KB5034439] on [.noloc]`Windows` Server 2022 Core AMIs. The KB applies only to [.noloc]`Windows` installations with a separate [.noloc]`WinRE` partition, which aren't included with any of our Amazon EKS Optimized [.noloc]`Windows` AMIs. - -|`1.27-2023.12.12` -|`1.27.7` -|`1.6.18` -|`1.1.2` -| - -|`1.27-2023.11.14` -|`1.27.7` -|`1.6.18` -|`1.1.2` -|Includes patches for `CVE-2023-5528`. - -|`1.27-2023.10.19` -|`1.27.6` -|`1.6.18` -|`1.1.2` -|Upgraded `containerd` to `1.6.18`. Added new <> (`SERVICE_IPV4_CIDR` and `EXCLUDED_SNAT_CIDRS`). - -|`1.27-2023-09.27` -|`1.27.6` -|`1.6.6` -|`1.1.2` -|Fixed a https://github.com/advisories/GHSA-6xv5-86q9-7xr8[security advisory] in `kubelet`. - -|`1.27-2023.09.12` -|`1.27.4` -|`1.6.6` -|`1.1.2` -|Upgraded the Amazon VPC CNI plugin to use the [.noloc]`Kubernetes` connector binary, which gets the [.noloc]`Pod` IP address from the [.noloc]`Kubernetes` API server. Merged https://github.com/aws/amazon-vpc-cni-plugins/pull/100[pull request #100]. - -|`1.27-2023.08.17` -|`1.27.4` -|`1.6.6` -|`1.1.2` -|Includes patches for `CVE-2023-3676`, `CVE-2023-3893`, and `CVE-2023-3955`. - -|`1.27-2023.08.08` -|`1.27.3` -|`1.6.6` -|`1.1.1` -| - -|`1.27-2023.07.11` -|`1.27.3` -|`1.6.6` -|`1.1.1` -| - -|`1.27-2023.06.20` -|`1.27.1` -|`1.6.6` -|`1.1.1` -|Resolved issue that was causing the DNS suffix search list to be incorrectly populated. - -|`1.27-2023.06.14` -|`1.27.1` -|`1.6.6` -|`1.1.1` -|Added support for host port mapping in CNI. Merged https://github.com/aws/amazon-vpc-cni-plugins/pull/93[pull request #93]. - -|`1.27-2023.06.06` -|`1.27.1` -|`1.6.6` -|`1.1.1` -|Fixed `containers-roadmap` https://github.com/aws/containers-roadmap/issues/2042[issue #2042], which caused nodes to fail pulling private Amazon ECR images. - -|`1.27-2023.05.17` -|`1.27.1` -|`1.6.6` -|`1.1.1` -| -|=== - -*[.noloc]`Kubernetes` version [.noloc]`1.26`*:: -+ -[cols="1,1,1,1,1", options="header"] -|=== -|AMI version -|kubelet version -|containerd version -|csi-proxy version -|Release notes - -|`1.26-2024.12.11` -|`1.26.15` -|`1.7.14` -|`1.1.3` -| - -|`1.26-2024.11.12` -|`1.26.15` -|`1.7.14` -|`1.1.3` -| - -|`1.26-2024.10.08` -|`1.26.15` -|`1.7.14` -|`1.1.3` -| - -|`1.26-2024.09.10` -|`1.26.15` -|`1.7.14` -|`1.1.3` -| - -|`1.26-2024.08.13` -|`1.26.15` -|`1.7.14` -|`1.1.3` -| - -|`1.26-2024.07.10` -|`1.26.15` -|`1.7.11` -|`1.1.2` -|Includes patches for `CVE-2024-5321`. - -|`1.26-2024.06.17` -|`1.26.15` -|`1.7.11` -|`1.1.2` -|Upgraded `containerd` to `1.7.11`. - -|`1.26-2024.05.14` -|`1.26.15` -|`1.6.28` -|`1.1.2` -|Upgraded `containerd` to `1.6.28`. Upgraded `kubelet` to `1.26.15`. - -|`1.26-2024.04.09` -|`1.26.12` -|`1.6.25` -|`1.1.2` -|Upgraded `containerd` to `1.6.25`. Rebuilt CNI and `csi-proxy` using `golang 1.22.1`. - -|`1.26-2024.03.12` -|`1.26.12` -|`1.6.18` -|`1.1.2` -| - -|`1.26-2024.02.13` -|`1.26.12` -|`1.6.18` -|`1.1.2` -| - -|`1.26-2024.01.11` -|`1.26.12` -|`1.6.18` -|`1.1.2` -|Excluded Standalone [.noloc]`Windows` Update https://support.microsoft.com/en-au/topic/kb5034439-windows-recovery-environment-update-for-azure-stack-hci-version-22h2-and-windows-server-2022-january-9-2024-6f9d26e6-784c-4503-a3c6-0beedda443ca[KB5034439] on [.noloc]`Windows` Server 2022 Core AMIs. The KB applies only to [.noloc]`Windows` installations with a separate [.noloc]`WinRE` partition, which aren't included with any of our Amazon EKS Optimized [.noloc]`Windows` AMIs. - -|`1.26-2023.12.12` -|`1.26.10` -|`1.6.18` -|`1.1.2` -| - -|`1.26-2023.11.14` -|`1.26.10` -|`1.6.18` -|`1.1.2` -|Includes patches for `CVE-2023-5528`. - -|`1.26-2023.10.19` -|`1.26.9` -|`1.6.18` -|`1.1.2` -|Upgraded `containerd` to `1.6.18`. Upgraded `kubelet` to `1.26.9`. Added new <> (`SERVICE_IPV4_CIDR` and `EXCLUDED_SNAT_CIDRS`). - -|`1.26-2023.09.12` -|`1.26.7` -|`1.6.6` -|`1.1.2` -|Upgraded the Amazon VPC CNI plugin to use the [.noloc]`Kubernetes` connector binary, which gets the [.noloc]`Pod` IP address from the [.noloc]`Kubernetes` API server. Merged https://github.com/aws/amazon-vpc-cni-plugins/pull/100[pull request #100]. - -|`1.26-2023.08.17` -|`1.26.7` -|`1.6.6` -|`1.1.2` -|Includes patches for `CVE-2023-3676`, `CVE-2023-3893`, and `CVE-2023-3955`. - -|`1.26-2023.08.08` -|`1.26.6` -|`1.6.6` -|`1.1.1` -| - -|`1.26-2023.07.11` -|`1.26.6` -|`1.6.6` -|`1.1.1` -| - -|`1.26-2023.06.20` -|`1.26.4` -|`1.6.6` -|`1.1.1` -|Resolved issue that was causing the DNS suffix search list to be incorrectly populated. - -|`1.26-2023.06.14` -|`1.26.4` -|`1.6.6` -|`1.1.1` -|Upgraded [.noloc]`Kubernetes` to `1.26.4`. Added support for host port mapping in CNI. Merged https://github.com/aws/amazon-vpc-cni-plugins/pull/93[pull request #93]. - -|`1.26-2023.05.09` -|`1.26.2` -|`1.6.6` -|`1.1.1` -|Fixed a bug causing network connectivity https://github.com/aws/containers-roadmap/issues/1126[issue #1126] on pods after node restart. Introduced a new <> (`ExcludedSnatCIDRs`). - -|`1.26-2023.04.26` -|`1.26.2` -|`1.6.6` -|`1.1.1` -| - -|`1.26-2023.04.11` -|`1.26.2` -|`1.6.6` -|`1.1.1` -|Added recovery mechanism for `kubelet` and `kube-proxy` on service crash. - -|`1.26-2023.03.24` -|`1.26.2` -|`1.6.6` -|`1.1.1` -| -|=== - -*[.noloc]`Kubernetes` version [.noloc]`1.25`*:: -+ -[cols="1,1,1,1,1", options="header"] -|=== -|AMI version -|kubelet version -|containerd version -|csi-proxy version -|Release notes - -|`1.25-2024.12.13` -|`1.25.16` -|`1.7.14` -|`1.1.3` -| - -|`1.25-2024.11.12` -|`1.25.16` -|`1.7.14` -|`1.1.3` -| - -|`1.25-2024.10.08` -|`1.25.16` -|`1.7.14` -|`1.1.3` -| - -|`1.25-2024.09.10` -|`1.25.16` -|`1.7.14` -|`1.1.3` -| - -|`1.25-2024.08.13` -|`1.25.16` -|`1.7.14` -|`1.1.3` -| - -|`1.25-2024.07.10` -|`1.25.16` -|`1.7.11` -|`1.1.2` -|Includes patches for `CVE-2024-5321`. - -|`1.25-2024.06.17` -|`1.25.16` -|`1.7.11` -|`1.1.2` -|Upgraded `containerd` to `1.7.11`. - -|`1.25-2024.05.14` -|`1.25.16` -|`1.6.28` -|`1.1.2` -|Upgraded `containerd` to `1.6.28`. - -|`1.25-2024.04.09` -|`1.25.16` -|`1.6.25` -|`1.1.2` -|Upgraded `containerd` to `1.6.25`. Rebuilt CNI and `csi-proxy` using `golang 1.22.1`. - -|`1.25-2024.03.12` -|`1.25.16` -|`1.6.18` -|`1.1.2` -| - -|`1.25-2024.02.13` -|`1.25.16` -|`1.6.18` -|`1.1.2` -| - -|`1.25-2024.01.11` -|`1.25.16` -|`1.6.18` -|`1.1.2` -|Excluded Standalone [.noloc]`Windows` Update https://support.microsoft.com/en-au/topic/kb5034439-windows-recovery-environment-update-for-azure-stack-hci-version-22h2-and-windows-server-2022-january-9-2024-6f9d26e6-784c-4503-a3c6-0beedda443ca[KB5034439] on [.noloc]`Windows` Server 2022 Core AMIs. The KB applies only to [.noloc]`Windows` installations with a separate [.noloc]`WinRE` partition, which aren't included with any of our Amazon EKS Optimized [.noloc]`Windows` AMIs. - -|`1.25-2023.12.12` -|`1.25.15` -|`1.6.18` -|`1.1.2` -| - -|`1.25-2023.11.14` -|`1.25.15` -|`1.6.18` -|`1.1.2` -|Includes patches for `CVE-2023-5528`. - -|`1.25-2023.10.19` -|`1.25.14` -|`1.6.18` -|`1.1.2` -|Upgraded `containerd` to `1.6.18`. Upgraded `kubelet` to `1.25.14`. Added new <> (`SERVICE_IPV4_CIDR` and `EXCLUDED_SNAT_CIDRS`). - -|`1.25-2023.09.12` -|`1.25.12` -|`1.6.6` -|`1.1.2` -|Upgraded the Amazon VPC CNI plugin to use the [.noloc]`Kubernetes` connector binary, which gets the [.noloc]`Pod` IP address from the [.noloc]`Kubernetes` API server. Merged https://github.com/aws/amazon-vpc-cni-plugins/pull/100[pull request #100]. - -|`1.25-2023.08.17` -|`1.25.12` -|`1.6.6` -|`1.1.2` -|Includes patches for `CVE-2023-3676`, `CVE-2023-3893`, and `CVE-2023-3955`. - -|`1.25-2023.08.08` -|`1.25.9` -|`1.6.6` -|`1.1.1` -| - -|`1.25-2023.07.11` -|`1.25.9` -|`1.6.6` -|`1.1.1` -| - -|`1.25-2023.06.20` -|`1.25.9` -|`1.6.6` -|`1.1.1` -|Resolved issue that was causing the DNS suffix search list to be incorrectly populated. - -|`1.25-2023.06.14` -|`1.25.9` -|`1.6.6` -|`1.1.1` -|Upgraded [.noloc]`Kubernetes` to `1.25.9`. Added support for host port mapping in CNI. Merged https://github.com/aws/amazon-vpc-cni-plugins/pull/93[pull request #93]. - -|`1.25-2023.05.09` -|`1.25.7` -|`1.6.6` -|`1.1.1` -|Fixed a bug causing network connectivity https://github.com/aws/containers-roadmap/issues/1126[issue #1126] on pods after node restart. Introduced a new <> (`ExcludedSnatCIDRs`). - -|`1.25-2023.04.11` -|`1.25.7` -|`1.6.6` -|`1.1.1` -|Added recovery mechanism for `kubelet` and `kube-proxy` on service crash. - -|`1.25-2023.03.27` -|`1.25.6` -|`1.6.6` -|`1.1.1` -|Installed a link:containers/domainless-windows-authentication-for-amazon-eks-windows-pods[domainless gMSA plugin,type="blog"] to facilitate [.noloc]`gMSA` authentication for [.noloc]`Windows` containers on Amazon EKS. - -|`1.25-2023.03.20` -|`1.25.6` -|`1.6.6` -|`1.1.1` -| - -|`1.25-2023.02.14` -|`1.25.6` -|`1.6.6` -|`1.1.1` -| -|=== - -*[.noloc]`Kubernetes` version [.noloc]`1.24`*:: -+ -[cols="1,1,1,1,1", options="header"] -|=== -|AMI version -|kubelet version -|containerd version -|csi-proxy version -|Release notes - -|`1.24-2024.12.11` -|`1.24.17` -|`1.7.14` -|`1.1.3` -| - -|`1.24-2024.11.12` -|`1.24.17` -|`1.7.14` -|`1.1.3` -| - -|`1.24-2024.10.08` -|`1.24.17` -|`1.7.14` -|`1.1.3` -| - -|`1.24-2024.09.10` -|`1.24.17` -|`1.7.14` -|`1.1.3` -| - -|`1.24-2024.08.13` -|`1.24.17` -|`1.7.14` -|`1.1.3` -| - -|`1.24-2024.07.10` -|`1.24.17` -|`1.7.11` -|`1.1.2` -|Includes patches for `CVE-2024-5321`. - -|`1.24-2024.06.17` -|`1.24.17` -|`1.7.11` -|`1.1.2` -|Upgraded `containerd` to `1.7.11`. - -|`1.24-2024.05.14` -|`1.24.17` -|`1.6.28` -|`1.1.2` -|Upgraded `containerd` to `1.6.28`. - -|`1.24-2024.04.09` -|`1.24.17` -|`1.6.25` -|`1.1.2` -|Upgraded `containerd` to `1.6.25`. Rebuilt CNI and `csi-proxy` using `golang 1.22.1`. - -|`1.24-2024.03.12` -|`1.24.17` -|`1.6.18` -|`1.1.2` -| - -|`1.24-2024.02.13` -|`1.24.17` -|`1.6.18` -|`1.1.2` -| - -|`1.24-2024.01.11` -|`1.24.17` -|`1.6.18` -|`1.1.2` -|Excluded Standalone [.noloc]`Windows` Update https://support.microsoft.com/en-au/topic/kb5034439-windows-recovery-environment-update-for-azure-stack-hci-version-22h2-and-windows-server-2022-january-9-2024-6f9d26e6-784c-4503-a3c6-0beedda443ca[KB5034439] on [.noloc]`Windows` Server 2022 Core AMIs. The KB applies only to [.noloc]`Windows` installations with a separate [.noloc]`WinRE` partition, which aren't included with any of our Amazon EKS Optimized [.noloc]`Windows` AMIs. - -|`1.24-2023.12.12` -|`1.24.17` -|`1.6.18` -|`1.1.2` -| - -|`1.24-2023.11.14` -|`1.24.17` -|`1.6.18` -|`1.1.2` -|Includes patches for `CVE-2023-5528`. - -|`1.24-2023.10.19` -|`1.24.17` -|`1.6.18` -|`1.1.2` -|Upgraded `containerd` to `1.6.18`. Upgraded `kubelet` to `1.24.17`. Added new <> (`SERVICE_IPV4_CIDR` and `EXCLUDED_SNAT_CIDRS`). - -|`1.24-2023.09.12` -|`1.24.16` -|`1.6.6` -|`1.1.2` -|Upgraded the Amazon VPC CNI plugin to use the [.noloc]`Kubernetes` connector binary, which gets the [.noloc]`Pod` IP address from the [.noloc]`Kubernetes` API server. Merged https://github.com/aws/amazon-vpc-cni-plugins/pull/100[pull request #100]. - -|`1.24-2023.08.17` -|`1.24.16` -|`1.6.6` -|`1.1.2` -|Includes patches for `CVE-2023-3676`, `CVE-2023-3893`, and `CVE-2023-3955`. - -|`1.24-2023.08.08` -|`1.24.13` -|`1.6.6` -|`1.1.1` -| - -|`1.24-2023.07.11` -|`1.24.13` -|`1.6.6` -|`1.1.1` -| - -|`1.24-2023.06.20` -|`1.24.13` -|`1.6.6` -|`1.1.1` -|Resolved issue that was causing the DNS suffix search list to be incorrectly populated. - -|`1.24-2023.06.14` -|`1.24.13` -|`1.6.6` -|`1.1.1` -|Upgraded [.noloc]`Kubernetes` to `1.24.13`. Added support for host port mapping in CNI. Merged https://github.com/aws/amazon-vpc-cni-plugins/pull/93[pull request #93]. - -|`1.24-2023.05.09` -|`1.24.7` -|`1.6.6` -|`1.1.1` -|Fixed a bug causing network connectivity https://github.com/aws/containers-roadmap/issues/1126[issue #1126] on pods after node restart. Introduced a new <> (`ExcludedSnatCIDRs`). - -|`1.24-2023.04.11` -|`1.24.7` -|`1.6.6` -|`1.1.1` -|Added recovery mechanism for `kubelet` and `kube-proxy` on service crash. - -|`1.24-2023.03.27` -|`1.24.7` -|`1.6.6` -|`1.1.1` -|Installed a link:containers/domainless-windows-authentication-for-amazon-eks-windows-pods[domainless gMSA plugin,type="blog"] to facilitate gMSA authentication for [.noloc]`Windows` containers on Amazon EKS. - -|`1.24-2023.03.20` -|`1.24.7` -|`1.6.6` -|`1.1.1` -|[.noloc]`Kubernetes` version downgraded to `1.24.7` because `1.24.10` has a reported issue in `kube-proxy`. - -|`1.24-2023.02.14` -|`1.24.10` -|`1.6.6` -|`1.1.1` -| - -|`1.24-2023.01.23` -|`1.24.7` -|`1.6.6` -|`1.1.1` -| - -|`1.24-2023.01.11` -|`1.24.7` -|`1.6.6` -|`1.1.1` -| - -|`1.24-2022.12.13` -|`1.24.7` -|`1.6.6` -|`1.1.1` -| - -|`1.24-2022.10.11` -|`1.24.7` -|`1.6.6` -|`1.1.1` -| -|=== -==== - -[[eks-ami-versions-windows-2022-full,eks-ami-versions-windows-2022-full.title]] -== Amazon EKS optimized [.noloc]`Windows` Server 2022 Full AMI - -The following tables list the current and previous versions of the Amazon EKS optimized [.noloc]`Windows` Server 2022 Full AMI. - -==== -[role="tablist"] -*[.noloc]`Kubernetes` version [.noloc]`1.31`*:: -+ -[cols="1,1,1,1,1", options="header"] -|=== -|AMI version -|kubelet version -|containerd version -|csi-proxy version -|Release notes - -|`1.31-2024.12.13` -|`1.31.3` -|`1.7.20` -|`1.1.3` -| - -|`1.31-2024.11.12` -|`1.31.1` -|`1.7.20` -|`1.1.3` -| - -|`1.31-2024.10.08` -|`1.31.1` -|`1.7.20` -|`1.1.3` -| - -|`1.31-2024.10.01` -|`1.31.1` -|`1.7.20` -|`1.1.3` -| - -|`1.31-2024.09.10` -|`1.31.0` -|`1.7.20` -|`1.1.3` -| -|=== - - -*[.noloc]`Kubernetes` version [.noloc]`1.30`*:: -+ -[cols="1,1,1,1,1", options="header"] -|=== -|AMI version -|kubelet version -|containerd version -|csi-proxy version -|Release notes - -|`1.30-2024.12.11` -|`1.30.7` -|`1.7.14` -|`1.1.3` -| - -|`1.30-2024.11.12` -|`1.30.4` -|`1.7.14` -|`1.1.3` -| - -|`1.30-2024.10.08` -|`1.30.4` -|`1.7.14` -|`1.1.3` -| - -|`1.30-2024.09.10` -|`1.30.2` -|`1.7.14` -|`1.1.3` -| - -|`1.30-2024.08.13` -|`1.30.2` -|`1.7.14` -|`1.1.3` -| - -|`1.30-2024.07.10` -|`1.30.2` -|`1.7.14` -|`1.1.2` -|Includes patches for `CVE-2024-5321`. - -|`1.30-2024.06.17` -|`1.30.0` -|`1.7.14` -|`1.1.2` -|Upgraded `containerd` to `1.7.14`. - -|`1.30-2024.05.15` -|`1.30.0` -|`1.6.28` -|`1.1.2` -| -|=== - - -*[.noloc]`Kubernetes` version [.noloc]`1.29`*:: -+ -[cols="1,1,1,1,1", options="header"] -|=== -|AMI version -|kubelet version -|containerd version -|csi-proxy version -|Release notes - -|`1.29-2024.12.11` -|`1.29.10` -|`1.7.14` -|`1.1.3` -| - -|`1.29-2024.11.12` -|`1.29.8` -|`1.7.14` -|`1.1.3` -| - -|`1.29-2024.10.08` -|`1.29.8` -|`1.7.14` -|`1.1.3` -| - -|`1.29-2024.09.10` -|`1.29.6` -|`1.7.14` -|`1.1.3` -| - -|`1.29-2024.08.13` -|`1.29.6` -|`1.7.14` -|`1.1.3` -| - -|`1.29-2024.07.10` -|`1.29.6` -|`1.7.11` -|`1.1.2` -|Includes patches for `CVE-2024-5321`. - -|`1.29-2024.06.17` -|`1.29.3` -|`1.7.11` -|`1.1.2` -| - -|`1.29-2024.05.15` -|`1.29.3` -|`1.7.11` -|`1.1.2` -|Upgraded `containerd` to `1.7.11`. Upgraded `kubelet` to `1.29.3`. - -|`1.29-2024.04.09` -|`1.29.0` -|`1.6.28` -|`1.1.2` -|Upgraded `containerd` to `1.6.28`. Rebuilt CNI and `csi-proxy` using `golang 1.22.1`. - -|`1.29-2024.03.12` -|`1.29.0` -|`1.6.25` -|`1.1.2` -| - -|`1.29-2024.02.13` -|`1.29.0` -|`1.6.25` -|`1.1.2` -| - -|`1.29-2024.02.06` -|`1.29.0` -|`1.6.25` -|`1.1.2` -|Fixed a bug where the pause image was incorrectly deleted by `kubelet` garbage collection process. - -|`1.29-2024.01.09` -|`1.29.0` -|`1.6.18` -|`1.1.2` -| -|=== - - -*[.noloc]`Kubernetes` version [.noloc]`1.28`*:: -+ -[cols="1,1,1,1,1", options="header"] -|=== -|AMI version -|kubelet version -|containerd version -|csi-proxy version -|Release notes - -|`1.28-2024.12.11` -|`1.28.15` -|`1.7.14` -|`1.1.3` -| - -|`1.28-2024.11.12` -|`1.28.13` -|`1.7.14` -|`1.1.3` -| - -|`1.28-2024.10.08` -|`1.28.13` -|`1.7.14` -|`1.1.3` -| - -|`1.28-2024.09.10` -|`1.28.11` -|`1.7.14` -|`1.1.3` -| - -|`1.28-2024.08.13` -|`1.28.11` -|`1.7.14` -|`1.1.3` -| - -|`1.28-2024.07.10` -|`1.28.11` -|`1.7.11` -|`1.1.2` -|Includes patches for `CVE-2024-5321`. - -|`1.28-2024.06.17` -|`1.28.8` -|`1.7.11` -|`1.1.2` -|Upgraded `containerd` to `1.7.11`. - -|`1.28-2024.05.14` -|`1.28.8` -|`1.6.28` -|`1.1.2` -|Upgraded `containerd` to `1.6.28`. Upgraded `kubelet` to `1.28.8`. - -|`1.28-2024.04.09` -|`1.28.5` -|`1.6.25` -|`1.1.2` -|Upgraded `containerd` to `1.6.25`. Rebuilt CNI and `csi-proxy` using `golang 1.22.1`. - -|`1.28-2024.03.12` -|`1.28.5` -|`1.6.18` -|`1.1.2` -| - -|`1.28-2024.02.13` -|`1.28.5` -|`1.6.18` -|`1.1.2` -| - -|`1.28-2024.01.09` -|`1.28.5` -|`1.6.18` -|`1.1.2` -| - -|`1.28-2023.12.12` -|`1.28.3` -|`1.6.18` -|`1.1.2` -| - -|`1.28-2023.11.14` -|`1.28.3` -|`1.6.18` -|`1.1.2` -|Includes patches for `CVE-2023-5528`. - -|`1.28-2023.10.19` -|`1.28.2` -|`1.6.18` -|`1.1.2` -|Upgraded `containerd` to `1.6.18`. Added new <> (`SERVICE_IPV4_CIDR` and `EXCLUDED_SNAT_CIDRS`). - -|`1.28-2023-09.27` -|`1.28.2` -|`1.6.6` -|`1.1.2` -|Fixed a https://github.com/advisories/GHSA-6xv5-86q9-7xr8[security advisory] in `kubelet`. - -|`1.28-2023.09.12` -|`1.28.1` -|`1.6.6` -|`1.1.2` -| -|=== - - -*[.noloc]`Kubernetes` version [.noloc]`1.27`*:: -+ -[cols="1,1,1,1,1", options="header"] -|=== -|AMI version -|kubelet version -|containerd version -|csi-proxy version -|Release notes - -|`1.27-2024.12.11` -|`1.27.16` -|`1.7.14` -|`1.1.3` -| - -|`1.27-2024.11.12` -|`1.27.16` -|`1.7.14` -|`1.1.3` -| - -|`1.27-2024.10.08` -|`1.27.16` -|`1.7.14` -|`1.1.3` -| - -|`1.27-2024.09.10` -|`1.27.15` -|`1.7.14` -|`1.1.3` -| - -|`1.27-2024.08.13` -|`1.27.15` -|`1.7.14` -|`1.1.3` -| - -|`1.27-2024.07.10` -|`1.27.15` -|`1.7.11` -|`1.1.2` -|Includes patches for `CVE-2024-5321`. - -|`1.27-2024.06.17` -|`1.27.12` -|`1.7.11` -|`1.1.2` -|Upgraded `containerd` to `1.7.11`. - -|`1.27-2024.05.14` -|`1.27.12` -|`1.6.28` -|`1.1.2` -|Upgraded `containerd` to `1.6.28`. Upgraded `kubelet` to `1.27.12`. - -|`1.27-2024.04.09` -|`1.27.9` -|`1.6.25` -|`1.1.2` -|Upgraded `containerd` to `1.6.25`. Rebuilt CNI and `csi-proxy` using `golang 1.22.1`. - -|`1.27-2024.03.12` -|`1.27.9` -|`1.6.18` -|`1.1.2` -| - -|`1.27-2024.02.13` -|`1.27.9` -|`1.6.18` -|`1.1.2` -| - -|`1.27-2024.01.09` -|`1.27.9` -|`1.6.18` -|`1.1.2` -| - -|`1.27-2023.12.12` -|`1.27.7` -|`1.6.18` -|`1.1.2` -| - -|`1.27-2023.11.14` -|`1.27.7` -|`1.6.18` -|`1.1.2` -|Includes patches for `CVE-2023-5528`. - -|`1.27-2023.10.19` -|`1.27.6` -|`1.6.18` -|`1.1.2` -|Upgraded `containerd` to `1.6.18`. Added new <> (`SERVICE_IPV4_CIDR` and `EXCLUDED_SNAT_CIDRS`). - -|`1.27-2023-09.27` -|`1.27.6` -|`1.6.6` -|`1.1.2` -|Fixed a https://github.com/advisories/GHSA-6xv5-86q9-7xr8[security advisory] in `kubelet`. - -|`1.27-2023.09.12` -|`1.27.4` -|`1.6.6` -|`1.1.2` -|Upgraded the Amazon VPC CNI plugin to use the [.noloc]`Kubernetes` connector binary, which gets the [.noloc]`Pod` IP address from the [.noloc]`Kubernetes` API server. Merged https://github.com/aws/amazon-vpc-cni-plugins/pull/100[pull request #100]. - -|`1.27-2023.08.17` -|`1.27.4` -|`1.6.6` -|`1.1.2` -|Includes patches for `CVE-2023-3676`, `CVE-2023-3893`, and `CVE-2023-3955`. - -|`1.27-2023.08.08` -|`1.27.3` -|`1.6.6` -|`1.1.1` -| - -|`1.27-2023.07.11` -|`1.27.3` -|`1.6.6` -|`1.1.1` -| - -|`1.27-2023.06.20` -|`1.27.1` -|`1.6.6` -|`1.1.1` -|Resolved issue that was causing the DNS suffix search list to be incorrectly populated. - -|`1.27-2023.06.14` -|`1.27.1` -|`1.6.6` -|`1.1.1` -|Added support for host port mapping in CNI. Merged https://github.com/aws/amazon-vpc-cni-plugins/pull/93[pull request #93]. - -|`1.27-2023.06.06` -|`1.27.1` -|`1.6.6` -|`1.1.1` -|Fixed `containers-roadmap` https://github.com/aws/containers-roadmap/issues/2042[issue #2042], which caused nodes to fail pulling private Amazon ECR images. - -|`1.27-2023.05.18` -|`1.27.1` -|`1.6.6` -|`1.1.1` -| -|=== - - -*[.noloc]`Kubernetes` version [.noloc]`1.26`*:: -+ -[cols="1,1,1,1,1", options="header"] -|=== -|AMI version -|kubelet version -|containerd version -|csi-proxy version -|Release notes - -|`1.26-2024.12.11` -|`1.26.15` -|`1.7.14` -|`1.1.3` -| - -|`1.26-2024.11.12` -|`1.26.15` -|`1.7.14` -|`1.1.3` -| - -|`1.26-2024.10.08` -|`1.26.15` -|`1.7.14` -|`1.1.3` -| - -|`1.26-2024.09.10` -|`1.26.15` -|`1.7.14` -|`1.1.3` -| - -|`1.26-2024.08.13` -|`1.26.15` -|`1.7.14` -|`1.1.3` -| - -|`1.26-2024.07.10` -|`1.26.15` -|`1.7.11` -|`1.1.2` -|Includes patches for `CVE-2024-5321`. - -|`1.26-2024.06.17` -|`1.26.15` -|`1.7.11` -|`1.1.2` -|Upgraded `containerd` to `1.7.11`. - -|`1.26-2024.05.14` -|`1.26.15` -|`1.6.28` -|`1.1.2` -|Upgraded `containerd` to `1.6.28`. Upgraded `kubelet` to `1.26.15`. - -|`1.26-2024.04.09` -|`1.26.12` -|`1.6.25` -|`1.1.2` -|Upgraded `containerd` to `1.6.25`. Rebuilt CNI and `csi-proxy` using `golang 1.22.1`. - -|`1.26-2024.03.12` -|`1.26.12` -|`1.6.18` -|`1.1.2` -| - -|`1.26-2024.02.13` -|`1.26.12` -|`1.6.18` -|`1.1.2` -| - -|`1.26-2024.01.09` -|`1.26.12` -|`1.6.18` -|`1.1.2` -| - -|`1.26-2023.12.12` -|`1.26.10` -|`1.6.18` -|`1.1.2` -| - -|`1.26-2023.11.14` -|`1.26.10` -|`1.6.18` -|`1.1.2` -|Includes patches for `CVE-2023-5528`. - -|`1.26-2023.10.19` -|`1.26.9` -|`1.6.18` -|`1.1.2` -|Upgraded `containerd` to `1.6.18`. Upgraded `kubelet` to `1.26.9`. Added new <> (`SERVICE_IPV4_CIDR` and `EXCLUDED_SNAT_CIDRS`). - -|`1.26-2023.09.12` -|`1.26.7` -|`1.6.6` -|`1.1.2` -|Upgraded the Amazon VPC CNI plugin to use the [.noloc]`Kubernetes` connector binary, which gets the [.noloc]`Pod` IP address from the [.noloc]`Kubernetes` API server. Merged https://github.com/aws/amazon-vpc-cni-plugins/pull/100[pull request #100]. - -|`1.26-2023.08.17` -|`1.26.7` -|`1.6.6` -|`1.1.2` -|Includes patches for `CVE-2023-3676`, `CVE-2023-3893`, and `CVE-2023-3955`. - -|`1.26-2023.08.08` -|`1.26.6` -|`1.6.6` -|`1.1.1` -| - -|`1.26-2023.07.11` -|`1.26.6` -|`1.6.6` -|`1.1.1` -| - -|`1.26-2023.06.20` -|`1.26.4` -|`1.6.6` -|`1.1.1` -|Resolved issue that was causing the DNS suffix search list to be incorrectly populated. - -|`1.26-2023.06.14` -|`1.26.4` -|`1.6.6` -|`1.1.1` -|Upgraded [.noloc]`Kubernetes` to `1.26.4`. Added support for host port mapping in CNI. Merged https://github.com/aws/amazon-vpc-cni-plugins/pull/93[pull request #93]. - -|`1.26-2023.05.09` -|`1.26.2` -|`1.6.6` -|`1.1.1` -|Fixed a bug causing network connectivity https://github.com/aws/containers-roadmap/issues/1126[issue #1126] on pods after node restart. Introduced a new <> (`ExcludedSnatCIDRs`). - -|`1.26-2023.04.26` -|`1.26.2` -|`1.6.6` -|`1.1.1` -| - -|`1.26-2023.04.11` -|`1.26.2` -|`1.6.6` -|`1.1.1` -|Added recovery mechanism for `kubelet` and `kube-proxy` on service crash. - -|`1.26-2023.03.24` -|`1.26.2` -|`1.6.6` -|`1.1.1` -| -|=== - - -*[.noloc]`Kubernetes` version [.noloc]`1.25`*:: -+ -[cols="1,1,1,1,1", options="header"] -|=== -|AMI version -|kubelet version -|containerd version -|csi-proxy version -|Release notes - -|`1.25-2024.12.13` -|`1.25.16` -|`1.7.14` -|`1.1.3` -| - -|`1.25-2024.11.12` -|`1.25.16` -|`1.7.14` -|`1.1.3` -| - -|`1.25-2024.10.08` -|`1.25.16` -|`1.7.14` -|`1.1.3` -| - -|`1.25-2024.09.10` -|`1.25.16` -|`1.7.14` -|`1.1.3` -| - -|`1.25-2024.08.13` -|`1.25.16` -|`1.7.14` -|`1.1.3` -| - -|`1.25-2024.07.10` -|`1.25.16` -|`1.7.11` -|`1.1.2` -|Includes patches for `CVE-2024-5321`. - -|`1.25-2024.06.17` -|`1.25.16` -|`1.7.11` -|`1.1.2` -|Upgraded `containerd` to `1.7.11`. - -|`1.25-2024.05.14` -|`1.25.16` -|`1.6.28` -|`1.1.2` -|Upgraded `containerd` to `1.6.28`. - -|`1.25-2024.04.09` -|`1.25.16` -|`1.6.25` -|`1.1.2` -|Upgraded `containerd` to `1.6.25`. Rebuilt CNI and `csi-proxy` using `golang 1.22.1`. - -|`1.25-2024.03.12` -|`1.25.16` -|`1.6.18` -|`1.1.2` -| - -|`1.25-2024.02.13` -|`1.25.16` -|`1.6.18` -|`1.1.2` -| - -|`1.25-2024.01.09` -|`1.25.16` -|`1.6.18` -|`1.1.2` -| - -|`1.25-2023.12.12` -|`1.25.15` -|`1.6.18` -|`1.1.2` -| - -|`1.25-2023.11.14` -|`1.25.15` -|`1.6.18` -|`1.1.2` -|Includes patches for `CVE-2023-5528`. - -|`1.25-2023.10.19` -|`1.25.14` -|`1.6.18` -|`1.1.2` -|Upgraded `containerd` to `1.6.18`. Upgraded `kubelet` to `1.25.14`. Added new <> (`SERVICE_IPV4_CIDR` and `EXCLUDED_SNAT_CIDRS`). - -|`1.25-2023.09.12` -|`1.25.12` -|`1.6.6` -|`1.1.2` -|Upgraded the Amazon VPC CNI plugin to use the [.noloc]`Kubernetes` connector binary, which gets the [.noloc]`Pod` IP address from the [.noloc]`Kubernetes` API server. Merged https://github.com/aws/amazon-vpc-cni-plugins/pull/100[pull request #100]. - -|`1.25-2023.08.17` -|`1.25.12` -|`1.6.6` -|`1.1.2` -|Includes patches for `CVE-2023-3676`, `CVE-2023-3893`, and `CVE-2023-3955`. - -|`1.25-2023.08.08` -|`1.25.9` -|`1.6.6` -|`1.1.1` -| - -|`1.25-2023.07.11` -|`1.25.9` -|`1.6.6` -|`1.1.1` -| - -|`1.25-2023.06.20` -|`1.25.9` -|`1.6.6` -|`1.1.1` -|Resolved issue that was causing the DNS suffix search list to be incorrectly populated. - -|`1.25-2023.06.14` -|`1.25.9` -|`1.6.6` -|`1.1.1` -|Upgraded [.noloc]`Kubernetes` to `1.25.9`. Added support for host port mapping in CNI. Merged https://github.com/aws/amazon-vpc-cni-plugins/pull/93[pull request #93]. - -|`1.25-2023.05.09` -|`1.25.7` -|`1.6.6` -|`1.1.1` -|Fixed a bug causing network connectivity https://github.com/aws/containers-roadmap/issues/1126[issue #1126] on pods after node restart. Introduced a new <> (`ExcludedSnatCIDRs`). - -|`1.25-2023.04.11` -|`1.25.7` -|`1.6.6` -|`1.1.1` -|Added recovery mechanism for `kubelet` and `kube-proxy` on service crash. - -|`1.25-2023.03.27` -|`1.25.6` -|`1.6.6` -|`1.1.1` -|Installed a link:containers/domainless-windows-authentication-for-amazon-eks-windows-pods[domainless gMSA plugin,type="blog"] to facilitate [.noloc]`gMSA` authentication for [.noloc]`Windows` containers on Amazon EKS. - -|`1.25-2023.03.20` -|`1.25.6` -|`1.6.6` -|`1.1.1` -| - -|`1.25-2023.02.14` -|`1.25.6` -|`1.6.6` -|`1.1.1` -| -|=== - - -*[.noloc]`Kubernetes` version [.noloc]`1.24`*:: +*Kubernetes version 1.32*:: + -[cols="1,1,1,1,1", options="header"] +[%header,cols="5"] |=== |AMI version |kubelet version @@ -1828,240 +851,72 @@ The following tables list the current and previous versions of the Amazon EKS op |csi-proxy version |Release notes -|`1.24-2024.12.11` -|`1.24.17` -|`1.7.14` -|`1.1.3` -| - -|`1.24-2024.11.12` -|`1.24.17` -|`1.7.14` -|`1.1.3` -| - -|`1.24-2024.10.08` -|`1.24.17` -|`1.7.14` -|`1.1.3` -| - -|`1.24-2024.09.10` -|`1.24.17` -|`1.7.14` -|`1.1.3` -| - -|`1.24-2024.08.13` -|`1.24.17` -|`1.7.14` -|`1.1.3` -| - -|`1.24-2024.07.10` -|`1.24.17` -|`1.7.11` -|`1.1.2` -|Includes patches for `CVE-2024-5321`. - -|`1.24-2024.06.17` -|`1.24.17` -|`1.7.11` -|`1.1.2` -|Upgraded `containerd` to `1.7.11`. - -|`1.24-2024.05.14` -|`1.24.17` -|`1.6.28` -|`1.1.2` -|Upgraded `containerd` to `1.6.28`. - -|`1.24-2024.04.09` -|`1.24.17` -|`1.6.25` -|`1.1.2` -|Upgraded `containerd` to `1.6.25`. Rebuilt CNI and `csi-proxy` using `golang 1.22.1`. - -|`1.24-2024.03.12` -|`1.24.17` -|`1.6.18` -|`1.1.2` -| - -|`1.24-2024.02.13` -|`1.24.17` -|`1.6.18` -|`1.1.2` -| - -|`1.24-2024.01.09` -|`1.24.17` -|`1.6.18` -|`1.1.2` -| - -|`1.24-2023.12.12` -|`1.24.17` -|`1.6.18` -|`1.1.2` -| - -|`1.24-2023.11.14` -|`1.24.17` -|`1.6.18` -|`1.1.2` -|Includes patches for `CVE-2023-5528`. - -|`1.24-2023.10.19` -|`1.24.17` -|`1.6.18` -|`1.1.2` -|Upgraded `containerd` to `1.6.18`. Upgraded `kubelet` to `1.24.17`. Added new <> (`SERVICE_IPV4_CIDR` and `EXCLUDED_SNAT_CIDRS`). - -|`1.24-2023.09.12` -|`1.24.16` -|`1.6.6` -|`1.1.2` -|Upgraded the Amazon VPC CNI plugin to use the [.noloc]`Kubernetes` connector binary, which gets the [.noloc]`Pod` IP address from the [.noloc]`Kubernetes` API server. Merged https://github.com/aws/amazon-vpc-cni-plugins/pull/100[pull request #100]. - -|`1.24-2023.08.17` -|`1.24.16` -|`1.6.6` -|`1.1.2` -|Includes patches for `CVE-2023-3676`, `CVE-2023-3893`, and `CVE-2023-3955`. - -|`1.24-2023.08.08` -|`1.24.13` -|`1.6.6` -|`1.1.1` -| - -|`1.24-2023.07.11` -|`1.24.13` -|`1.6.6` -|`1.1.1` -| - -|`1.24-2023.06.20` -|`1.24.13` -|`1.6.6` -|`1.1.1` -|Resolved issue that was causing the DNS suffix search list to be incorrectly populated. - -|`1.24-2023.06.14` -|`1.24.13` -|`1.6.6` -|`1.1.1` -|Upgraded [.noloc]`Kubernetes` to `1.24.13`. Added support for host port mapping in CNI. Merged https://github.com/aws/amazon-vpc-cni-plugins/pull/93[pull request #93]. - -|`1.24-2023.05.09` -|`1.24.7` -|`1.6.6` -|`1.1.1` -|Fixed a bug causing network connectivity https://github.com/aws/containers-roadmap/issues/1126[issue #1126] on pods after node restart. Introduced a new <> (`ExcludedSnatCIDRs`). - -|`1.24-2023.04.11` -|`1.24.7` -|`1.6.6` -|`1.1.1` -|Added recovery mechanism for `kubelet` and `kube-proxy` on service crash. - -|`1.24-2023.03.27` -|`1.24.7` -|`1.6.6` -|`1.1.1` -|Installed a link:containers/domainless-windows-authentication-for-amazon-eks-windows-pods[domainless gMSA plugin,type="blog"] to facilitate [.noloc]`gMSA` authentication for [.noloc]`Windows` containers on Amazon EKS. - -|`1.24-2023.03.20` -|`1.24.7` -|`1.6.6` -|`1.1.1` -|[.noloc]`Kubernetes` version downgraded to `1.24.7` because `1.24.10` has a reported issue in `kube-proxy`. - -|`1.24-2023.02.14` -|`1.24.10` -|`1.6.6` -|`1.1.1` -| - -|`1.24-2023.01.23` -|`1.24.7` -|`1.6.6` -|`1.1.1` +|`1.32-2025.10.18` +|`1.32.9` +|`1.7.28` +|`1.2.1` | -|`1.24-2023.01.11` -|`1.24.7` -|`1.6.6` -|`1.1.1` -| +|`1.32-2025.09.13` +|`1.32.8` +|`1.7.28` +|`1.2.1` +|Changed GMSA plugin logs to Windows Events -|`1.24-2022.12.14` -|`1.24.7` -|`1.6.6` -|`1.1.1` +|`1.32-2025.08.18` +|`1.32.7` +|`1.7.27` +|`1.2.1` | -|`1.24-2022.10.11` -|`1.24.7` -|`1.6.6` -|`1.1.1` +|`1.32-2025.07.16` +|`1.32.5` +|`1.7.27` +|`1.2.1` | -|=== -==== -[[eks-ami-versions-windows-2019-core,eks-ami-versions-windows-2019-core.title]] -== Amazon EKS optimized [.noloc]`Windows` Server 2019 Core AMI - -The following tables list the current and previous versions of the Amazon EKS optimized [.noloc]`Windows` Server 2019 Core AMI. - -==== -[role="tablist"] -*[.noloc]`Kubernetes` version [.noloc]`1.31`*:: -+ -[cols="1,1,1,1,1", options="header"] -|=== -|AMI version -|kubelet version -|containerd version -|csi-proxy version -|Release notes +|`1.32-2025.06.13` +|`1.32.5` +|`1.7.27` +|`1.2.1` +|Upgraded `containrd` to `1.7.27` -|`1.31-2024.12.13` -|`1.31.3` +|`1.32-2025.05.17` +|`1.32.5` |`1.7.20` -|`1.1.3` +|`1.2.1` | -|`1.31-2024.11.12` -|`1.31.1` +|`1.32-2025.04.14` +|`1.32.1` |`1.7.20` |`1.1.3` | -|`1.31-2024.10.08` -|`1.31.1` +|`1.32-2025.03.14` +|`1.32.1` |`1.7.20` |`1.1.3` | -|`1.31-2024.10.01` -|`1.31.1` +|`1.32-2025.02.18` +|`1.32.1` |`1.7.20` |`1.1.3` | -|`1.31-2024.09.10` -|`1.31.0` +|`1.32-2025.01.01` +|`1.32.0` |`1.7.20` |`1.1.3` | + |=== -*[.noloc]`Kubernetes` version [.noloc]`1.30`*:: +*Kubernetes version 1.31*:: + -[cols="1,1,1,1,1", options="header"] +[%header,cols="5"] |=== |AMI version |kubelet version @@ -2069,749 +924,608 @@ The following tables list the current and previous versions of the Amazon EKS op |csi-proxy version |Release notes -|`1.30-2024.12.11` -|`1.30.7` -|`1.7.14` -|`1.1.3` -| - -|`1.30-2024.11.12` -|`1.30.4` -|`1.7.14` -|`1.1.3` +|`1.31-2025.10.18` +|`1.31.13` +|`1.7.28` +|`1.2.1` | -|`1.30-2024.10.08` -|`1.30.4` -|`1.7.14` -|`1.1.3` -| +|`1.31-2025.09.13` +|`1.31.12` +|`1.7.28` +|`1.2.1` +|Changed GMSA plugin logs to Windows Events -|`1.30-2024.09.10` -|`1.30.2` -|`1.7.14` -|`1.1.3` +|`1.31-2025.08.18` +|`1.31.11` +|`1.7.27` +|`1.2.1` | -|`1.30-2024.08.13` -|`1.30.2` -|`1.7.14` -|`1.1.3` +|`1.31-2025.07.16` +|`1.31.9` +|`1.7.27` +|`1.2.1` | -|`1.30-2024.07.10` -|`1.30.2` -|`1.7.14` -|`1.1.2` -|Includes patches for `CVE-2024-5321`. - -|`1.30-2024.06.17` -|`1.30.0` -|`1.7.14` -|`1.1.2` -|Upgraded `containerd` to `1.7.14`. +|`1.31-2025.06.13` +|`1.31.9` +|`1.7.27` +|`1.2.1` +|Upgraded `containerd` to `1.7.27`. -|`1.30-2024.05.15` -|`1.30.0` -|`1.6.28` -|`1.1.2` +|`1.31-2025.05.17` +|`1.31.9` +|`1.7.20` +|`1.2.1` | -|=== - - -*[.noloc]`Kubernetes` version [.noloc]`1.29`*:: -+ -[cols="1,1,1,1,1", options="header"] -|=== -|AMI version -|kubelet version -|containerd version -|csi-proxy version -|Release notes -|`1.29-2024.12.11` -|`1.29.10` -|`1.7.14` +|`1.31-2025.04.14` +|`1.31.5` +|`1.7.20` |`1.1.3` | -|`1.29-2024.11.12` -|`1.29.8` -|`1.7.14` +|`1.31-2025.03.14` +|`1.31.5` +|`1.7.20` |`1.1.3` | -|`1.29-2024.10.08` -|`1.29.8` -|`1.7.14` +|`1.31-2025.02.15` +|`1.31.5` +|`1.7.20` |`1.1.3` | -|`1.29-2024.09.10` -|`1.29.6` -|`1.7.14` +|`1.31-2025.01.15` +|`1.31.4` +|`1.7.20` |`1.1.3` | -|`1.29-2024.08.13` -|`1.29.6` -|`1.7.14` +|`1.31-2025.01.01` +|`1.31.4` +|`1.7.20` |`1.1.3` -| - -|`1.29-2024.07.10` -|`1.29.6` -|`1.7.11` -|`1.1.2` -|Includes patches for `CVE-2024-5321`. - -|`1.29-2024.06.17` -|`1.29.3` -|`1.7.11` -|`1.1.2` -| - -|`1.29-2024.05.15` -|`1.29.3` -|`1.7.11` -|`1.1.2` -|Upgraded `containerd` to `1.7.11`. Upgraded `kubelet` to `1.29.3`. - -|`1.29-2024.04.09` -|`1.29.0` -|`1.6.28` -|`1.1.2` -|Upgraded `containerd` to `1.6.28`. Rebuilt CNI and `csi-proxy` using `golang 1.22.1`. - -|`1.29-2024.03.13` -|`1.29.0` -|`1.6.25` -|`1.1.2` -| - -|`1.29-2024.02.13` -|`1.29.0` -|`1.6.25` -|`1.1.2` -| - -|`1.29-2024.02.06` -|`1.29.0` -|`1.6.25` -|`1.1.2` -|Fixed a bug where the pause image was incorrectly deleted by `kubelet` garbage collection process. - -|`1.29-2024.01.09` -|`1.29.0` -|`1.6.18` -|`1.1.2` -| -|=== - - -*[.noloc]`Kubernetes` version [.noloc]`1.28`*:: -+ -[cols="1,1,1,1,1", options="header"] -|=== -|AMI version -|kubelet version -|containerd version -|csi-proxy version -|Release notes +|Includes patches for `CVE-2024-9042`. -|`1.28-2024.12.11` -|`1.28.15` -|`1.7.14` +|`1.31-2024.12.13` +|`1.31.3` +|`1.7.20` |`1.1.3` | -|`1.28-2024.11.12` -|`1.28.13` -|`1.7.14` +|`1.31-2024.11.12` +|`1.31.1` +|`1.7.20` |`1.1.3` | -|`1.28-2024.10.08` -|`1.28.13` -|`1.7.14` +|`1.31-2024.10.08` +|`1.31.1` +|`1.7.20` |`1.1.3` | -|`1.28-2024.09.10` -|`1.28.11` -|`1.7.14` +|`1.31-2024.10.01` +|`1.31.1` +|`1.7.20` |`1.1.3` | -|`1.28-2024.08.13` -|`1.28.11` -|`1.7.14` +|`1.31-2024.09.10` +|`1.31.0` +|`1.7.20` |`1.1.3` | -|`1.28-2024.07.10` -|`1.28.11` -|`1.7.11` -|`1.1.2` -|Includes patches for `CVE-2024-5321`. - -|`1.28-2024.06.17` -|`1.28.8` -|`1.7.11` -|`1.1.2` -|Upgraded `containerd` to `1.7.11`. +|=== -|`1.28-2024.05.14` -|`1.28.8` -|`1.6.28` -|`1.1.2` -|Upgraded `containerd` to `1.6.28`. Upgraded `kubelet` to `1.28.8`. -|`1.28-2024.04.09` -|`1.28.5` -|`1.6.25` -|`1.1.2` -|Upgraded `containerd` to `1.6.25`. Rebuilt CNI and `csi-proxy` using `golang 1.22.1`. +*Kubernetes version 1.30*:: ++ +[%header,cols="5"] +|=== +|AMI version +|kubelet version +|containerd version +|csi-proxy version +|Release notes -|`1.28-2024.03.13` -|`1.28.5` -|`1.6.18` -|`1.1.2` +|`1.30-2025.10.18` +|`1.30.14` +|`1.7.28` +|`1.2.1` | -|`1.28-2024.02.13` -|`1.28.5` -|`1.6.18` -|`1.1.2` -| +|`1.30-2025.09.13` +|`1.30.14` +|`1.7.28` +|`1.2.1` +|Changed GMSA plugin logs to Windows Events -|`1.28-2024.01.09` -|`1.28.5` -|`1.6.18` -|`1.1.2` +|`1.30-2025.08.18` +|`1.30.14` +|`1.7.27` +|`1.2.1` | -|`1.28-2023.12.12` -|`1.28.3` -|`1.6.18` -|`1.1.2` +|`1.30-2025.07.16` +|`1.30.13` +|`1.7.27` +|`1.2.1` | -|`1.28-2023.11.14` -|`1.28.3` -|`1.6.18` -|`1.1.2` -|Includes patches for `CVE-2023-5528`. +|`1.30-2025.06.13` +|`1.30.13` +|`1.7.27` +|`1.2.1` +|Upgraded `containerd` to `1.7.27`. -|`1.28-2023.10.19` -|`1.28.2` -|`1.6.18` -|`1.1.2` -|Upgraded `containerd` to `1.6.18`. Added new <> (`SERVICE_IPV4_CIDR` and `EXCLUDED_SNAT_CIDRS`). +|`1.30-2025.05.17` +|`1.30.13` +|`1.7.20` +|`1.2.1` +| -|`1.28-2023-09.27` -|`1.28.2` -|`1.6.6` -|`1.1.2` -|Fixed a https://github.com/advisories/GHSA-6xv5-86q9-7xr8[security advisory] in `kubelet`. +|`1.30-2025.04.14` +|`1.30.9` +|`1.7.20` +|`1.1.3` +| -|`1.28-2023.09.12` -|`1.28.1` -|`1.6.6` -|`1.1.2` +|`1.30-2025.03.14` +|`1.30.9` +|`1.7.20` +|`1.1.3` +|Upgraded `contianerd` to `1.7.20`. + +|`1.30-2025.02.15` +|`1.30.9` +|`1.7.14` +|`1.1.3` | -|=== +|`1.30-2025.01.15` +|`1.30.8` +|`1.7.14` +|`1.1.3` +| -*[.noloc]`Kubernetes` version [.noloc]`1.27`*:: -+ -[cols="1,1,1,1,1", options="header"] -|=== -|AMI version -|kubelet version -|containerd version -|csi-proxy version -|Release notes +|`1.30-2025.01.01` +|`1.30.8` +|`1.7.14` +|`1.1.3` +|Includes patches for `CVE-2024-9042`. -|`1.27-2024.12.11` -|`1.27.16` +|`1.30-2024.12.11` +|`1.30.7` |`1.7.14` |`1.1.3` | -|`1.27-2024.11.12` -|`1.27.16` +|`1.30-2024.11.12` +|`1.30.4` |`1.7.14` |`1.1.3` | -|`1.27-2024.10.08` -|`1.27.16` +|`1.30-2024.10.08` +|`1.30.4` |`1.7.14` |`1.1.3` | -|`1.27-2024.09.10` -|`1.27.15` +|`1.30-2024.09.10` +|`1.30.2` |`1.7.14` |`1.1.3` | -|`1.27-2024.08.13` -|`1.27.15` +|`1.30-2024.08.13` +|`1.30.2` |`1.7.14` |`1.1.3` | -|`1.27-2024.07.10` -|`1.27.15` -|`1.7.11` +|`1.30-2024.07.10` +|`1.30.2` +|`1.7.14` |`1.1.2` |Includes patches for `CVE-2024-5321`. -|`1.27-2024.06.17` -|`1.27.12` -|`1.7.11` +|`1.30-2024.06.17` +|`1.30.0` +|`1.7.14` |`1.1.2` -|Upgraded `containerd` to `1.7.11`. +|Upgraded `containerd` to `1.7.14`. -|`1.27-2024.05.14` -|`1.27.12` +|`1.30-2024.05.15` +|`1.30.0` |`1.6.28` |`1.1.2` -|Upgraded `containerd` to `1.6.28`. Upgraded `kubelet` to `1.27.12`. - -|`1.27-2024.04.09` -|`1.27.9` -|`1.6.25` -|`1.1.2` -|Upgraded `containerd` to `1.6.25`. Rebuilt CNI and `csi-proxy` using `golang 1.22.1`. - -|`1.27-2024.03.13` -|`1.27.9` -|`1.6.18` -|`1.1.2` | -|`1.27-2024.02.13` -|`1.27.9` -|`1.6.18` -|`1.1.2` -| +|=== -|`1.27-2024.01.09` -|`1.27.9` -|`1.6.18` -|`1.1.2` -| -|`1.27-2023.12.12` -|`1.27.7` -|`1.6.18` -|`1.1.2` -| +*Kubernetes version 1.29*:: ++ +[%header,cols="5"] +|=== +|AMI version +|kubelet version +|containerd version +|csi-proxy version +|Release notes -|`1.27-2023.11.14` -|`1.27.7` -|`1.6.18` -|`1.1.2` -|Includes patches for `CVE-2023-5528`. +|`1.29-2025.10.18` +|`1.29.15` +|`1.7.28` +|`1.2.1` +| -|`1.27-2023.10.19` -|`1.27.6` -|`1.6.18` -|`1.1.2` -|Upgraded `containerd` to `1.6.18`. Added new <> (`SERVICE_IPV4_CIDR` and `EXCLUDED_SNAT_CIDRS`). +|`1.29-2025.09.13` +|`1.29.15` +|`1.7.28` +|`1.2.1` +|Changed GMSA plugin logs to Windows Events -|`1.27-2023-09.27` -|`1.27.6` -|`1.6.6` -|`1.1.2` -|Fixed a https://github.com/advisories/GHSA-6xv5-86q9-7xr8[security advisory] in `kubelet`. +|`1.29-2025.08.18` +|`1.29.15` +|`1.7.27` +|`1.2.1` +| -|`1.27-2023.09.12` -|`1.27.4` -|`1.6.6` -|`1.1.2` -|Upgraded the Amazon VPC CNI plugin to use the [.noloc]`Kubernetes` connector binary, which gets the [.noloc]`Pod` IP address from the [.noloc]`Kubernetes` API server. Merged https://github.com/aws/amazon-vpc-cni-plugins/pull/100[pull request #100]. +|`1.29-2025.07.16` +|`1.29.15` +|`1.7.27` +|`1.2.1` +| -|`1.27-2023.08.17` -|`1.27.4` -|`1.6.6` -|`1.1.2` -|Includes patches for `CVE-2023-3676`, `CVE-2023-3893`, and `CVE-2023-3955`. +|`1.29-2025.06.13` +|`1.29.15` +|`1.7.27` +|`1.2.1` +|Upgraded `containerd` to `1.7.27`. -|`1.27-2023.08.08` -|`1.27.3` -|`1.6.6` -|`1.1.1` +|`1.29-2025.05.17` +|`1.29.15` +|`1.7.20` +|`1.2.1` | -|`1.27-2023.07.11` -|`1.27.3` -|`1.6.6` -|`1.1.1` +|`1.29-2025.04.14` +|`1.29.13` +|`1.7.20` +|`1.1.3` | -|`1.27-2023.06.20` -|`1.27.1` -|`1.6.6` -|`1.1.1` -|Resolved issue that was causing the DNS suffix search list to be incorrectly populated. - -|`1.27-2023.06.14` -|`1.27.1` -|`1.6.6` -|`1.1.1` -|Added support for host port mapping in CNI. Merged https://github.com/aws/amazon-vpc-cni-plugins/pull/93[pull request #93]. - -|`1.27-2023.06.06` -|`1.27.1` -|`1.6.6` -|`1.1.1` -|Fixed `containers-roadmap` https://github.com/aws/containers-roadmap/issues/2042[issue #2042], which caused nodes to fail pulling private Amazon ECR images. +|`1.29-2025.03.14` +|`1.29.13` +|`1.7.20` +|`1.1.3` +|Upgraded `containerd` to `1.7.20`. -|`11.27-2023.05.18` -|`1.27.1` -|`1.6.6` -|`1.1.1` +|`1.29-2025.02.15` +|`1.29.13` +|`1.7.14` +|`1.1.3` | -|=== +|`1.29-2025.01.15` +|`1.29.12` +|`1.7.14` +|`1.1.3` +| -*[.noloc]`Kubernetes` version [.noloc]`1.26`*:: -+ -[cols="1,1,1,1,1", options="header"] -|=== -|AMI version -|kubelet version -|containerd version -|csi-proxy version -|Release notes +|`1.29-2025.01.01` +|`1.29.12` +|`1.7.14` +|`1.1.3` +|Includes patches for `CVE-2024-9042`. -|`1.26-2024.12.11` -|`1.26.15` +|`1.29-2024.12.11` +|`1.29.10` |`1.7.14` |`1.1.3` | -|`1.26-2024.11.12` -|`1.26.15` +|`1.29-2024.11.12` +|`1.29.8` |`1.7.14` |`1.1.3` | -|`1.26-2024.10.09` -|`1.26.15` +|`1.29-2024.10.08` +|`1.29.8` |`1.7.14` |`1.1.3` | -|`1.26-2024.09.10` -|`1.26.15` +|`1.29-2024.09.10` +|`1.29.6` |`1.7.14` |`1.1.3` | -|`1.26-2024.08.13` -|`1.26.15` +|`1.29-2024.08.13` +|`1.29.6` |`1.7.14` |`1.1.3` | -|`1.26-2024.07.10` -|`1.26.15` +|`1.29-2024.07.10` +|`1.29.6` |`1.7.11` |`1.1.2` |Includes patches for `CVE-2024-5321`. -|`1.26-2024.06.17` -|`1.26.15` +|`1.29-2024.06.17` +|`1.29.3` |`1.7.11` |`1.1.2` -|Upgraded `containerd` to `1.7.11`. +| -|`1.26-2024.05.14` -|`1.26.15` -|`1.6.28` +|`1.29-2024.05.15` +|`1.29.3` +|`1.7.11` |`1.1.2` -|Upgraded `containerd` to `1.6.28`. Upgraded `kubelet` to `1.26.15`. +|Upgraded `containerd` to `1.7.11`. Upgraded `kubelet` to `1.29.3`. -|`1.26-2024.04.09` -|`1.26.12` -|`1.6.25` +|`1.29-2024.04.09` +|`1.29.0` +|`1.6.28` |`1.1.2` -|Upgraded `containerd` to `1.6.25`. Rebuilt CNI and `csi-proxy` using `golang 1.22.1`. +|Upgraded `containerd` to `1.6.28`. Rebuilt CNI and `csi-proxy` using `golang 1.22.1`. -|`1.26-2024.03.13` -|`1.26.12` -|`1.6.18` +|`1.29-2024.03.12` +|`1.29.0` +|`1.6.25` |`1.1.2` | -|`1.26-2024.02.13` -|`1.26.12` -|`1.6.18` +|`1.29-2024.02.13` +|`1.29.0` +|`1.6.25` |`1.1.2` | -|`1.26-2024.01.09` -|`1.26.12` -|`1.6.18` +|`1.29-2024.02.06` +|`1.29.0` +|`1.6.25` |`1.1.2` -| +|Fixed a bug where the pause image was incorrectly deleted by `kubelet` garbage collection process. -|`1.26-2023.12.12` -|`1.26.10` +|`1.29-2024.01.09` +|`1.29.0` |`1.6.18` |`1.1.2` | -|`1.26-2023.11.14` -|`1.26.10` -|`1.6.18` -|`1.1.2` -|Includes patches for `CVE-2023-5528`. - -|`1.26-2023.10.19` -|`1.26.9` -|`1.6.18` -|`1.1.2` -|Upgraded `containerd` to `1.6.18`. Upgraded `kubelet` to `1.26.9`. Added new <> (`SERVICE_IPV4_CIDR` and `EXCLUDED_SNAT_CIDRS`). - -|`1.26-2023.09.12` -|`1.26.7` -|`1.6.6` -|`1.1.2` -|Upgraded the Amazon VPC CNI plugin to use the [.noloc]`Kubernetes` connector binary, which gets the [.noloc]`Pod` IP address from the [.noloc]`Kubernetes` API server. Merged https://github.com/aws/amazon-vpc-cni-plugins/pull/100[pull request #100]. +|=== -|`1.26-2023.08.17` -|`1.26.7` -|`1.6.6` -|`1.1.2` -|Includes patches for `CVE-2023-3676`, `CVE-2023-3893`, and `CVE-2023-3955`. -|`1.26-2023.08.08` -|`1.26.6` -|`1.6.6` -|`1.1.1` +*Kubernetes version 1.28*:: ++ +[%header,cols="5"] +|=== +|AMI version +|kubelet version +|containerd version +|csi-proxy version +|Release notes + +|`1.28-2025.10.18` +|`1.28.15` +|`1.7.28` +|`1.2.1` | -|`1.26-2023.07.11` -|`1.26.6` -|`1.6.6` -|`1.1.1` +|`1.28-2025.09.13` +|`1.28.15` +|`1.7.28` +|`1.2.1` +|Changed GMSA plugin logs to Windows Events + +|`1.28-2025.08.18` +|`1.28.15` +|`1.7.27` +|`1.2.1` | -|`1.26-2023.06.20` -|`1.26.4` -|`1.6.6` -|`1.1.1` -|Resolved issue that was causing the DNS suffix search list to be incorrectly populated. +|`1.28-2025.07.16` +|`1.28.15` +|`1.7.27` +|`1.2.1` +| -|`1.26-2023.06.14` -|`1.26.4` -|`1.6.6` -|`1.1.1` -|Upgraded [.noloc]`Kubernetes` to `1.26.4`. Added support for host port mapping in CNI. Merged https://github.com/aws/amazon-vpc-cni-plugins/pull/93[pull request #93]. +|`1.28-2025.06.13` +|`1.28.15` +|`1.7.27` +|`1.2.1` +|Upgraded `containerd` to `1.7.27`. -|`1.26-2023.05.09` -|`1.26.2` -|`1.6.6` -|`1.1.1` -|Fixed a bug causing network connectivity https://github.com/aws/containers-roadmap/issues/1126[issue #1126] on pods after node restart. Introduced a new <> (`ExcludedSnatCIDRs`). +|`1.28-2025.05.17` +|`1.28.15` +|`1.7.20` +|`1.2.1` +| -|`1.26-2023.04.26` -|`1.26.2` -|`1.6.6` -|`1.1.1` +|`1.28-2025.04.14` +|`1.28.15` +|`1.7.20` +|`1.1.3` | -|`1.26-2023.04.11` -|`1.26.2` -|`1.6.6` -|`1.1.1` -|Added recovery mechanism for `kubelet` and `kube-proxy` on service crash. +|`1.28-2025.03.14` +|`1.28.15` +|`1.7.20` +|`1.1.3` +|Upgraded `containerd` to `1.7.20`. -|`1.26-2023.03.24` -|`1.26.2` -|`1.6.6` -|`1.1.1` +|`1.28-2025.02.15` +|`1.28.15` +|`1.7.14` +|`1.1.3` | -|=== +|`1.28-2025.01.15` +|`1.28.15` +|`1.7.14` +|`1.1.3` +| -*[.noloc]`Kubernetes` version [.noloc]`1.25`*:: -+ -[cols="1,1,1,1,1", options="header"] -|=== -|AMI version -|kubelet version -|containerd version -|csi-proxy version -|Release notes +|`1.28-2025.01.01` +|`1.28.15` +|`1.7.14` +|`1.1.3` +|Includes patches for `CVE-2024-9042`. -|`1.25-2024.12.13` -|`1.25.16` +|`1.28-2024.12.11` +|`1.28.15` |`1.7.14` |`1.1.3` | -|`1.25-2024.11.12` -|`1.25.16` +|`1.28-2024.11.12` +|`1.28.13` |`1.7.14` |`1.1.3` | -|`1.25-2024.10.08` -|`1.25.16` +|`1.28-2024.10.08` +|`1.28.13` |`1.7.14` |`1.1.3` | -|`1.25-2024.09.10` -|`1.25.16` +|`1.28-2024.09.10` +|`1.28.11` |`1.7.14` |`1.1.3` | -|`1.25-2024.08.13` -|`1.25.16` +|`1.28-2024.08.13` +|`1.28.11` |`1.7.14` |`1.1.3` | -|`1.25-2024.07.10` -|`1.25.16` +|`1.28-2024.07.10` +|`1.28.11` |`1.7.11` |`1.1.2` |Includes patches for `CVE-2024-5321`. -|`1.25-2024.06.17` -|`1.25.16` +|`1.28-2024.06.17` +|`1.28.8` |`1.7.11` |`1.1.2` |Upgraded `containerd` to `1.7.11`. -|`1.25-2024.05.14` -|`1.25.16` +|`1.28-2024.05.14` +|`1.28.8` |`1.6.28` |`1.1.2` -|Upgraded `containerd` to `1.6.28`. +|Upgraded `containerd` to `1.6.28`. Upgraded `kubelet` to `1.28.8`. -|`1.25-2024.04.09` -|`1.25.16` +|`1.28-2024.04.09` +|`1.28.5` |`1.6.25` |`1.1.2` |Upgraded `containerd` to `1.6.25`. Rebuilt CNI and `csi-proxy` using `golang 1.22.1`. -|`1.25-2024.03.13` -|`1.25.16` +|`1.28-2024.03.12` +|`1.28.5` |`1.6.18` |`1.1.2` | -|`1.25-2024.02.13` -|`1.25.16` +|`1.28-2024.02.13` +|`1.28.5` |`1.6.18` |`1.1.2` | -|`1.25-2024.01.09` -|`1.25.16` +|`1.28-2024.01.09` +|`1.28.5` |`1.6.18` |`1.1.2` | -|`1.25-2023.12.12` -|`1.25.15` +|`1.28-2023.12.12` +|`1.28.3` |`1.6.18` |`1.1.2` | -|`1.25-2023.11.14` -|`1.25.15` +|`1.28-2023.11.14` +|`1.28.3` |`1.6.18` |`1.1.2` |Includes patches for `CVE-2023-5528`. -|`1.25-2023.10.19` -|`1.25.14` +|`1.28-2023.10.19` +|`1.28.2` |`1.6.18` |`1.1.2` -|Upgraded `containerd` to `1.6.18`. Upgraded `kubelet` to `1.25.14`. Added new <> (`SERVICE_IPV4_CIDR` and `EXCLUDED_SNAT_CIDRS`). +|Upgraded `containerd` to `1.6.18`. Added new <> (`SERVICE_IPV4_CIDR` and `EXCLUDED_SNAT_CIDRS`). -|`1.25-2023.09.12` -|`1.25.12` +|`1.28-2023-09.27` +|`1.28.2` |`1.6.6` |`1.1.2` -|Upgraded the Amazon VPC CNI plugin to use the [.noloc]`Kubernetes` connector binary, which gets the [.noloc]`Pod` IP address from the [.noloc]`Kubernetes` API server. Merged https://github.com/aws/amazon-vpc-cni-plugins/pull/100[pull request #100]. +|Fixed a https://github.com/advisories/GHSA-6xv5-86q9-7xr8[security advisory] in `kubelet`. -|`1.25-2023.08.17` -|`1.25.12` +|`1.28-2023.09.12` +|`1.28.1` |`1.6.6` |`1.1.2` -|Includes patches for `CVE-2023-3676`, `CVE-2023-3893`, and `CVE-2023-3955`. - -|`1.25-2023.08.08` -|`1.25.9` -|`1.6.6` -|`1.1.1` | -|`1.25-2023.07.11` -|`1.25.9` -|`1.6.6` -|`1.1.1` -| +|=== -|`1.25-2023.06.20` -|`1.25.9` -|`1.6.6` -|`1.1.1` -|Resolved issue that was causing the DNS suffix search list to be incorrectly populated. -|`1.25-2023.06.14` -|`1.25.9` -|`1.6.6` -|`1.1.1` -|Upgraded [.noloc]`Kubernetes` to `1.25.9`. Added support for host port mapping in CNI. Merged https://github.com/aws/amazon-vpc-cni-plugins/pull/93[pull request #93]. -|`1.25-2023.05.09` -|`1.25.7` -|`1.6.6` -|`1.1.1` -|Fixed a bug causing network connectivity https://github.com/aws/containers-roadmap/issues/1126[issue #1126] on pods after node restart. Introduced a new <> (`ExcludedSnatCIDRs`). +==== -|`1.25-2023.04.11` -|`1.25.7` -|`1.6.6` -|`1.1.1` -|Added recovery mechanism for `kubelet` and `kube-proxy` on service crash. +[#eks-ami-versions-windows-2019-core] +== Amazon EKS optimized Windows Server 2019 Core AMI -|`1.25-2023.03.27` -|`1.25.6` -|`1.6.6` -|`1.1.1` -|Installed a link:containers/domainless-windows-authentication-for-amazon-eks-windows-pods[domainless gMSA plugin,type="blog"] to facilitate [.noloc]`gMSA` authentication for [.noloc]`Windows` containers on Amazon EKS. +The following tables list the current and previous versions of the Amazon EKS optimized Windows Server 2019 Core AMI. -|`1.25-2023.03.20` -|`1.25.6` -|`1.6.6` -|`1.1.1` +==== +[role="tablist"] +*Kubernetes version 1.34*:: ++ +[%header,cols="5"] +|=== +|AMI version +|kubelet version +|containerd version +|csi-proxy version +|Release notes + + +|`1.34-2025.10.18` +|`1.34.1` +|`2.1.4` +|`1.2.1` | -|`1.25-2023.02.14` -|`1.25.6` -|`1.6.6` -|`1.1.1` +|`1.34-2025.09.13` +|`1.34.0` +|`2.1.4` +|`1.2.1` | -|=== +|=== -*[.noloc]`Kubernetes` version [.noloc]`1.24`*:: +*Kubernetes version 1.33*:: + -[cols="1,1,1,1,1", options="header"] +[%header,cols="5"] |=== |AMI version |kubelet version @@ -2819,204 +1533,191 @@ The following tables list the current and previous versions of the Amazon EKS op |csi-proxy version |Release notes -|`1.24-2024.12.11` -|`1.24.17` -|`1.7.14` -|`1.1.3` +|`1.33-2025.10.18` +|`1.33.5` +|`1.7.28` +|`1.2.1` | -|`1.24-2024.11.12` -|`1.24.17` -|`1.7.14` -|`1.1.3` -| +|`1.33-2025.09.13` +|`1.33.4` +|`1.7.28` +|`1.2.1` +|Changed GMSA plugin logs to Windows Events -|`1.24-2024.10.08` -|`1.24.17` -|`1.7.14` -|`1.1.3` +|`1.33-2025.08.18` +|`1.33.3` +|`1.7.27` +|`1.2.1` | -|`1.24-2024.09.10` -|`1.24.17` -|`1.7.14` -|`1.1.3` +|`1.33-2025.07.16` +|`1.33.1` +|`1.7.27` +|`1.2.1` | -|`1.24-2024.08.13` -|`1.24.17` -|`1.7.14` -|`1.1.3` +|`1.33-2025.06.13` +|`1.33.1` +|`1.7.27` +|`1.2.1` | -|`1.24-2024.07.10` -|`1.24.17` -|`1.7.11` -|`1.1.2` -|Includes patches for `CVE-2024-5321`. - -|`1.24-2024.06.17` -|`1.24.17` -|`1.7.11` -|`1.1.2` -|Upgraded `containerd` to `1.7.11`. +|`1.33-2025.05.17` +|`1.33.1` +|`1.7.27` +|`1.2.1` +| -|`1.24-2024.05.14` -|`1.24.17` -|`1.6.28` -|`1.1.2` -|Upgraded `containerd` to `1.6.28`. +|=== -|`1.24-2024.04.09` -|`1.24.17` -|`1.6.25` -|`1.1.2` -|Upgraded `containerd` to `1.6.25`. Rebuilt CNI and `csi-proxy` using `golang 1.22.1`. +*Kubernetes version 1.32*:: ++ +[%header,cols="5"] +|=== +|AMI version +|kubelet version +|containerd version +|csi-proxy version +|Release notes -|`1.24-2024.03.13` -|`1.24.17` -|`1.6.18` -|`1.1.2` +|`1.32-2025.10.18` +|`1.32.9` +|`1.7.28` +|`1.2.1` | -|`1.24-2024.02.13` -|`1.24.17` -|`1.6.18` -|`1.1.2` -| +|`1.32-2025.09.13` +|`1.32.8` +|`1.7.28` +|`1.2.1` +|Changed GMSA plugin logs to Windows Events -|`1.24-2024.01.09` -|`1.24.17` -|`1.6.18` -|`1.1.2` +|`1.32-2025.08.18` +|`1.32.7` +|`1.7.27` +|`1.2.1` | -|`1.24-2023.12.12` -|`1.24.17` -|`1.6.18` -|`1.1.2` +|`1.32-2025.07.16` +|`1.32.5` +|`1.7.27` +|`1.2.1` | -|`1.24-2023.11.14` -|`1.24.17` -|`1.6.18` -|`1.1.2` -|Includes patches for `CVE-2023-5528`. - -|`1.24-2023.10.19` -|`1.24.17` -|`1.6.18` -|`1.1.2` -|Upgraded `containerd` to `1.6.18`. Upgraded `kubelet` to `1.24.17`. Added new <> (`SERVICE_IPV4_CIDR` and `EXCLUDED_SNAT_CIDRS`). - -|`1.24-2023.09.12` -|`1.24.16` -|`1.6.6` -|`1.1.2` -|Upgraded the Amazon VPC CNI plugin to use the [.noloc]`Kubernetes` connector binary, which gets the [.noloc]`Pod` IP address from the [.noloc]`Kubernetes` API server. Merged https://github.com/aws/amazon-vpc-cni-plugins/pull/100[pull request #100]. +|`1.32-2025.06.13` +|`1.32.5` +|`1.7.27` +|`1.2.1` +|Upgraded `containerd` to `1.7.27`. -|`1.24-2023.08.17` -|`1.24.16` -|`1.6.6` -|`1.1.2` -|Includes patches for `CVE-2023-3676`, `CVE-2023-3893`, and `CVE-2023-3955`. +|`1.32-2025.05.17` +|`1.32.5` +|`1.7.20` +|`1.2.1` +| -|`1.24-2023.08.08` -|`1.24.13` -|`1.6.6` -|`1.1.1` +|`1.32-2025.04.14` +|`1.32.1` +|`1.7.20` +|`1.1.3` | -|`1.24-2023.07.11` -|`1.24.13` -|`1.6.6` -|`1.1.1` +|`1.32-2025.03.14` +|`1.32.1` +|`1.7.20` +|`1.1.3` | -|`1.24-2023.06.20` -|`1.24.13` -|`1.6.6` -|`1.1.1` -|Resolved issue that was causing the DNS suffix search list to be incorrectly populated. +|`1.32-2025.02.18` +|`1.32.1` +|`1.7.20` +|`1.1.3` +| -|`1.24-2023.06.14` -|`1.24.13` -|`1.6.6` -|`1.1.1` -|Upgraded [.noloc]`Kubernetes` to `1.24.13`. Added support for host port mapping in CNI. Merged https://github.com/aws/amazon-vpc-cni-plugins/pull/93[pull request #93]. +|`1.32-2025.01.15` +|`1.32.4` +|`1.7.20` +|`1.1.3` +| -|`1.24-2023.05.09` -|`1.24.7` -|`1.6.6` -|`1.1.1` -|Fixed a bug causing network connectivity https://github.com/aws/containers-roadmap/issues/1126[issue #1126] on pods after node restart. Introduced a new <> (`ExcludedSnatCIDRs`). +|=== -|`1.24-2023.04.11` -|`1.24.7` -|`1.6.6` -|`1.1.1` -|Added recovery mechanism for `kubelet` and `kube-proxy` on service crash. +*Kubernetes version 1.31*:: ++ +[%header,cols="5"] +|=== +|AMI version +|kubelet version +|containerd version +|csi-proxy version +|Release notes -|`1.24-2023.03.27` -|`1.24.7` -|`1.6.6` -|`1.1.1` -|Installed a link:containers/domainless-windows-authentication-for-amazon-eks-windows-pods[domainless gMSA plugin,type="blog"] to facilitate [.noloc]`gMSA` authentication for [.noloc]`Windows` containers on Amazon EKS. +|`1.31-2025.10.18` +|`1.31.13` +|`1.7.28` +|`1.2.1` +| -|`1.24-2023.03.20` -|`1.24.7` -|`1.6.6` -|`1.1.1` -|[.noloc]`Kubernetes` version downgraded to `1.24.7` because `1.24.10` has a reported issue in `kube-proxy`. +|`1.31-2025.09.13` +|`1.31.12` +|`1.7.28` +|`1.2.1` +|Changed GMSA plugin logs to Windows Events -|`1.24-2023.02.14` -|`1.24.10` -|`1.6.6` -|`1.1.1` +|`1.31-2025.08.18` +|`1.31.11` +|`1.7.27` +|`1.2.1` | -|`1.24-2023.01.23` -|`1.24.7` -|`1.6.6` -|`1.1.1` +|`1.31-2025.07.16` +|`1.31.9` +|`1.7.27` +|`1.2.1` | -|`1.24-2023.01.11` -|`1.24.7` -|`1.6.6` -|`1.1.1` +|`1.31-2025.06.13` +|`1.31.9` +|`1.7.27` +|`1.2.1` +|Upgraded `containerd` to `1.7.27`. + +|`1.31-2025.05.17` +|`1.31.9` +|`1.7.20` +|`1.2.1` | -|`1.24-2022.12.13` -|`1.24.7` -|`1.6.6` -|`1.1.1` +|`1.31-2025.04.14` +|`1.31.5` +|`1.7.20` +|`1.1.3` | -|`1.24-2022.11.08` -|`1.24.7` -|`1.6.6` -|`1.1.1` +|`1.31-2025.03.14` +|`1.31.5` +|`1.7.20` +|`1.1.3` | -|=== -==== -[[eks-ami-versions-windows-2019-full,eks-ami-versions-windows-2019-full.title]] -== Amazon EKS optimized [.noloc]`Windows` Server 2019 Full AMI +|`1.31-2025.02.15` +|`1.31.5` +|`1.7.20` +|`1.1.3` +| -The following tables list the current and previous versions of the Amazon EKS optimized [.noloc]`Windows` Server 2019 Full AMI. +|`1.31-2025.01.15` +|`1.31.4` +|`1.7.20` +|`1.1.3` +| -==== -[role="tablist"] -*[.noloc]`Kubernetes` version [.noloc]`1.31`*:: -+ -[cols="1,1,1,1,1", options="header"] -|=== -|AMI version -|kubelet version -|containerd version -|csi-proxy version -|Release notes +|`1.31-2025.01.01` +|`1.31.4` +|`1.7.20` +|`1.1.3` +|Includes patches for `CVE-2024-9042`. |`1.31-2024.12.13` |`1.31.3` @@ -3047,12 +1748,13 @@ The following tables list the current and previous versions of the Amazon EKS op |`1.7.20` |`1.1.3` | + |=== -*[.noloc]`Kubernetes` version [.noloc]`1.30`*:: +*Kubernetes version 1.30*:: + -[cols="1,1,1,1,1", options="header"] +[%header,cols="5"] |=== |AMI version |kubelet version @@ -3060,6 +1762,72 @@ The following tables list the current and previous versions of the Amazon EKS op |csi-proxy version |Release notes +|`1.30-2025.10.18` +|`1.30.14` +|`1.7.28` +|`1.2.1` +| + +|`1.30-2025.09.13` +|`1.30.14` +|`1.7.28` +|`1.2.1` +|Changed GMSA plugin logs to Windows Events + +|`1.30-2025.08.18` +|`1.30.14` +|`1.7.27` +|`1.2.1` +| + +|`1.30-2025.07.16` +|`1.30.13` +|`1.7.27` +|`1.2.1` +| + +|`1.30-2025.06.13` +|`1.30.13` +|`1.7.27` +|`1.2.1` +|Upgraded `containerd` to `1.7.27`. + +|`1.30-2025.05.17` +|`1.30.13` +|`1.7.20` +|`1.2.1` +| + +|`1.30-2025.04.14` +|`1.30.9` +|`1.7.20` +|`1.1.3` +| + +|`1.30-2025.03.14` +|`1.30.9` +|`1.7.20` +|`1.1.3` +|Upgraded `containerd` to `1.7.20`. + +|`1.30-2025-02-15` +|`1.30.9` +|`1.7.14` +|`1.1.3` +| + +|`1.30-2025.01.15` +|`1.30.8` +|`1.7.14` +|`1.1.3` +| + +|`1.30-2025.01.01` +|`1.30.8` +|`1.7.14` +|`1.1.3` +|Includes patches for `CVE-2024-9042`. + |`1.30-2024.12.11` |`1.30.7` |`1.7.14` @@ -3107,12 +1875,13 @@ The following tables list the current and previous versions of the Amazon EKS op |`1.6.28` |`1.1.2` | + |=== -*[.noloc]`Kubernetes` version [.noloc]`1.29`*:: +*Kubernetes version 1.29*:: + -[cols="1,1,1,1,1", options="header"] +[%header,cols="5"] |=== |AMI version |kubelet version @@ -3120,6 +1889,72 @@ The following tables list the current and previous versions of the Amazon EKS op |csi-proxy version |Release notes +|`1.29-2025.10.18` +|`1.29.15` +|`1.7.28` +|`1.2.1` +| + +|`1.29-2025.09.13` +|`1.29.15` +|`1.7.28` +|`1.2.1` +|Changed GMSA plugin logs to Windows Events + +|`1.29-2025.08.18` +|`1.29.15` +|`1.7.27` +|`1.2.1` +| + +|`1.29-2025.07.16` +|`1.29.15` +|`1.7.27` +|`1.2.1` +| + +|`1.29-2025.06.13` +|`1.29.15` +|`1.7.27` +|`1.2.1` +|Upgraded `containerd` to `1.7.27`. + +|`1.29-2025.05.17` +|`1.29.15` +|`1.7.20` +|`1.2.1` +| + +|`1.29-2025.04.14` +|`1.29.13` +|`1.7.20` +|`1.1.3` +| + +|`1.29-2025.03.14` +|`1.29.13` +|`1.7.20` +|`1.1.3` +|Upgraded `containerd` to `1.7.20`. + +|`1.29-2025.02.15` +|`1.29.13` +|`1.7.14` +|`1.1.3` +| + +|`1.29-2025.01.15` +|`1.29.12` +|`1.7.14` +|`1.1.3` +| + +|`1.29-2025.01.01` +|`1.29.12` +|`1.7.14` +|`1.1.3` +|Includes patches for `CVE-2024-9042`. + |`1.29-2024.12.11` |`1.29.10` |`1.7.14` @@ -3197,12 +2032,13 @@ The following tables list the current and previous versions of the Amazon EKS op |`1.6.18` |`1.1.2` | + |=== -*[.noloc]`Kubernetes` version [.noloc]`1.28`*:: +*Kubernetes version 1.28*:: + -[cols="1,1,1,1,1", options="header"] +[%header,cols="5"] |=== |AMI version |kubelet version @@ -3210,6 +2046,72 @@ The following tables list the current and previous versions of the Amazon EKS op |csi-proxy version |Release notes +|`1.28-2025.10.18` +|`1.28.15` +|`1.7.28` +|`1.2.1` +| + +|`1.28-2025.09.13` +|`1.28.15` +|`1.7.28` +|`1.2.1` +|Changed GMSA plugin logs to Windows Events + +|`1.28-2025.08.18` +|`1.28.15` +|`1.7.27` +|`1.2.1` +| + +|`1.28-2025.07.16` +|`1.28.15` +|`1.7.27` +|`1.2.1` +| + +|`1.28-2025.06.13` +|`1.28.15` +|`1.7.20` +|`1.2.1` +|Upgraded `containerd` to `1.7.27`. + +|`1.28-2025.05.17` +|`1.28.15` +|`1.7.20` +|`1.2.1` +| + +|`1.28-2025.04.14` +|`1.28.15` +|`1.7.20` +|`1.1.3` +| + +|`1.28-2025.03.14` +|`1.28.15` +|`1.7.20` +|`1.1.3` +|Upgraded `containerd` to `1.7.20`. + +|`1.28-2025.02.15` +|`1.28.15` +|`1.7.14` +|`1.1.3` +| + +|`1.28-2025-01-15` +|`1.28.15` +|`1.7.14` +|`1.1.3` +| + +|`1.28-2025-01-01` +|`1.28.15` +|`1.7.14` +|`1.1.3` +|Includes patches for `CVE-2024-9042`. + |`1.28-2024.12.11` |`1.28.15` |`1.7.14` @@ -3311,12 +2213,22 @@ The following tables list the current and previous versions of the Amazon EKS op |`1.6.6` |`1.1.2` | + |=== +==== + + +[#eks-ami-versions-windows-2019-full] +== Amazon EKS optimized Windows Server 2019 Full AMI -*[.noloc]`Kubernetes` version [.noloc]`1.27`*:: +The following tables list the current and previous versions of the Amazon EKS optimized Windows Server 2019 Full AMI. + +==== +[role="tablist"] +*Kubernetes version 1.34*:: + -[cols="1,1,1,1,1", options="header"] +[%header,cols="5"] |=== |AMI version |kubelet version @@ -3324,155 +2236,254 @@ The following tables list the current and previous versions of the Amazon EKS op |csi-proxy version |Release notes -|`1.27-2024.12.11` -|`1.27.16` -|`1.7.14` -|`1.1.3` + +|`1.34-2025.10.18` +|`1.34.1` +|`2.1.4` +|`1.2.1` | -|`1.27-2024.11.12` -|`1.27.16` -|`1.7.14` -|`1.1.3` +|`1.34-2025.09.13` +|`1.34.0` +|`2.1.4` +|`1.2.1` | -|`1.27-2024.10.08` -|`1.27.16` -|`1.7.14` +|=== + +*Kubernetes version 1.33*:: ++ +[%header,cols="5"] +|=== +|AMI version +|kubelet version +|containerd version +|csi-proxy version +|Release notes + +|`1.33-2025.10.18` +|`1.33.5` +|`1.7.28` +|`1.2.1` +| + +|`1.33-2025.09.13` +|`1.33.4` +|`1.7.28` +|`1.2.1` +|Changed GMSA plugin logs to Windows Events + +|`1.33-2025.08.18` +|`1.33.3` +|`1.7.27` +|`1.2.1` +| + +|`1.33-2025.07.16` +|`1.33.1` +|`1.7.27` +|`1.2.1` +| + +|`1.33-2025.06.13` +|`1.33.1` +|`1.7.27` +|`1.2.1` +| + +|`1.33-2025.05.17` +|`1.33.1` +|`1.7.27` +|`1.2.1` +| + +|=== + +*Kubernetes version 1.32*:: ++ +[%header,cols="5"] +|=== +|AMI version +|kubelet version +|containerd version +|csi-proxy version +|Release notes + +|`1.32-2025.10.18` +|`1.32.9` +|`1.7.28` +|`1.2.1` +| + +|`1.32-2025.09.13` +|`1.32.8` +|`1.7.28` +|`1.2.1` +|Changed GMSA plugin logs to Windows Events + +|`1.32-2025.08.18` +|`1.32.7` +|`1.7.27` +|`1.2.1` +| + +|`1.32-2025.07.16` +|`1.32.5` +|`1.7.27` +|`1.2.1` +| + +|`1.32-2025.06.13` +|`1.32.5` +|`1.7.27` +|`1.2.1` +|Upgraded `containerd` to `1.7.27`. + +|`1.32-2025.05.17` +|`1.32.5` +|`1.7.20` +|`1.2.1` +| + +|`1.32-2025.04.14` +|`1.32.1` +|`1.7.20` |`1.1.3` | -|`1.27-2024.09.10` -|`1.27.15` -|`1.7.14` +|`1.32-2025.03.14` +|`1.32.1` +|`1.7.20` |`1.1.3` | -|`1.27-2024.08.13` -|`1.27.15` -|`1.7.14` +|`1.32-2025.02.18` +|`1.32.1` +|`1.7.20` |`1.1.3` | -|`1.27-2024.07.10` -|`1.27.15` -|`1.7.11` -|`1.1.2` -|Includes patches for `CVE-2024-5321`. +|`1.32-2025.01.15` +|`1.32.0` +|`1.7.20` +|`1.1.3` +| -|`1.27-2024.06.17` -|`1.27.12` -|`1.7.11` -|`1.1.2` -|Upgraded `containerd` to `1.7.11`. +|=== -|`1.27-2024.05.14` -|`1.27.12` -|`1.6.28` -|`1.1.2` -|Upgraded `containerd` to `1.6.28`. Upgraded `kubelet` to `1.27.12`. -|`1.27-2024.04.09` -|`1.27.9` -|`1.6.25` -|`1.1.2` -|Upgraded `containerd` to `1.6.25`. Rebuilt CNI and `csi-proxy` using `golang 1.22.1`. +*Kubernetes version 1.31*:: ++ +[%header,cols="5"] +|=== +|AMI version +|kubelet version +|containerd version +|csi-proxy version +|Release notes -|`1.27-2024.03.13` -|`1.27.9` -|`1.6.18` -|`1.1.2` +|`1.31-2025.10.18` +|`1.31.13` +|`1.7.28` +|`1.2.1` | -|`1.27-2024.02.13` -|`1.27.9` -|`1.6.18` -|`1.1.2` -| +|`1.31-2025.09.13` +|`1.31.12` +|`1.7.28` +|`1.2.1` +|Changed GMSA plugin logs to Windows Events -|`1.27-2024.01.09` -|`1.27.9` -|`1.6.18` -|`1.1.2` +|`1.31-2025.08.18` +|`1.31.11` +|`1.7.27` +|`1.2.1` | -|`1.27-2023.12.12` -|`1.27.7` -|`1.6.18` -|`1.1.2` +|`1.31-2025.07.16` +|`1.31.9` +|`1.7.27` +|`1.2.1` | -|`1.27-2023.11.14` -|`1.27.7` -|`1.6.18` -|`1.1.2` -|Includes patches for `CVE-2023-5528`. +|`1.31-2025.06.13` +|`1.31.9` +|`1.7.27` +|`1.2.1` +|Upgraded `containerd` to `1.7.27`. -|`1.27-2023.10.19` -|`1.27.6` -|`1.6.18` -|`1.1.2` -|Upgraded `containerd` to `1.6.18`. Added new <> (`SERVICE_IPV4_CIDR` and `EXCLUDED_SNAT_CIDRS`). +|`1.31-2025.05.17` +|`1.31.9` +|`1.7.20` +|`1.2.1` +| -|`1.27-2023-09.27` -|`1.27.6` -|`1.6.6` -|`1.1.2` -|Fixed a https://github.com/advisories/GHSA-6xv5-86q9-7xr8[security advisory] in `kubelet`. +|`1.31-2025.04.14` +|`1.31.5` +|`1.7.20` +|`1.1.3` +| -|`1.27-2023.09.12` -|`1.27.4` -|`1.6.6` -|`1.1.2` -|Upgraded the Amazon VPC CNI plugin to use the [.noloc]`Kubernetes` connector binary, which gets the [.noloc]`Pod` IP address from the [.noloc]`Kubernetes` API server. Merged https://github.com/aws/amazon-vpc-cni-plugins/pull/100[pull request #100]. +|`1.31-2025.03.14` +|`1.31.5` +|`1.7.20` +|`1.1.3` +| -|`1.27-2023.08.17` -|`1.27.4` -|`1.6.6` -|`1.1.2` -|Includes patches for `CVE-2023-3676`, `CVE-2023-3893`, and `CVE-2023-3955`. +|`1.31-2025.02.15` +|`1.31.5` +|`1.7.20` +|`1.1.3` +| -|`1.27-2023.08.08` -|`1.27.3` -|`1.6.6` -|`1.1.1` +|`1.31-2025.01.15` +|`1.31.4` +|`1.7.20` +|`1.1.3` | -|`1.27-2023.07.11` -|`1.27.3` -|`1.6.6` -|`1.1.1` +|`1.31-2025.01.01` +|`1.31.4` +|`1.7.20` +|`1.1.3` +|Includes patches for `CVE-2024-9042`. + +|`1.31-2024.12.13` +|`1.31.3` +|`1.7.20` +|`1.1.3` | -|`1.27-2023.06.20` -|`1.27.1` -|`1.6.6` -|`1.1.1` -|Resolved issue that was causing the DNS suffix search list to be incorrectly populated. +|`1.31-2024.11.12` +|`1.31.1` +|`1.7.20` +|`1.1.3` +| -|`1.27-2023.06.14` -|`1.27.1` -|`1.6.6` -|`1.1.1` -|Added support for host port mapping in CNI. Merged https://github.com/aws/amazon-vpc-cni-plugins/pull/93[pull request #93]. +|`1.31-2024.10.08` +|`1.31.1` +|`1.7.20` +|`1.1.3` +| -|`1.27-2023.06.06` -|`1.27.1` -|`1.6.6` -|`1.1.1` -|Fixed `containers-roadmap` https://github.com/aws/containers-roadmap/issues/2042[issue #2042], which caused nodes to fail pulling private Amazon ECR images. +|`1.31-2024.10.01` +|`1.31.1` +|`1.7.20` +|`1.1.3` +| -|`1.27-2023.05.17` -|`1.27.1` -|`1.6.6` -|`1.1.1` +|`1.31-2024.09.10` +|`1.31.0` +|`1.7.20` +|`1.1.3` | + |=== -*[.noloc]`Kubernetes` version [.noloc]`1.26`*:: +*Kubernetes version 1.30*:: + -[cols="1,1,1,1,1", options="header"] +[%header,cols="5"] |=== |AMI version |kubelet version @@ -3480,514 +2491,460 @@ The following tables list the current and previous versions of the Amazon EKS op |csi-proxy version |Release notes -|`1.26-2024.12.11` -|`1.26.15` +|`1.30-2025.10.18` +|`1.30.14` +|`1.7.28` +|`1.2.1` +| + +|`1.30-2025.09.13` +|`1.30.14` +|`1.7.28` +|`1.2.1` +|Changed GMSA plugin logs to Windows Events + +|`1.30-2025.08.18` +|`1.30.14` +|`1.7.27` +|`1.2.1` +| + +|`1.30-2025.07.16` +|`1.30.13` +|`1.7.27` +|`1.2.1` +| + +|`1.30-2025.06.13` +|`1.30.13` +|`1.7.27` +|`1.2.1` +|Upgraded `containerd` to `1.7.27`. + +|`1.30-2025.05.17` +|`1.30.13` +|`1.7.20` +|`1.2.1` +| + +|`1.30-2025.04.14` +|`1.30.9` +|`1.7.20` +|`1.1.3` +| + +|`1.30-2025.03.14` +|`1.30.9` +|`1.7.20` +|`1.1.3` +|Upgraded `containerd` to `1.7.20`. + +|`1.30-2025.02.15` +|`1.30.9` |`1.7.14` |`1.1.3` | -|`1.26-2024.11.12` -|`1.26.15` +|`1.30-2025.01.15` +|`1.30.8` |`1.7.14` |`1.1.3` | -|`1.26-2024.10.08` -|`1.26.15` +|`1.30-2025.01.01` +|`1.30.8` +|`1.7.14` +|`1.1.3` +|Includes patches for `CVE-2024-9042`. + +|`1.30-2024.12.11` +|`1.30.7` +|`1.7.14` +|`1.1.3` +| + +|`1.30-2024.11.12` +|`1.30.4` +|`1.7.14` +|`1.1.3` +| + +|`1.30-2024.10.08` +|`1.30.4` |`1.7.14` |`1.1.3` | -|`1.26-2024.09.10` -|`1.26.15` +|`1.30-2024.09.10` +|`1.30.2` |`1.7.14` |`1.1.3` | -|`1.26-2024.08.13` -|`1.26.15` +|`1.30-2024.08.13` +|`1.30.2` |`1.7.14` |`1.1.3` | -|`1.26-2024.07.10` -|`1.26.15` -|`1.7.11` +|`1.30-2024.07.10` +|`1.30.2` +|`1.7.14` |`1.1.2` |Includes patches for `CVE-2024-5321`. -|`1.26-2024.06.17` -|`1.26.15` -|`1.7.11` +|`1.30-2024.06.17` +|`1.30.0` +|`1.7.14` |`1.1.2` -|Upgraded `containerd` to `1.7.11`. +|Upgraded `containerd` to `1.7.14`. -|`1.26-2024.05.14` -|`1.26.15` +|`1.30-2024.05.15` +|`1.30.0` |`1.6.28` |`1.1.2` -|Upgraded `containerd` to `1.6.28`. Upgraded `kubelet` to `1.26.15`. - -|`1.26-2024.04.09` -|`1.26.12` -|`1.6.25` -|`1.1.2` -|Upgraded `containerd` to `1.6.25`. Rebuilt CNI and `csi-proxy` using `golang 1.22.1`. - -|`1.26-2024.03.13` -|`1.26.12` -|`1.6.18` -|`1.1.2` -| - -|`1.26-2024.02.13` -|`1.26.12` -|`1.6.18` -|`1.1.2` -| - -|`1.26-2024.01.09` -|`1.26.12` -|`1.6.18` -|`1.1.2` | -|`1.26-2023.12.12` -|`1.26.10` -|`1.6.18` -|`1.1.2` -| +|=== -|`1.26-2023.11.14` -|`1.26.10` -|`1.6.18` -|`1.1.2` -|Includes patches for `CVE-2023-5528`. -|`1.26-2023.10.19` -|`1.26.9` -|`1.6.18` -|`1.1.2` -|Upgraded `containerd` to `1.6.18`. Upgraded `kubelet` to `1.26.9`. Added new <> (`SERVICE_IPV4_CIDR` and `EXCLUDED_SNAT_CIDRS`). +*Kubernetes version 1.29*:: ++ +[%header,cols="5"] +|=== +|AMI version +|kubelet version +|containerd version +|csi-proxy version +|Release notes -|`1.26-2023.09.12` -|`1.26.7` -|`1.6.6` -|`1.1.2` -|Upgraded the Amazon VPC CNI plugin to use the [.noloc]`Kubernetes` connector binary, which gets the [.noloc]`Pod` IP address from the [.noloc]`Kubernetes` API server. Merged https://github.com/aws/amazon-vpc-cni-plugins/pull/100[pull request #100]. +|`1.29-2025.10.18` +|`1.29.15` +|`1.7.28` +|`1.2.1` +| -|`1.26-2023.08.17` -|`1.26.7` -|`1.6.6` -|`1.1.2` -|Includes patches for `CVE-2023-3676`, `CVE-2023-3893`, and `CVE-2023-3955`. +|`1.29-2025.09.13` +|`1.29.15` +|`1.7.28` +|`1.2.1` +|Changed GMSA plugin logs to Windows Events -|`1.26-2023.08.08` -|`1.26.6` -|`1.6.6` -|`1.1.1` +|`1.29-2025.08.18` +|`1.29.15` +|`1.7.27` +|`1.2.1` | -|`1.26-2023.07.11` -|`1.26.6` -|`1.6.6` -|`1.1.1` +|`1.29-2025.07.16` +|`1.29.15` +|`1.7.27` +|`1.2.1` | -|`1.26-2023.06.20` -|`1.26.4` -|`1.6.6` -|`1.1.1` -|Resolved issue that was causing the DNS suffix search list to be incorrectly populated. - -|`1.26-2023.06.14` -|`1.26.4` -|`1.6.6` -|`1.1.1` -|Upgraded [.noloc]`Kubernetes` to `1.26.4`. Added support for host port mapping in CNI. Merged https://github.com/aws/amazon-vpc-cni-plugins/pull/93[pull request #93]. +|`1.29-2025.06.13` +|`1.29.15` +|`1.7.27` +|`1.2.1` +|Upgraded `containerd` to `1.7.27`. -|`1.26-2023.05.09` -|`1.26.2` -|`1.6.6` -|`1.1.1` -|Fixed a bug causing network connectivity https://github.com/aws/containers-roadmap/issues/1126[issue #1126] on pods after node restart. Introduced a new <> (`ExcludedSnatCIDRs`). +|`1.29-2025.05.17` +|`1.29.15` +|`1.7.20` +|`1.2.1` +| -|`1.26-2023.04.26` -|`1.26.2` -|`1.6.6` -|`1.1.1` +|`1.29-2025.04.14` +|`1.29.13` +|`1.7.20` +|`1.1.3` | -|`1.26-2023.04.11` -|`1.26.2` -|`1.6.6` -|`1.1.1` -|Added recovery mechanism for `kubelet` and `kube-proxy` on service crash. +|`1.29-2025.03.14` +|`1.29.13` +|`1.7.20` +|`1.1.3` +|Upgraded `containerd` to `1.7.20`. -|`1.26-2023.03.24` -|`1.26.2` -|`1.6.6` -|`1.1.1` +|`1.29-2025.02.15` +|`1.29.13` +|`1.7.14` +|`1.1.3` | -|=== +|`1.29-2025.01.15` +|`1.29.12` +|`1.7.14` +|`1.1.3` +| -*[.noloc]`Kubernetes` version [.noloc]`1.25`*:: -+ -[cols="1,1,1,1,1", options="header"] -|=== -|AMI version -|kubelet version -|containerd version -|csi-proxy version -|Release notes +|`1.29-2025.01.01` +|`1.29.12` +|`1.7.14` +|`1.1.3` +|Includes patches for `CVE-2024-9042`. -|`1.25-2024.12.13` -|`1.25.16` +|`1.29-2024.12.11` +|`1.29.10` |`1.7.14` |`1.1.3` | -|`1.25-2024.11.12` -|`1.25.16` +|`1.29-2024.11.12` +|`1.29.8` |`1.7.14` |`1.1.3` | -|`1.25-2024.10.08` -|`1.25.16` +|`1.29-2024.10.08` +|`1.29.8` |`1.7.14` |`1.1.3` | -|`1.25-2024.09.10` -|`1.25.16` +|`1.29-2024.09.10` +|`1.29.6` |`1.7.14` |`1.1.3` | -|`1.25-2024.08.13` -|`1.25.16` +|`1.29-2024.08.13` +|`1.29.6` |`1.7.14` |`1.1.3` | -|`1.25-2024.07.10` -|`1.25.16` +|`1.29-2024.07.10` +|`1.29.6` |`1.7.11` |`1.1.2` |Includes patches for `CVE-2024-5321`. -|`1.25-2024.06.17` -|`1.25.16` +|`1.29-2024.06.17` +|`1.29.3` +|`1.7.11` +|`1.1.2` +| + +|`1.29-2024.05.15` +|`1.29.3` |`1.7.11` |`1.1.2` -|Upgraded `containerd` to `1.7.11`. +|Upgraded `containerd` to `1.7.11`. Upgraded `kubelet` to `1.29.3`. -|`1.25-2024.05.14` -|`1.25.16` +|`1.29-2024.04.09` +|`1.29.0` |`1.6.28` |`1.1.2` -|Upgraded `containerd` to `1.6.28`. +|Upgraded `containerd` to `1.6.28`. Rebuilt CNI and `csi-proxy` using `golang 1.22.1`. -|`1.25-2024.04.09` -|`1.25.16` +|`1.29-2024.03.13` +|`1.29.0` |`1.6.25` |`1.1.2` -|Upgraded `containerd` to `1.6.25`. Rebuilt CNI and `csi-proxy` using `golang 1.22.1`. - -|`1.25-2024.03.13` -|`1.25.16` -|`1.6.18` -|`1.1.2` | -|`1.25-2024.02.13` -|`1.25.16` -|`1.6.18` +|`1.29-2024.02.13` +|`1.29.0` +|`1.6.25` |`1.1.2` | -|`1.25-2024.01.09` -|`1.25.16` -|`1.6.18` +|`1.29-2024.02.06` +|`1.29.0` +|`1.6.25` |`1.1.2` -| +|Fixed a bug where the pause image was incorrectly deleted by `kubelet` garbage collection process. -|`1.25-2023.12.12` -|`1.25.15` +|`1.29-2024.01.09` +|`1.29.0` |`1.6.18` |`1.1.2` | -|`1.25-2023.11.14` -|`1.25.15` -|`1.6.18` -|`1.1.2` -|Includes patches for `CVE-2023-5528`. - -|`1.25-2023.10.19` -|`1.25.14` -|`1.6.18` -|`1.1.2` -|Upgraded `containerd` to `1.6.18`. Upgraded `kubelet` to `1.25.14`. Added new <> (`SERVICE_IPV4_CIDR` and `EXCLUDED_SNAT_CIDRS`). +|=== -|`1.25-2023.09.12` -|`1.25.12` -|`1.6.6` -|`1.1.2` -|Upgraded the Amazon VPC CNI plugin to use the [.noloc]`Kubernetes` connector binary, which gets the [.noloc]`Pod` IP address from the [.noloc]`Kubernetes` API server. Merged https://github.com/aws/amazon-vpc-cni-plugins/pull/100[pull request #100]. -|`1.25-2023.08.17` -|`1.25.12` -|`1.6.6` -|`1.1.2` -|Includes patches for `CVE-2023-3676`, `CVE-2023-3893`, and `CVE-2023-3955`. +*Kubernetes version 1.28*:: ++ +[%header,cols="5"] +|=== +|AMI version +|kubelet version +|containerd version +|csi-proxy version +|Release notes -|`1.25-2023.08.08` -|`1.25.9` -|`1.6.6` -|`1.1.1` +|`1.28-2025.10.18` +|`1.28.15` +|`1.7.28` +|`1.2.1` | -|`1.25-2023.07.11` -|`1.25.9` -|`1.6.6` -|`1.1.1` +|`1.28-2025.09.13` +|`1.28.15` +|`1.7.28` +|`1.2.1` +|Changed GMSA plugin logs to Windows Events + +|`1.28-2025.08.18` +|`1.28.15` +|`1.7.27` +|`1.2.1` | -|`1.25-2023.06.20` -|`1.25.9` -|`1.6.6` -|`1.1.1` -|Resolved issue that was causing the DNS suffix search list to be incorrectly populated. +|`1.28-2025.07.16` +|`1.28.15` +|`1.7.27` +|`1.2.1` +| -|`1.25-2023.06.14` -|`1.25.9` -|`1.6.6` -|`1.1.1` -|Upgraded [.noloc]`Kubernetes` to `1.25.9`. Added support for host port mapping in CNI. Merged https://github.com/aws/amazon-vpc-cni-plugins/pull/93[pull request #93]. +|`1.28-2025.06.13` +|`1.28.15` +|`1.7.27` +|`1.2.1` +|Upgraded `containerd` to `1.7.27`. -|`1.25-2023.05.09` -|`1.25.7` -|`1.6.6` -|`1.1.1` -|Fixed a bug causing network connectivity https://github.com/aws/containers-roadmap/issues/1126[issue #1126] on pods after node restart. Introduced a new <> (`ExcludedSnatCIDRs`). +|`1.28-2025.05.17` +|`1.28.15` +|`1.7.20` +|`1.2.1` +| -|`1.25-2023.04.11` -|`1.25.7` -|`1.6.6` -|`1.1.1` -|Added recovery mechanism for `kubelet` and `kube-proxy` on service crash. +|`1.28-2025.04.14` +|`1.28.15` +|`1.7.20` +|`1.1.3` +| -|`1.25-2023.03.27` -|`1.25.6` -|`1.6.6` -|`1.1.1` -|Installed a link:containers/domainless-windows-authentication-for-amazon-eks-windows-pods[domainless gMSA plugin,type="blog"] to facilitate [.noloc]`gMSA` authentication for [.noloc]`Windows` containers on Amazon EKS. +|`1.28-2025.03.14` +|`1.28.15` +|`1.7.20` +|`1.1.3` +|Upgraded `containerd` to `1.7.20`. -|`1.25-2023.03.20` -|`1.25.6` -|`1.6.6` -|`1.1.1` +|`1.28-2025.02.15` +|`1.28.15` +|`1.7.14` +|`1.1.3` | -|`1.25-2023.02.14` -|`1.25.6` -|`1.6.6` -|`1.1.1` +|`1.28-2025-01-15` +|`1.28.15` +|`1.7.14` +|`1.1.3` | -|=== - -*[.noloc]`Kubernetes` version [.noloc]`1.24`*:: -+ -[cols="1,1,1,1,1", options="header"] -|=== -|AMI version -|kubelet version -|containerd version -|csi-proxy version -|Release notes +|`1.28-2025-01-01` +|`1.28.15` +|`1.7.14` +|`1.1.3` +|Includes patches for `CVE-2024-9042`. -|`1.24-2024.12.11` -|`1.24.17` +|`1.28-2024.12.11` +|`1.28.15` |`1.7.14` |`1.1.3` | -|`1.24-2024.11.12` -|`1.24.17` +|`1.28-2024.11.12` +|`1.28.13` |`1.7.14` |`1.1.3` | -|`1.24-2024.10.08` -|`1.24.17` +|`1.28-2024.10.08` +|`1.28.13` |`1.7.14` |`1.1.3` | -|`1.24-2024.09.10` -|`1.24.17` +|`1.28-2024.09.10` +|`1.28.11` |`1.7.14` |`1.1.3` | -|`1.24-2024.08.13` -|`1.24.17` +|`1.28-2024.08.13` +|`1.28.11` |`1.7.14` |`1.1.3` | -|`1.24-2024.07.10` -|`1.24.17` +|`1.28-2024.07.10` +|`1.28.11` |`1.7.11` |`1.1.2` |Includes patches for `CVE-2024-5321`. -|`1.24-2024.06.17` -|`1.24.17` +|`1.28-2024.06.17` +|`1.28.8` |`1.7.11` |`1.1.2` |Upgraded `containerd` to `1.7.11`. -|`1.24-2024.05.14` -|`1.24.17` +|`1.28-2024.05.14` +|`1.28.8` |`1.6.28` |`1.1.2` -|Upgraded `containerd` to `1.6.28`. +|Upgraded `containerd` to `1.6.28`. Upgraded `kubelet` to `1.28.8`. -|`1.24-2024.04.09` -|`1.24.17` +|`1.28-2024.04.09` +|`1.28.5` |`1.6.25` |`1.1.2` |Upgraded `containerd` to `1.6.25`. Rebuilt CNI and `csi-proxy` using `golang 1.22.1`. -|`1.24-2024.03.13` -|`1.24.17` +|`1.28-2024.03.13` +|`1.28.5` |`1.6.18` |`1.1.2` | -|`1.24-2024.02.13` -|`1.24.17` +|`1.28-2024.02.13` +|`1.28.5` |`1.6.18` |`1.1.2` | -|`1.24-2024.01.09` -|`1.24.17` +|`1.28-2024.01.09` +|`1.28.5` |`1.6.18` |`1.1.2` | -|`1.24-2023.12.12` -|`1.24.17` +|`1.28-2023.12.12` +|`1.28.3` |`1.6.18` |`1.1.2` | -|`1.24-2023.11.14` -|`1.24.17` +|`1.28-2023.11.14` +|`1.28.3` |`1.6.18` |`1.1.2` |Includes patches for `CVE-2023-5528`. -|`1.24-2023.10.19` -|`1.24.17` +|`1.28-2023.10.19` +|`1.28.2` |`1.6.18` |`1.1.2` -|Upgraded `containerd` to `1.6.18`. Upgraded `kubelet` to `1.24.17`. Added new <> (`SERVICE_IPV4_CIDR` and `EXCLUDED_SNAT_CIDRS`). +|Upgraded `containerd` to `1.6.18`. Added new <> (`SERVICE_IPV4_CIDR` and `EXCLUDED_SNAT_CIDRS`). -|`1.24-2023.09.12` -|`1.24.16` +|`1.28-2023-09.27` +|`1.28.2` |`1.6.6` |`1.1.2` -|Upgraded the Amazon VPC CNI plugin to use the [.noloc]`Kubernetes` connector binary, which gets the [.noloc]`Pod` IP address from the [.noloc]`Kubernetes` API server. Merged https://github.com/aws/amazon-vpc-cni-plugins/pull/100[pull request #100]. +|Fixed a https://github.com/advisories/GHSA-6xv5-86q9-7xr8[security advisory] in `kubelet`. -|`1.24-2023.08.17` -|`1.24.16` +|`1.28-2023.09.12` +|`1.28.1` |`1.6.6` |`1.1.2` -|Includes patches for `CVE-2023-3676`, `CVE-2023-3893`, and `CVE-2023-3955`. - -|`1.24-2023.08.08` -|`1.24.13` -|`1.6.6` -|`1.1.1` -| - -|`1.24-2023.07.11` -|`1.24.13` -|`1.6.6` -|`1.1.1` -| - -|`1.24-2023.06.21` -|`1.24.13` -|`1.6.6` -|`1.1.1` -|Resolved issue that was causing the DNS suffix search list to be incorrectly populated. - -|`1.24-2023.06.14` -|`1.24.13` -|`1.6.6` -|`1.1.1` -|Upgraded [.noloc]`Kubernetes` to `1.24.13`. Added support for host port mapping in CNI. Merged https://github.com/aws/amazon-vpc-cni-plugins/pull/93[pull request #93]. - -|`1.24-2023.05.09` -|`1.24.7` -|`1.6.6` -|`1.1.1` -|Fixed a bug causing network connectivity https://github.com/aws/containers-roadmap/issues/1126[issue #1126] on pods after node restart. Introduced a new <> (`ExcludedSnatCIDRs`). - -|`1.24-2023.04.11` -|`1.24.7` -|`1.6.6` -|`1.1.1` -|Added recovery mechanism for `kubelet` and `kube-proxy` on service crash. - -|`1.24-2023.03.27` -|`1.24.7` -|`1.6.6` -|`1.1.1` -|Installed a link:containers/domainless-windows-authentication-for-amazon-eks-windows-pods[domainless gMSA plugin,type="blog"] to facilitate [.noloc]`gMSA` authentication for [.noloc]`Windows` containers on Amazon EKS. - -|`1.24-2023.03.20` -|`1.24.7` -|`1.6.6` -|`1.1.1` -|[.noloc]`Kubernetes` version downgraded to `1.24.7` because `1.24.10` has a reported issue in `kube-proxy`. - -|`1.24-2023.02.14` -|`1.24.10` -|`1.6.6` -|`1.1.1` | -|`1.24-2023.01.23` -|`1.24.7` -|`1.6.6` -|`1.1.1` -| +|=== -|`1.24-2023.01.11` -|`1.24.7` -|`1.6.6` -|`1.1.1` -| -|`1.24-2022.12.14` -|`1.24.7` -|`1.6.6` -|`1.1.1` -| -|`1.24-2022.10.12` -|`1.24.7` -|`1.6.6` -|`1.1.1` -| -|=== -==== +==== \ No newline at end of file diff --git a/latest/ug/nodes/eks-compute.adoc b/latest/ug/nodes/eks-compute.adoc index b730be14e..785a6d239 100644 --- a/latest/ug/nodes/eks-compute.adoc +++ b/latest/ug/nodes/eks-compute.adoc @@ -1,44 +1,24 @@ -//!!NODE_ROOT -[[eks-compute,eks-compute.title]] +include::../attributes.txt[] + +[#eks-compute] = Manage compute resources by using nodes -:doctype: book -:sectnums: -:toc: left -:icons: font -:experimental: -:idprefix: -:idseparator: - -:sourcedir: . -:info_doctype: chapter -:info_title: Manage compute resources by using nodes :info_titleabbrev: Manage compute -:keywords: nodes, node groups -:info_abstract: Your Amazon EKS cluster can schedule Pods on any combination of self-managed nodes, Amazon EKS managed node groups, and Fargate in the {aws} Cloud and hybrid nodes on-premises. - -include::../attributes.txt[] [abstract] -- -Your Amazon EKS cluster can schedule [.noloc]`Pods` on any combination of self-managed nodes, Amazon EKS managed node groups, Fargate, and Amazon EKS Hybrid Nodes in the {aws} Cloud and hybrid nodes on-premises. +Your Amazon EKS cluster can schedule Pods on any combination of self-managed nodes, Amazon EKS managed node groups, Fargate, and Amazon EKS Hybrid Nodes in the {aws} Cloud and hybrid nodes on-premises. -- -A [.noloc]`Kubernetes` node is a machine that runs containerized applications. Each node has the following components: +A Kubernetes node is a machine that runs containerized applications. Each node has the following components: * *https://kubernetes.io/docs/setup/production-environment/container-runtimes/[Container runtime]* – Software that's responsible for running the containers. -* *https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/[kubelet]* – Makes sure that containers are healthy and running within their associated [.noloc]`Pod`. -* *https://kubernetes.io/docs/reference/command-line-tools-reference/kube-proxy/[kube-proxy]* – Maintains network rules that allow communication to your [.noloc]`Pods`. +* *https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/[kubelet]* – Makes sure that containers are healthy and running within their associated Pod. +* *https://kubernetes.io/docs/reference/command-line-tools-reference/kube-proxy/[kube-proxy]* – Maintains network rules that allow communication to your Pods. -For more information, see https://kubernetes.io/docs/concepts/architecture/nodes/[Nodes] in the [.noloc]`Kubernetes` documentation. +For more information, see https://kubernetes.io/docs/concepts/architecture/nodes/[Nodes] in the Kubernetes documentation. -Your Amazon EKS cluster can schedule [.noloc]`Pods` on any combination of <>, <>, <>, <>, and <>. To learn more about nodes deployed in your cluster, see <>. +Your Amazon EKS cluster can schedule Pods on any combination of <>, <>, <>, <>, and <>. To learn more about nodes deployed in your cluster, see <>. -[IMPORTANT] -==== - -{aws} Fargate with Amazon EKS isn't available in {aws} GovCloud (US-East) and {aws} GovCloud (US-West). -Amazon EKS Hybrid Nodes isn't available in {aws} GovCloud Regions and China Regions. - -==== [NOTE] ==== @@ -54,19 +34,20 @@ The following table provides several criteria to evaluate when deciding which op [NOTE] ==== -[.noloc]`Bottlerocket` has some specific differences from the general information in this table. For more information, see the [.noloc]`Bottlerocket` https://github.com/bottlerocket-os/bottlerocket/blob/develop/README.md[documentation] on [.noloc]`GitHub`. +Bottlerocket has some specific differences from the general information in this table. For more information, see the Bottlerocket https://github.com/bottlerocket-os/bottlerocket/blob/develop/README.md[documentation] on GitHub. ==== [role="no-scroll"] -[cols="1,1,1,1", options="header"] +[%header,cols="4"] |=== |Criteria |EKS managed node groups |EKS Auto Mode |Amazon EKS Hybrid Nodes + |Can be deployed to link:outposts/latest/userguide/what-is-outposts.html[{aws} Outposts,type="documentation"] |No |No @@ -77,12 +58,12 @@ The following table provides several criteria to evaluate when deciding which op |No |No -|Can run containers that require [.noloc]`Windows` +|Can run containers that require Windows |Yes |No |No -|Can run containers that require [.noloc]`Linux` +|Can run containers that require Linux |Yes |Yes |Yes @@ -118,7 +99,7 @@ The following table provides several criteria to evaluate when deciding which op |Must deploy and manage Amazon EC2 instances |Yes -|No - Learn about xref:automode-learn-instances[EC2 managed instances] +|No - Learn about <> |Yes – the on-premises physical or virtual machines are managed by you with your choice of tooling. |Must secure, maintain, and patch the operating system of Amazon EC2 instances @@ -128,17 +109,17 @@ The following table provides several criteria to evaluate when deciding which op |Can provide bootstrap arguments at deployment of a node, such as extra https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/[kubelet] arguments. |Yes – Using `eksctl` or a <> with a custom AMI. -|No - xref:create-node-class[Use a `NodeClass` to configure nodes] +|No - <> |Yes - you can customize bootstrap arguments with nodeadm. See <>. -|Can assign IP addresses to [.noloc]`Pods` from a different CIDR block than the IP address assigned to the node. +|Can assign IP addresses to Pods from a different CIDR block than the IP address assigned to the node. |Yes – Using a launch template with a custom AMI. For more information, see <>. |No |Yes - see <>. |Can SSH into node |Yes -|No - xref:auto-troubleshoot[Learn how to troubleshoot nodes] +|No - <> |Yes |Can deploy your own custom AMI to nodes @@ -156,22 +137,22 @@ The following table provides several criteria to evaluate when deciding which op |No |Yes - the operating system running on your physical or virtual machines is managed by you with your choice of tooling. See <>. -|Must update node [.noloc]`Kubernetes` version on your own +|Must update node Kubernetes version on your own |<> – If you deployed an Amazon EKS optimized AMI, you're notified in the Amazon EKS console when updates are available. You can perform the update with one-click in the console. If you deployed a custom AMI, you're not notified in the Amazon EKS console when updates are available. You must perform the update on your own. |No |Yes - you manage hybrid nodes upgrades with your own choice of tooling or with `nodeadm`. See <>. -|Can use Amazon EBS storage with [.noloc]`Pods` +|Can use Amazon EBS storage with Pods |<> -|Yes, as an integrated capability. Learn how to xref:create-storage-class[create a storage class.] +|Yes, as an integrated capability. Learn how to <> |No -|Can use Amazon EFS storage with [.noloc]`Pods` +|Can use Amazon EFS storage with Pods |<> |Yes |No -|Can use Amazon FSx for Lustre storage with [.noloc]`Pods` +|Can use Amazon FSx for Lustre storage with Pods |<> |Yes |No @@ -186,17 +167,17 @@ The following table provides several criteria to evaluate when deciding which op |Yes |No - pods run in on-premises environment. -|Can assign different VPC security groups to individual [.noloc]`Pods` -|<> – [.noloc]`Linux` nodes only +|Can assign different VPC security groups to individual Pods +|<> – Linux nodes only |No |No -|Can run [.noloc]`Kubernetes` [.noloc]`DaemonSets` +|Can run Kubernetes DaemonSets |Yes |Yes |Yes -|Support `HostPort` and `HostNetwork` in the [.noloc]`Pod` manifest +|Support `HostPort` and `HostNetwork` in the Pod manifest |Yes |Yes |Yes @@ -212,7 +193,7 @@ The following table provides several criteria to evaluate when deciding which op |No |Pricing -|Cost of Amazon EC2 instance that runs multiple [.noloc]`Pods`. For more information, see link:ec2/pricing/[Amazon EC2 pricing,type="marketing"]. +|Cost of Amazon EC2 instance that runs multiple Pods. For more information, see link:ec2/pricing/[Amazon EC2 pricing,type="marketing"]. | When EKS Auto Mode is enabled in your cluster, you pay a separate fee, in addition to the standard EC2 instance charges, for the instances launched using Auto Mode's compute capability. The amount varies with the instance type launched and the {aws} region where your cluster is located. For more information, see link:eks/pricing/["Amazon EKS pricing",type="marketing"]. |Cost of hybrid nodes vCPU per hour. For more information, see link:eks/pricing/[Amazon EKS pricing,type="marketing"]. @@ -230,4 +211,4 @@ include::eks-optimized-amis.adoc[leveloffset=+1] include::node-health.adoc[leveloffset=+1] -include::hybrid-nodes.adoc[leveloffset=+1] +include::hybrid-nodes-overview.adoc[leveloffset=+1] \ No newline at end of file diff --git a/latest/ug/nodes/eks-custom-ami-windows.adoc b/latest/ug/nodes/eks-custom-ami-windows.adoc index 273d87e76..a50c84be2 100644 --- a/latest/ug/nodes/eks-custom-ami-windows.adoc +++ b/latest/ug/nodes/eks-custom-ami-windows.adoc @@ -1,16 +1,16 @@ -//!!NODE_ROOT
include::../attributes.txt[] + [.topic] -[[eks-custom-ami-windows,eks-custom-ami-windows.title]] -= Build a custom [.noloc]`Windows` AMI with Image Builder +[#eks-custom-ami-windows] += Build a custom Windows AMI with Image Builder :info_titleabbrev: Custom builds [abstract] -- -You can use EC2 Image Builder to create custom Amazon EKS optimized [.noloc]`Windows` AMIs. +You can use EC2 Image Builder to create custom Amazon EKS optimized Windows AMIs. -- -You can use EC2 Image Builder to create custom Amazon EKS optimized [.noloc]`Windows` AMIs with one of the following options: +You can use EC2 Image Builder to create custom Amazon EKS optimized Windows AMIs with one of the following options: @@ -24,92 +24,85 @@ With both methods, you must create your own Image Builder recipe. For more infor The following *Amazon-managed* components for `eks` include patches for `CVE-2024-5321`. -* `1.24.5` and higher -* `1.25.4` and higher -* `1.26.4` and higher -* `1.27.2` and higher * `1.28.2` and higher * `1.29.2` and higher * `1.30.1` and higher +* All versions for Kubernetes 1.31 and higher ==== -[[custom-windows-ami-as-base,custom-windows-ami-as-base.title]] -== Using an Amazon EKS optimized [.noloc]`Windows` AMI as a base +[#custom-windows-ami-as-base] +== Using an Amazon EKS optimized Windows AMI as a base -This option is the recommended way to build your custom [.noloc]`Windows` AMIs. The Amazon EKS optimized [.noloc]`Windows` AMIs we provide are more frequently updated than the Amazon-managed build component. +This option is the recommended way to build your custom Windows AMIs. The Amazon EKS optimized Windows AMIs we provide are more frequently updated than the Amazon-managed build component. . Start a new Image Builder recipe. + .. Open the EC2 Image Builder console at https://console.aws.amazon.com/imagebuilder. .. In the left navigation pane, choose *Image recipes*. .. Choose *Create image recipe*. -. In the *Recipe details* section, enter a *Name* and *Version*. -. Specify the ID of the Amazon EKS optimized [.noloc]`Windows` AMI in the *Base image* section. +. In the *Recipe details* section, enter a *Name* and *Version*. +. Specify the ID of the Amazon EKS optimized Windows AMI in the *Base image* section. + .. Choose *Enter custom AMI ID*. -.. Retrieve the AMI ID for the [.noloc]`Windows` OS version that you require. For more information, see <>. +.. Retrieve the AMI ID for the Windows OS version that you require. For more information, see <>. .. Enter the custom *AMI ID*. If the AMI ID isn't found, make sure that the {aws} Region for the AMI ID matches the {aws} Region shown in the upper right of your console. . (Optional) To get the latest security updates, add the `update-windows` component in the *Build components -* section. + .. From the dropdown list to the right of the *Find components by name* search box, choose *Amazon-managed*. .. In the *Find components by name* search box, enter `update-windows`. -.. Select the check box of the *`update-windows`* search result. This component includes the latest [.noloc]`Windows` patches for the operating system. -. Complete the remaining image recipe inputs with your required configurations. For more information, see link:imagebuilder/latest/userguide/create-image-recipes.html#create-image-recipe-version-console[Create a new image recipe version (console),type="documentation"] in the Image Builder User Guide. +.. Select the check box of the *`update-windows`* search result. This component includes the latest Windows patches for the operating system. +. Complete the remaining image recipe inputs with your required configurations. For more information, see link:imagebuilder/latest/userguide/create-image-recipes.html#create-image-recipe-version-console[Create a new image recipe version (console),type="documentation"] in the Image Builder User Guide. . Choose *Create recipe*. . Use the new image recipe in a new or existing image pipeline. Once your image pipeline runs successfully, your custom AMI will be listed as an output image and is ready for use. For more information, see link:imagebuilder/latest/userguide/start-build-image-pipeline.html[Create an image pipeline using the EC2 Image Builder console wizard,type="documentation"]. -[[custom-windows-ami-build-component,custom-windows-ami-build-component.title]] +[#custom-windows-ami-build-component] == Using the Amazon-managed build component -When using an Amazon EKS optimized [.noloc]`Windows` AMI as a base isn't viable, you can use the Amazon-managed build component instead. This option may lag behind the most recent supported [.noloc]`Kubernetes` versions. +When using an Amazon EKS optimized Windows AMI as a base isn't viable, you can use the Amazon-managed build component instead. This option may lag behind the most recent supported Kubernetes versions. . Start a new Image Builder recipe. + .. Open the EC2 Image Builder console at https://console.aws.amazon.com/imagebuilder. .. In the left navigation pane, choose *Image recipes*. .. Choose *Create image recipe*. -. In the *Recipe details* section, enter a *Name* and *Version*. +. In the *Recipe details* section, enter a *Name* and *Version*. . Determine which option you will be using to create your custom AMI in the *Base image* section: + -** *Select managed images* – Choose *Windows* for your *Image Operating System (OS)*. Then choose one of the following options for *Image origin*. +** *Select managed images* – Choose *Windows* for your *Image Operating System (OS)*. Then choose one of the following options for *Image origin*. + -*** *Quick start (Amazon-managed)* – In the *Image name* dropdown, choose an Amazon EKS supported [.noloc]`Windows` Server version. For more information, see <>. -*** *Images owned by me* – For *Image name*, choose the ARN of your own image with your own license. The image that you provide can't already have Amazon EKS components installed. +*** *Quick start (Amazon-managed)* – In the *Image name* dropdown, choose an Amazon EKS supported Windows Server version. For more information, see <>. +*** *Images owned by me* – For *Image name*, choose the ARN of your own image with your own license. The image that you provide can't already have Amazon EKS components installed. ** *Enter custom AMI ID* – For AMI ID, enter the ID for your AMI with your own license. The image that you provide can't already have Amazon EKS components installed. . In the *Build components - Windows* section, do the following: + -.. From the dropdown list to the right of the *Find components by name* search box, choose *Amazon-managed*. +.. From the dropdown list to the right of the *Find components by name* search box, choose *Amazon-managed*. .. In the *Find components by name* search box, enter `eks`. .. Select the check box of the *`eks-optimized-ami-windows`* search result, even though the result returned may not be the version that you want. .. In the *Find components by name* search box, enter `update-windows` . -.. Select the check box of the *update-windows* search result. This component includes the latest [.noloc]`Windows` patches for the operating system. +.. Select the check box of the *update-windows* search result. This component includes the latest Windows patches for the operating system. . In the *Selected components* section, do the following: + -.. Choose *Versioning options* for *`eks-optimized-ami-windows`*. +.. Choose *Versioning options* for *`eks-optimized-ami-windows`*. .. Choose *Specify component version*. -.. In the *Component Version* field, enter [.replaceable]`version.x`, replacing [.replaceable]`version` with a supported [.noloc]`Kubernetes` version. Entering an [.replaceable]`x` for part of the version number indicates to use the latest component version that also aligns with the part of the version you explicitly define. Pay attention to the console output as it will advise you on whether your desired version is available as a managed component. Keep in mind that the most recent [.noloc]`Kubernetes` versions may not be available for the build component. For more information about available versions, see <>. -+ -NOTE: The following `eks-optimized-ami-windows` build component versions require `eksctl` version `0.129` or lower: - -*** `1.24.0` +.. In the *Component Version* field, enter [.replaceable]`version.x`, replacing [.replaceable]`version` with a supported Kubernetes version. Entering an [.replaceable]`x` for part of the version number indicates to use the latest component version that also aligns with the part of the version you explicitly define. Pay attention to the console output as it will advise you on whether your desired version is available as a managed component. Keep in mind that the most recent Kubernetes versions may not be available for the build component. For more information about available versions, see <>. -. Complete the remaining image recipe inputs with your required configurations. For more information, see link:imagebuilder/latest/userguide/create-image-recipes.html#create-image-recipe-version-console[Create a new image recipe version (console),type="documentation"] in the Image Builder User Guide. +. Complete the remaining image recipe inputs with your required configurations. For more information, see link:imagebuilder/latest/userguide/create-image-recipes.html#create-image-recipe-version-console[Create a new image recipe version (console),type="documentation"] in the Image Builder User Guide. . Choose *Create recipe*. . Use the new image recipe in a new or existing image pipeline. Once your image pipeline runs successfully, your custom AMI will be listed as an output image and is ready for use. For more information, see link:imagebuilder/latest/userguide/start-build-image-pipeline.html[Create an image pipeline using the EC2 Image Builder console wizard,type="documentation"]. -[[custom-windows-ami-component-versions,custom-windows-ami-component-versions.title]] +[#custom-windows-ami-component-versions] == Retrieving information about `eks-optimized-ami-windows` component versions -You can retrieve specific information regarding what is installed with each component. For example, you can verify what `kubelet` version is installed. The components go through functional testing on the Amazon EKS supported [.noloc]`Windows` operating systems versions. For more information, see <>. Any other [.noloc]`Windows` OS versions that aren't listed as supported or have reached end of support might not be compatible with the component. +You can retrieve specific information regarding what is installed with each component. For example, you can verify what `kubelet` version is installed. The components go through functional testing on the Amazon EKS supported Windows operating systems versions. For more information, see <>. Any other Windows OS versions that aren't listed as supported or have reached end of support might not be compatible with the component. . Open the EC2 Image Builder console at https://console.aws.amazon.com/imagebuilder. . In the left navigation pane, choose *Components*. -. From the dropdown list to the right of the *Find components by name* search box, change *Owned by me* to *Quick start (Amazon-managed)*. +. From the dropdown list to the right of the *Find components by name* search box, change *Owned by me* to *Quick start (Amazon-managed)*. . In the *Find components by name* box, enter `eks`. . (Optional) If you are using a recent version, sort the *Version* column in descending order by choosing it twice. . Choose the *`eks-optimized-ami-windows`* link with a desired version. -The *Description* in the resulting page shows the specific information. +The *Description* in the resulting page shows the specific information. \ No newline at end of file diff --git a/latest/ug/nodes/eks-linux-ami-versions.adoc b/latest/ug/nodes/eks-linux-ami-versions.adoc index 4c91a40ea..ce7f7df46 100644 --- a/latest/ug/nodes/eks-linux-ami-versions.adoc +++ b/latest/ug/nodes/eks-linux-ami-versions.adoc @@ -1,7 +1,7 @@ -//!!NODE_ROOT
include::../attributes.txt[] + [.topic] -[[eks-linux-ami-versions,eks-linux-ami-versions.title]] +[#eks-linux-ami-versions] = Retrieve Amazon Linux AMI version information :info_titleabbrev: Get version information @@ -10,11 +10,11 @@ include::../attributes.txt[] This topic gives the location of Amazon EKS optimized Amazon Linux AMIs version information. -- -Amazon EKS optimized Amazon Linux AMIs are versioned by [.noloc]`Kubernetes` version and the release date of the AMI in the following format: +Amazon EKS optimized Amazon Linux AMIs are versioned by Kubernetes version and the release date of the AMI in the following format: [source,none,subs="verbatim,attributes"] ---- k8s_major_version.k8s_minor_version.k8s_patch_version-release_date ---- -Each AMI release includes various versions of https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/[kubelet], the [.noloc]`Linux` kernel, and https://containerd.io/[containerd]. The accelerated AMIs also include various versions of the [.noloc]`NVIDIA` driver. You can find this version information in the https://github.com/awslabs/amazon-eks-ami/blob/main/CHANGELOG.md[Changelog] on [.noloc]`GitHub`. +Each AMI release includes various versions of https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/[kubelet], the Linux kernel, and https://containerd.io/[containerd]. The accelerated AMIs also include various versions of the NVIDIA driver. You can find this version information in the https://github.com/awslabs/amazon-eks-ami/blob/main/CHANGELOG.md[Changelog] on GitHub. \ No newline at end of file diff --git a/latest/ug/nodes/eks-optimized-ami-bottlerocket.adoc b/latest/ug/nodes/eks-optimized-ami-bottlerocket.adoc index 91ce64da0..d0bb545bd 100644 --- a/latest/ug/nodes/eks-optimized-ami-bottlerocket.adoc +++ b/latest/ug/nodes/eks-optimized-ami-bottlerocket.adoc @@ -1,58 +1,53 @@ -//!!NODE_ROOT
include::../attributes.txt[] + [.topic] -[[eks-optimized-ami-bottlerocket,eks-optimized-ami-bottlerocket.title]] -= Create nodes with optimized [.noloc]`Bottlerocket` AMIs +[#eks-optimized-ami-bottlerocket] += Create nodes with optimized Bottlerocket AMIs :info_titleabbrev: Bottlerocket [abstract] -- -[.noloc]`Bottlerocket` is an open source [.noloc]`Linux` distribution that's sponsored and supported by {aws}. [.noloc]`Bottlerocket` includes only the essential software to run containers, which improves resource usage, reduces security threats, and lowers management overhead. +Bottlerocket is an open source Linux distribution that's sponsored and supported by {aws}. Bottlerocket includes only the essential software to run containers, which improves resource usage, reduces security threats, and lowers management overhead. -- -link:bottlerocket/[Bottlerocket,type="marketing"] is an open source [.noloc]`Linux` distribution that's sponsored and supported by {aws}. [.noloc]`Bottlerocket` is purpose-built for hosting container workloads. With [.noloc]`Bottlerocket`, you can improve the availability of containerized deployments and reduce operational costs by automating updates to your container infrastructure. [.noloc]`Bottlerocket` includes only the essential software to run containers, which improves resource usage, reduces security threats, and lowers management overhead. The [.noloc]`Bottlerocket` AMI includes `containerd`, `kubelet`, and {aws} IAM Authenticator. In addition to managed node groups and self-managed nodes, [.noloc]`Bottlerocket` is also supported by https://karpenter.sh/[Karpenter]. +link:bottlerocket/[Bottlerocket,type="marketing"] is an open source Linux distribution that's sponsored and supported by {aws}. Bottlerocket is purpose-built for hosting container workloads. With Bottlerocket, you can improve the availability of containerized deployments and reduce operational costs by automating updates to your container infrastructure. Bottlerocket includes only the essential software to run containers, which improves resource usage, reduces security threats, and lowers management overhead. The Bottlerocket AMI includes `containerd`, `kubelet`, and {aws} IAM Authenticator. In addition to managed node groups and self-managed nodes, Bottlerocket is also supported by https://karpenter.sh/[Karpenter]. -[[bottlerocket-advantages,bottlerocket-advantages.title]] +[#bottlerocket-advantages] == Advantages -Using [.noloc]`Bottlerocket` with your Amazon EKS cluster has the following advantages: - - - -* *Higher uptime with lower operational cost and lower management complexity* – [.noloc]`Bottlerocket` has a smaller resource footprint, shorter boot times, and is less vulnerable to security threats than other [.noloc]`Linux` distributions. [.noloc]`Bottlerocket's` smaller footprint helps to reduce costs by using less storage, compute, and networking resources. -* *Improved security from automatic OS updates* – Updates to [.noloc]`Bottlerocket` are applied as a single unit which can be rolled back, if necessary. This removes the risk of corrupted or failed updates that can leave the system in an unusable state. With [.noloc]`Bottlerocket`, security updates can be automatically applied as soon as they're available in a minimally disruptive manner and be rolled back if failures occur. -* *Premium support* – {aws} provided builds of [.noloc]`Bottlerocket` on Amazon EC2 is covered under the same {aws} Support plans that also cover {aws} services such as Amazon EC2, Amazon EKS, and Amazon ECR. +Using Bottlerocket with your Amazon EKS cluster has the following advantages: +* *Higher uptime with lower operational cost and lower management complexity* – Bottlerocket has a smaller resource footprint, shorter boot times, and is less vulnerable to security threats than other Linux distributions. Bottlerocket's smaller footprint helps to reduce costs by using less storage, compute, and networking resources. +* *Improved security from automatic OS updates* – Updates to Bottlerocket are applied as a single unit which can be rolled back, if necessary. This removes the risk of corrupted or failed updates that can leave the system in an unusable state. With Bottlerocket, security updates can be automatically applied as soon as they're available in a minimally disruptive manner and be rolled back if failures occur. +* *Premium support* – {aws} provided builds of Bottlerocket on Amazon EC2 is covered under the same {aws} Support plans that also cover {aws} services such as Amazon EC2, Amazon EKS, and Amazon ECR. -[[bottlerocket-considerations,bottlerocket-considerations.title]] +[#bottlerocket-considerations] == Considerations -Consider the following when using [.noloc]`Bottlerocket` for your AMI type: +Consider the following when using Bottlerocket for your AMI type: - - -* [.noloc]`Bottlerocket` supports Amazon EC2 instances with `x86_64` and `arm64` processors. The [.noloc]`Bottlerocket` AMI isn't recommended for use with Amazon EC2 instances with an Inferentia chip. -* [.noloc]`Bottlerocket` images don't include an SSH server or a shell. You can employ out-of-band access methods to allow SSH. These approaches enable the admin container and to pass some bootstrapping configuration steps with user data. For more information, refer to the following sections in https://github.com/bottlerocket-os/bottlerocket/blob/develop/README.md[Bottlerocket OS] on [.noloc]`GitHub`: +* Bottlerocket supports Amazon EC2 instances with `x86_64` and `arm64` processors. +* Bottlerocket supports Amazon EC2 instances with GPUs. For more information, see <>. +* Bottlerocket images don't include an SSH server or a shell. You can employ out-of-band access methods to allow SSH. These approaches enable the admin container and to pass some bootstrapping configuration steps with user data. For more information, refer to the following sections in https://github.com/bottlerocket-os/bottlerocket/blob/develop/README.md[Bottlerocket OS] on GitHub: + ** https://github.com/bottlerocket-os/bottlerocket/blob/develop/README.md#exploration[Exploration] ** https://github.com/bottlerocket-os/bottlerocket/blob/develop/README.md#admin-container[Admin container] ** https://github.com/bottlerocket-os/bottlerocket/blob/develop/README.md#kubernetes-settings[Kubernetes settings] -* [.noloc]`Bottlerocket` uses different container types: +* Bottlerocket uses different container types: + -** By default, a https://github.com/bottlerocket-os/bottlerocket-control-container[control container] is enabled. This container runs the https://github.com/aws/amazon-ssm-agent[{aws} Systems Manager agent] that you can use to run commands or start shell sessions on Amazon EC2 [.noloc]`Bottlerocket` instances. For more information, see link:systems-manager/latest/userguide/session-manager-getting-started.html[Setting up Session Manager,type="documentation"] in the _{aws} Systems Manager User Guide_. -** If an SSH key is given when creating the node group, an admin container is enabled. We recommend using the admin container only for development and testing scenarios. We don't recommend using it for production environments. For more information, see https://github.com/bottlerocket-os/bottlerocket/blob/develop/README.md#admin-container[Admin container] on [.noloc]`GitHub`. +** By default, a https://github.com/bottlerocket-os/bottlerocket-control-container[control container] is enabled. This container runs the https://github.com/aws/amazon-ssm-agent[{aws} Systems Manager agent] that you can use to run commands or start shell sessions on Amazon EC2 Bottlerocket instances. For more information, see link:systems-manager/latest/userguide/session-manager-getting-started.html[Setting up Session Manager,type="documentation"] in the _{aws} Systems Manager User Guide_. +** If an SSH key is given when creating the node group, an admin container is enabled. We recommend using the admin container only for development and testing scenarios. We don't recommend using it for production environments. For more information, see https://github.com/bottlerocket-os/bottlerocket/blob/develop/README.md#admin-container[Admin container] on GitHub. - -[[bottlerocket-more-information,bottlerocket-more-information.title]] +[#bottlerocket-more-information] == More information -For more information about using Amazon EKS optimized [.noloc]`Bottlerocket` AMIs, see the following sections: +For more information about using Amazon EKS optimized Bottlerocket AMIs, see the following sections: -* For details about [.noloc]`Bottlerocket`, see the https://bottlerocket.dev/en/[Bottlerocket Documentation]. +* For details about Bottlerocket, see the https://bottlerocket.dev/en/[Bottlerocket Documentation]. * For version information resources, see <>. -* To use [.noloc]`Bottlerocket` with managed node groups, see <>. -* To launch self-managed [.noloc]`Bottlerocket` nodes, see <>. -* To retrieve the latest IDs of the Amazon EKS optimized [.noloc]`Bottlerocket` AMIs, see <>. +* To use Bottlerocket with managed node groups, see <>. +* To launch self-managed Bottlerocket nodes, see <>. +* To retrieve the latest IDs of the Amazon EKS optimized Bottlerocket AMIs, see <>. * For details on compliance support, see <>. include::eks-ami-versions-bottlerocket.adoc[leveloffset=+1] @@ -60,3 +55,5 @@ include::eks-ami-versions-bottlerocket.adoc[leveloffset=+1] include::retrieve-ami-id-bottlerocket.adoc[leveloffset=+1] include::bottlerocket-compliance-support.adoc[leveloffset=+1] + +include::bottlerocket-fips-amis.adoc[leveloffset=+1] diff --git a/latest/ug/nodes/eks-optimized-ami.adoc b/latest/ug/nodes/eks-optimized-ami.adoc index 3b1549e15..e5e01e221 100644 --- a/latest/ug/nodes/eks-optimized-ami.adoc +++ b/latest/ug/nodes/eks-optimized-ami.adoc @@ -1,84 +1,65 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[eks-optimized-ami,eks-optimized-ami.title]] +[#eks-optimized-ami] = Create nodes with optimized Amazon Linux AMIs :info_titleabbrev: Amazon Linux -include::../attributes.txt[] - -include::al2023.adoc[leveloffset=+1] - -include::eks-linux-ami-versions.adoc[leveloffset=+1] - -include::retrieve-ami-id.adoc[leveloffset=+1] - -include::eks-ami-build-scripts.adoc[leveloffset=+1] - [abstract] -- -The Amazon EKS optimized Amazon Linux AMIs are built on top of Amazon Linux 2 (AL2) and Amazon Linux 2023 (AL2023). They are configured to serve as the base images for Amazon EKS nodes. +The Amazon EKS-optimized Amazon Linux AMIs are built on top of Amazon Linux 2 (AL2) and Amazon Linux 2023 (AL2023). They are configured to serve as the base images for Amazon EKS nodes. -- -The Amazon EKS optimized Amazon Linux AMIs are built on top of Amazon Linux 2 (AL2) and Amazon Linux 2023 (AL2023). They are configured to serve as the base images for Amazon EKS nodes. The AMIs are configured to work with Amazon EKS and they include the following components: +The Amazon EKS-optimized Amazon Linux AMIs are built on top of Amazon Linux 2 (AL2) and Amazon Linux 2023 (AL2023). They are configured to serve as the base images for Amazon EKS nodes. The AMIs are configured to work with Amazon EKS and they include the following components: * `kubelet` * {aws} IAM Authenticator -* [.noloc]`Docker` (Amazon EKS version `1.23` and earlier) * `containerd` [NOTE] ==== * You can track security or privacy events for Amazon Linux at the https://alas.aws.amazon.com/[Amazon Linux security center] by choosing the tab for your desired version. You can also subscribe to the applicable RSS feed. Security and privacy events include an overview of the issue, what packages are affected, and how to update your instances to correct the issue. -* Before deploying an accelerated or [.noloc]`Arm` AMI, review the information in <> and <>. -* For [.noloc]`Kubernetes` version `1.23`, you can use an optional bootstrap flag to test migration from [.noloc]`Docker` to `containerd`. For more information, see <>. +* Before deploying an accelerated or Arm AMI, review the information in <> and <>. * Amazon EC2 `P2` instances aren't supported on Amazon EKS because they require `NVIDIA` driver version 470 or earlier. * Any newly created managed node groups in clusters on version `1.30` or newer will automatically default to using AL2023 as the node operating system. Previously, new node groups would default to AL2. You can continue to use AL2 by choosing it as the AMI type when creating a new node group. -* Support for AL2 will end on June 30th, 2025. For more information, see link:amazon-linux-2/faqs/[Amazon Linux 2 FAQs,type="marketing"]. +* Amazon EKS will no longer publish EKS-optimized Amazon Linux 2 (AL2) AMIs after November 26th, 2025. Additionally, Kubernetes version `1.32` is the last version for which Amazon EKS will release AL2 AMIs. From version `1.33` onwards, Amazon EKS will continue to release AL2023 and Bottlerocket based AMIs. ==== -[[gpu-ami,gpu-ami.title]] -== Amazon EKS optimized accelerated Amazon Linux AMIs - -The Amazon EKS optimized accelerated Amazon Linux AMIs are built on top of the standard Amazon EKS optimized Amazon Linux AMIs. They are configured to serve as optional images for Amazon EKS nodes to support GPU, link:machine-learning/inferentia/[Inferentia,type="marketing"], and link:machine-learning/trainium/[Trainium,type="marketing"] based workloads. +[#gpu-ami] +== Amazon EKS-optimized accelerated Amazon Linux AMIs -In addition to the standard Amazon EKS optimized AMI configuration, the accelerated AMIs include the following: +The Amazon EKS-optimized accelerated Amazon Linux AMIs are built on top of the standard Amazon EKS-optimized Amazon Linux AMIs. They are configured to serve as optional images for Amazon EKS nodes to support GPU, link:machine-learning/inferentia/[Inferentia,type="marketing"], and link:machine-learning/trainium/[Trainium,type="marketing"] based workloads. -* [.noloc]`NVIDIA` drivers -* `nvidia-container-toolkit` -* {aws} [.noloc]`Neuron` driver +For more information, see <>. -For a list of the latest components included in the accelerated AMIs, see the `amazon-eks-ami` https://github.com/awslabs/amazon-eks-ami/releases[Releases] on [.noloc]`GitHub`. +[#arm-ami] +== Amazon EKS-optimized Arm Amazon Linux AMIs -[NOTE] -==== +Arm instances deliver significant cost savings for scale-out and Arm-based applications such as web servers, containerized microservices, caching fleets, and distributed data stores. When adding Arm nodes to your cluster, review the following considerations. -* Make sure to specify the applicable instance type in your node {aws} CloudFormation template. By using the Amazon EKS optimized accelerated AMIs, you agree to https://s3.amazonaws.com/EULA/NVidiaEULAforAWS.pdf[NVIDIA's Cloud End User License Agreement (EULA)]. -* The Amazon EKS optimized accelerated AMIs were previously referred to as the _Amazon EKS optimized AMIs with GPU support_. -* Previous versions of the Amazon EKS optimized accelerated AMIs installed the `nvidia-docker` repository. The repository is no longer included in Amazon EKS AMI version `v20200529` and later. +* If your cluster was deployed before August 17, 2020, you must do a one-time upgrade of critical cluster add-on manifests. This is so that Kubernetes can pull the correct image for each hardware architecture in use in your cluster. For more information about updating cluster add-ons, see <>. If you deployed your cluster on or after August 17, 2020, then your CoreDNS, `kube-proxy`, and Amazon VPC CNI plugin for Kubernetes add-ons are already multi-architecture capable. +* Applications deployed to Arm nodes must be compiled for Arm. +* If you have DaemonSets that are deployed in an existing cluster, or you want to deploy them to a new cluster that you also want to deploy Arm nodes in, then verify that your DaemonSet can run on all hardware architectures in your cluster. +* You can run Arm node groups and x86 node groups in the same cluster. If you do, consider deploying multi-architecture container images to a container repository such as Amazon Elastic Container Registry and then adding node selectors to your manifests so that Kubernetes knows what hardware architecture a Pod can be deployed to. For more information, see link:AmazonECR/latest/userguide/docker-push-multi-architecture-image.html[Pushing a multi-architecture image,type="documentation"] in the _Amazon ECR User Guide_ and the link:containers/introducing-multi-architecture-container-images-for-amazon-ecr[Introducing multi-architecture container images for Amazon ECR,type="blog"] blog post. -==== +[#linux-more-information] +== More information -For details on running workloads on Amazon EKS optimized accelerated Amazon Linux AMIs, see <>. +For more information about using Amazon EKS-optimized Amazon Linux AMIs, see the following sections: -[[arm-ami,arm-ami.title]] -== Amazon EKS optimized [.noloc]`Arm` Amazon Linux AMIs +* To use Amazon Linux with managed node groups, see <>. +* To launch self-managed Amazon Linux nodes, see <>. +* For version information, see <>. +* To retrieve the latest IDs of the Amazon EKS-optimized Amazon Linux AMIs, see <>. +* For open-source scripts that are used to build the Amazon EKS-optimized AMIs, see <>. -Arm instances deliver significant cost savings for scale-out and [.noloc]`Arm`-based applications such as web servers, containerized microservices, caching fleets, and distributed data stores. When adding [.noloc]`Arm` nodes to your cluster, review the following considerations. +include::al2023.adoc[leveloffset=+1] -* If your cluster was deployed before August 17, 2020, you must do a one-time upgrade of critical cluster add-on manifests. This is so that [.noloc]`Kubernetes` can pull the correct image for each hardware architecture in use in your cluster. For more information about updating cluster add-ons, see <>. If you deployed your cluster on or after August 17, 2020, then your [.noloc]`CoreDNS`, `kube-proxy`, and [.noloc]`Amazon VPC CNI plugin for Kubernetes` add-ons are already multi-architecture capable. -* Applications deployed to [.noloc]`Arm` nodes must be compiled for Arm. -* If you have [.noloc]`DaemonSets` that are deployed in an existing cluster, or you want to deploy them to a new cluster that you also want to deploy [.noloc]`Arm` nodes in, then verify that your [.noloc]`DaemonSet` can run on all hardware architectures in your cluster. -* You can run [.noloc]`Arm` node groups and x86 node groups in the same cluster. If you do, consider deploying multi-architecture container images to a container repository such as Amazon Elastic Container Registry and then adding node selectors to your manifests so that [.noloc]`Kubernetes` knows what hardware architecture a [.noloc]`Pod` can be deployed to. For more information, see link:AmazonECR/latest/userguide/docker-push-multi-architecture-image.html[Pushing a multi-architecture image,type="documentation"] in the _Amazon ECR User Guide_ and the link:containers/introducing-multi-architecture-container-images-for-amazon-ecr[Introducing multi-architecture container images for Amazon ECR,type="blog"] blog post. +include::eks-linux-ami-versions.adoc[leveloffset=+1] -[[linux-more-information,linux-more-information.title]] -== More information +include::retrieve-ami-id.adoc[leveloffset=+1] -For more information about using Amazon EKS optimized Amazon Linux AMIs, see the following sections: +include::eks-ami-build-scripts.adoc[leveloffset=+1] -* To use Amazon Linux with managed node groups, see <>. -* To launch self-managed Amazon Linux nodes, see <>. -* For version information, see <>. -* To retrieve the latest IDs of the Amazon EKS optimized Amazon Linux AMIs, see <>. -* For open-source scripts that are used to build the Amazon EKS optimized AMIs, see <>. diff --git a/latest/ug/nodes/eks-optimized-amis.adoc b/latest/ug/nodes/eks-optimized-amis.adoc index f7dccea02..ef4b4b601 100644 --- a/latest/ug/nodes/eks-optimized-amis.adoc +++ b/latest/ug/nodes/eks-optimized-amis.adoc @@ -1,15 +1,9 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[eks-optimized-amis,eks-optimized-amis.title]] +[#eks-optimized-amis] = Create nodes with pre-built optimized images -:info_doctype: section -:info_title: Create nodes with pre-built optimized images :info_titleabbrev: Pre-built optimized AMIs -:keywords: optimized, custom, AMI -:info_abstract: You can deploy nodes with pre-built Amazon EKS optimized Amazon Machine Images (AMIs) or your own custom \ - AMIs - -include::../attributes.txt[] [abstract] -- @@ -24,7 +18,7 @@ With Amazon EKS Auto Mode, EKS manages the EC2 instance including selecting and [[Topic List]] [.topic] -include::dockershim-deprecation.adoc[leveloffset=+1] +include::eks-ami-deprecation-faqs.adoc[leveloffset=+1] include::eks-optimized-ami.adoc[leveloffset=+1] diff --git a/latest/ug/nodes/eks-optimized-windows-ami.adoc b/latest/ug/nodes/eks-optimized-windows-ami.adoc index 1ed169a8f..0507648fa 100644 --- a/latest/ug/nodes/eks-optimized-windows-ami.adoc +++ b/latest/ug/nodes/eks-optimized-windows-ami.adoc @@ -1,17 +1,16 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[eks-optimized-windows-ami,eks-optimized-windows-ami.title]] -= Create nodes with optimized [.noloc]`Windows` AMIs +[#eks-optimized-windows-ami] += Create nodes with optimized Windows AMIs :info_titleabbrev: Windows -include::../attributes.txt[] - [abstract] -- -[.noloc]`Windows` Amazon EKS optimized AMIs are built on top of [.noloc]`Windows` Server 2019. +Windows Amazon EKS optimized AMIs are built on top of Windows Server 2019. -- -[.noloc]`Windows` Amazon EKS optimized AMIs are built on top of [.noloc]`Windows` Server 2019 and [.noloc]`Windows` Server 2022. They are configured to serve as the base image for Amazon EKS nodes. By default, the AMIs include the following components: +Windows Amazon EKS optimized AMIs are built on top of Windows Server 2019 and Windows Server 2022. They are configured to serve as the base image for Amazon EKS nodes. By default, the AMIs include the following components: * https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/[kubelet] * https://kubernetes.io/docs/reference/command-line-tools-reference/kube-proxy/[kube-proxy] @@ -23,71 +22,71 @@ include::../attributes.txt[] [NOTE] ==== -You can track security or privacy events for [.noloc]`Windows` Server with the https://portal.msrc.microsoft.com/en-us/security-guidance[Microsoft security update guide]. +You can track security or privacy events for Windows Server with the https://portal.msrc.microsoft.com/en-us/security-guidance[Microsoft security update guide]. ==== -Amazon EKS offers AMIs that are optimized for [.noloc]`Windows` containers in the following variants: +Amazon EKS offers AMIs that are optimized for Windows containers in the following variants: -* Amazon EKS-optimized [.noloc]`Windows` Server 2019 Core AMI -* Amazon EKS-optimized [.noloc]`Windows` Server 2019 Full AMI -* Amazon EKS-optimized [.noloc]`Windows` Server 2022 Core AMI -* Amazon EKS-optimized [.noloc]`Windows` Server 2022 Full AMI +* Amazon EKS-optimized Windows Server 2019 Core AMI +* Amazon EKS-optimized Windows Server 2019 Full AMI +* Amazon EKS-optimized Windows Server 2022 Core AMI +* Amazon EKS-optimized Windows Server 2022 Full AMI [IMPORTANT] ==== -* The Amazon EKS-optimized [.noloc]`Windows` Server [.noloc]`20H2` Core AMI is deprecated. No new versions of this AMI will be released. -* To ensure that you have the latest security updates by default, Amazon EKS maintains optimized [.noloc]`Windows` AMIs for the last 4 months. Each new AMI will be available for 4 months from the time of initial release. After this period, older AMIs are made private and are no longer accessible. We encourage using the latest AMIs to avoid security vulnerabilities and losing access to older AMIs which have reached the end of their supported lifetime. While we can't guarantee that we can provide access to AMIs that have been made private, you can request access by filing a ticket with {aws} Support. +* The Amazon EKS-optimized Windows Server 20H2 Core AMI is deprecated. No new versions of this AMI will be released. +* To ensure that you have the latest security updates by default, Amazon EKS maintains optimized Windows AMIs for the last 4 months. Each new AMI will be available for 4 months from the time of initial release. After this period, older AMIs are made private and are no longer accessible. We encourage using the latest AMIs to avoid security vulnerabilities and losing access to older AMIs which have reached the end of their supported lifetime. While we can't guarantee that we can provide access to AMIs that have been made private, you can request access by filing a ticket with {aws} Support. ==== -[[windows-ami-release-calendar,windows-ami-release-calendar.title]] +[#windows-ami-release-calendar] == Release calendar -The following table lists the release and end of support dates for [.noloc]`Windows` versions on Amazon EKS. If an end date is blank, it's because the version is still supported. +The following table lists the release and end of support dates for Windows versions on Amazon EKS. If an end date is blank, it's because the version is still supported. -[cols="1,1,1", options="header"] +[%header,cols="3"] |=== |Windows version |Amazon EKS release |Amazon EKS end of support -|[.noloc]`Windows` Server 2022 Core -|[.noloc]`10/17/2022` +|Windows Server 2022 Core +|10/17/2022 | -|[.noloc]`Windows` Server 2022 Full -|[.noloc]`10/17/2022` +|Windows Server 2022 Full +|10/17/2022 | -|[.noloc]`Windows` Server [.noloc]`20H2` Core -|[.noloc]`8/12/2021` -|[.noloc]`8/9/2022` +|Windows Server 20H2 Core +|8/12/2021 +|8/9/2022 -|[.noloc]`Windows` Server 2004 Core -|[.noloc]`8/19/2020` -|[.noloc]`12/14/2021` +|Windows Server 2004 Core +|8/19/2020 +|12/14/2021 -|[.noloc]`Windows` Server 2019 Core -|[.noloc]`10/7/2019` +|Windows Server 2019 Core +|10/7/2019 | -|[.noloc]`Windows` Server 2019 Full -|[.noloc]`10/7/2019` +|Windows Server 2019 Full +|10/7/2019 | -|[.noloc]`Windows` Server 1909 Core -|[.noloc]`10/7/2019` -|[.noloc]`12/8/2020` +|Windows Server 1909 Core +|10/7/2019 +|12/8/2020 |=== -[[bootstrap-script-configuration-parameters,bootstrap-script-configuration-parameters.title]] +[#bootstrap-script-configuration-parameters] == Bootstrap script configuration parameters -When you create a [.noloc]`Windows` node, there's a script on the node that allows for configuring different parameters. Depending on your setup, this script can be found on the node at a location similar to: `C:\Program Files\Amazon\EKS\Start-EKSBootstrap.ps1`. You can specify custom parameter values by specifying them as arguments to the bootstrap script. For example, you can update the user data in the launch template. For more information, see <>. +When you create a Windows node, there's a script on the node that allows for configuring different parameters. Depending on your setup, this script can be found on the node at a location similar to: `C:\Program Files\Amazon\EKS\Start-EKSBootstrap.ps1`. You can specify custom parameter values by specifying them as arguments to the bootstrap script. For example, you can update the user data in the launch template. For more information, see <>. The script includes the following command-line parameters: @@ -97,8 +96,8 @@ The script includes the following command-line parameters: * `-APIServerEndpoint` – Specifies the Amazon EKS cluster API server endpoint (optional). Only valid when used with `-Base64ClusterCA`. Bypasses calling `Get-EKSCluster`. * `-Base64ClusterCA` – Specifies the base64 encoded cluster CA content (optional). Only valid when used with `-APIServerEndpoint`. Bypasses calling `Get-EKSCluster`. * `-DNSClusterIP` – Overrides the IP address to use for DNS queries within the cluster (optional). Defaults to `10.100.0.10` or `172.20.0.10` based on the IP address of the primary interface. -* `-ServiceCIDR` – Overrides the [.noloc]`Kubernetes` service IP address range from which cluster services are addressed. Defaults to `172.20.0.0/16` or `10.100.0.0/16` based on the IP address of the primary interface. -* `-ExcludedSnatCIDRs` – A list of `IPv4` CIDRs to exclude from Source Network Address Translation (SNAT). This means that the pod private IP which is VPC addressable wouldn't be translated to the IP address of the instance ENI's primary `IPv4` address for outbound traffic. By default, the `IPv4` CIDR of the VPC for the Amazon EKS [.noloc]`Windows` node is added. Specifying CIDRs to this parameter also additionally excludes the specified CIDRs. For more information, see <>. +* `-ServiceCIDR` – Overrides the Kubernetes service IP address range from which cluster services are addressed. Defaults to `172.20.0.0/16` or `10.100.0.0/16` based on the IP address of the primary interface. +* `-ExcludedSnatCIDRs` – A list of `IPv4` CIDRs to exclude from Source Network Address Translation (SNAT). This means that the pod private IP which is VPC addressable wouldn't be translated to the IP address of the instance ENI's primary `IPv4` address for outbound traffic. By default, the `IPv4` CIDR of the VPC for the Amazon EKS Windows node is added. Specifying CIDRs to this parameter also additionally excludes the specified CIDRs. For more information, see <>. In addition to the command line parameters, you can also specify some environment variable parameters. When specifying a command line parameter, it takes precedence over the respective environment variable. The environment variable(s) should be defined as machine (or system) scoped as the bootstrap script will only read machine-scoped variables. @@ -108,16 +107,16 @@ The script takes into account the following environment variables: * `EXCLUDED_SNAT_CIDRS` – Should be a comma separated string. Refer to the `ExcludedSnatCIDRs` command line parameter for the definition. -[[ad-and-gmsa-support,ad-and-gmsa-support.title]] -=== [.noloc]`gMSA` authentication support +[#ad-and-gmsa-support] +=== gMSA authentication support -Amazon EKS Windows [.noloc]`Pods` allow different types of group Managed Service Account ([.noloc]`gMSA`) authentication. +Amazon EKS Windows Pods allow different types of group Managed Service Account (gMSA) authentication. -* Amazon EKS supports [.noloc]`Active Directory` domain identities for authentication. For more information on domain-joined [.noloc]`gMSA`, see link:containers/windows-authentication-on-amazon-eks-windows-pods[Windows Authentication on Amazon EKS Windowspods,type="blog"] on the {aws} blog. -* Amazon EKS offers a plugin that enables non-domain-joined [.noloc]`Windows` nodes to retrieve [.noloc]`gMSA` credentials with a portable user identity. For more information on domainless [.noloc]`gMSA`, see link:containers/domainless-windows-authentication-for-amazon-eks-windows-pods[Domainless Windows Authentication for Amazon EKS Windowspods,type="blog"] on the {aws} blog. +* Amazon EKS supports Active Directory domain identities for authentication. For more information on domain-joined gMSA, see link:containers/windows-authentication-on-amazon-eks-windows-pods[Windows Authentication on Amazon EKS Windowspods,type="blog"] on the {aws} blog. +* Amazon EKS offers a plugin that enables non-domain-joined Windows nodes to retrieve gMSA credentials with a portable user identity. For more information on domainless gMSA, see link:containers/domainless-windows-authentication-for-amazon-eks-windows-pods[Domainless Windows Authentication for Amazon EKS Windowspods,type="blog"] on the {aws} blog. -[[windows-cached-container-images,windows-cached-container-images.title]] +[#windows-cached-container-images] == Cached container images Amazon EKS Windows optimized AMIs have certain container images cached for the `containerd` runtime. Container images are cached when building custom AMIs using Amazon-managed build components. For more information, see <>. @@ -128,17 +127,17 @@ The following cached container images are for the `containerd` runtime: * `mcr.microsoft.com/windows/nanoserver` * `mcr.microsoft.com/windows/servercore` -[[windows-more-information,windows-more-information.title]] +[#windows-more-information] == More information -For more information about using Amazon EKS optimized [.noloc]`Windows` AMIs, see the following sections: +For more information about using Amazon EKS optimized Windows AMIs, see the following sections: * For details on running workloads on Amazon EKS optimized accelerated Windows AMIs, see <>. -* To use [.noloc]`Windows` with managed node groups, see <>. -* To launch self-managed [.noloc]`Windows` nodes, see <>. +* To use Windows with managed node groups, see <>. +* To launch self-managed Windows nodes, see <>. * For version information, see <>. -* To retrieve the latest IDs of the Amazon EKS optimized [.noloc]`Windows` AMIs, see <>. -* To use Amazon EC2 Image Builder to create custom Amazon EKS optimized [.noloc]`Windows` AMIs, see <>. +* To retrieve the latest IDs of the Amazon EKS optimized Windows AMIs, see <>. +* To use Amazon EC2 Image Builder to create custom Amazon EKS optimized Windows AMIs, see <>. * For best practices, see https://aws.github.io/aws-eks-best-practices/windows/docs/ami/[Amazon EKS optimized Windows AMI management] in the _EKS Best Practices Guide_. include::self-managed-windows-server-2022.adoc[leveloffset=+1] @@ -147,4 +146,4 @@ include::eks-ami-versions-windows.adoc[leveloffset=+1] include::retrieve-windows-ami-id.adoc[leveloffset=+1] -include::eks-custom-ami-windows.adoc[leveloffset=+1] +include::eks-custom-ami-windows.adoc[leveloffset=+1] \ No newline at end of file diff --git a/latest/ug/nodes/eks-partner-amis.adoc b/latest/ug/nodes/eks-partner-amis.adoc index b52ddc992..1df5547eb 100644 --- a/latest/ug/nodes/eks-partner-amis.adoc +++ b/latest/ug/nodes/eks-partner-amis.adoc @@ -1,8 +1,8 @@ -//!!NODE_ROOT
include::../attributes.txt[] + [.topic] -[[eks-partner-amis,eks-partner-amis.title]] -= Create nodes with optimized [.noloc]`Ubuntu Linux` AMIs +[#eks-partner-amis] += Create nodes with optimized Ubuntu Linux AMIs :info_titleabbrev: Ubuntu Linux [abstract] @@ -12,4 +12,4 @@ Canonical has partnered with Amazon EKS to create node AMIs that you can use in Canonical has partnered with Amazon EKS to create node AMIs that you can use in your clusters. -https://www.canonical.com/[Canonical] delivers a built-for-purpose [.noloc]`Kubernetes` Node OS image. This minimized [.noloc]`Ubuntu` image is optimized for Amazon EKS and includes the custom {aws} kernel that is jointly developed with {aws}. For more information, see https://cloud-images.ubuntu.com/aws-eks/[Ubuntu on Amazon Elastic Kubernetes Service (EKS)] and <> . For information about support, see the link:premiumsupport/faqs/#Third-party_software[Third-party software,type="marketing"] section of the _{aws} Premium Support FAQs_. +https://www.canonical.com/[Canonical] delivers a built-for-purpose Kubernetes Node OS image. This minimized Ubuntu image is optimized for Amazon EKS and includes the custom {aws} kernel that is jointly developed with {aws}. For more information, see https://cloud-images.ubuntu.com/aws-eks/[Ubuntu on Amazon Elastic Kubernetes Service (EKS)] and <> . For information about support, see the link:premiumsupport/faqs/#Third-party_software[Third-party software,type="marketing"] section of the _{aws} Premium Support FAQs_. \ No newline at end of file diff --git a/latest/ug/nodes/fargate-getting-started.adoc b/latest/ug/nodes/fargate-getting-started.adoc index 6f1f3c81b..986236734 100644 --- a/latest/ug/nodes/fargate-getting-started.adoc +++ b/latest/ug/nodes/fargate-getting-started.adoc @@ -1,67 +1,60 @@ -//!!NODE_ROOT
include::../attributes.txt[] + [.topic] -[[fargate-getting-started,fargate-getting-started.title]] +[#fargate-getting-started] = Get started with {aws} Fargate for your cluster :info_titleabbrev: Get started [abstract] -- -This topic describes how to get started running [.noloc]`Pods` on {aws} Fargate with your Amazon EKS cluster. +This topic describes how to get started running Pods on {aws} Fargate with your Amazon EKS cluster. -- -[IMPORTANT] -==== - -{aws} Fargate with Amazon EKS isn't available in {aws} GovCloud (US-East) and {aws} GovCloud (US-West). - -==== +This topic describes how to get started running Pods on {aws} Fargate with your Amazon EKS cluster. -This topic describes how to get started running [.noloc]`Pods` on {aws} Fargate with your Amazon EKS cluster. - -If you restrict access to the public endpoint of your cluster using CIDR blocks, we recommend that you also enable private endpoint access. This way, Fargate [.noloc]`Pods` can communicate with the cluster. Without the private endpoint enabled, the CIDR blocks that you specify for public access must include the outbound sources from your VPC. For more information, see <>. +If you restrict access to the public endpoint of your cluster using CIDR blocks, we recommend that you also enable private endpoint access. This way, Fargate Pods can communicate with the cluster. Without the private endpoint enabled, the CIDR blocks that you specify for public access must include the outbound sources from your VPC. For more information, see <>. .Prerequisite An existing cluster. If you don't already have an Amazon EKS cluster, see <>. -[[fargate-gs-check-compatibility,fargate-gs-check-compatibility.title]] -== Step 1: Ensure that existing nodes can communicate with Fargate [.noloc]`Pods` +[#fargate-gs-check-compatibility] +== Step 1: Ensure that existing nodes can communicate with Fargate Pods If you're working with a new cluster with no nodes, or a cluster with only managed node groups (see <>), you can skip to <>. -Assume that you're working with an existing cluster that already has nodes that are associated with it. Make sure that [.noloc]`Pods` on these nodes can communicate freely with the [.noloc]`Pods` that are running on Fargate. [.noloc]`Pods` that are running on Fargate are automatically configured to use the cluster security group for the cluster that they're associated with. Ensure that any existing nodes in your cluster can send and receive traffic to and from the cluster security group. Managed node groups are automatically configured to use the cluster security group as well, so you don't need to modify or check them for this compatibility (see <>). +Assume that you're working with an existing cluster that already has nodes that are associated with it. Make sure that Pods on these nodes can communicate freely with the Pods that are running on Fargate. Pods that are running on Fargate are automatically configured to use the cluster security group for the cluster that they're associated with. Ensure that any existing nodes in your cluster can send and receive traffic to and from the cluster security group. Managed node groups are automatically configured to use the cluster security group as well, so you don't need to modify or check them for this compatibility (see <>). -For existing node groups that were created with `eksctl` or the Amazon EKS managed {aws} CloudFormation templates, you can add the cluster security group to the nodes manually. Or, alternatively, you can modify the Auto Scaling group launch template for the node group to attach the cluster security group to the instances. For more information, see link:vpc/latest/userguide/VPC_SecurityGroups.html#SG_Changing_Group_Membership[Changing an instance's security groups,type="documentation"] in the _Amazon VPC User Guide_. +For existing node groups that were created with `eksctl` or the Amazon EKS managed {aws} CloudFormation templates, you can add the cluster security group to the nodes manually. Or, alternatively, you can modify the Auto Scaling group launch template for the node group to attach the cluster security group to the instances. For more information, see link:vpc/latest/userguide/VPC_SecurityGroups.html#SG_Changing_Group_Membership[Changing an instance's security groups,type="documentation"] in the _Amazon VPC User Guide_. -You can check for a security group for your cluster in the {aws-management-console} under the *Networking* section for the cluster. Or, you can do this using the following {aws} CLI command. When using this command, replace [.replaceable]`my-cluster` with the name of your cluster. +You can check for a security group for your cluster in the {aws-management-console} under the *Networking* section for the cluster. Or, you can do this using the following {aws} CLI command. When using this command, replace `` with the name of your cluster. [source,bash,subs="verbatim,attributes"] ---- -aws eks describe-cluster --name my-cluster --query cluster.resourcesVpcConfig.clusterSecurityGroupId +aws eks describe-cluster --name --query cluster.resourcesVpcConfig.clusterSecurityGroupId ---- -[[fargate-sg-pod-execution-role,fargate-sg-pod-execution-role.title]] -== Step 2: Create a Fargate [.noloc]`Pod` execution role +[#fargate-sg-pod-execution-role] +== Step 2: Create a Fargate Pod execution role -When your cluster creates [.noloc]`Pods` on {aws} Fargate, the components that run on the Fargate infrastructure must make calls to {aws} APIs on your behalf. The Amazon EKS [.noloc]`Pod` execution role provides the IAM permissions to do this. To create an {aws} Fargate [.noloc]`Pod` execution role, see <>. +When your cluster creates Pods on {aws} Fargate, the components that run on the Fargate infrastructure must make calls to {aws} APIs on your behalf. The Amazon EKS Pod execution role provides the IAM permissions to do this. To create an {aws} Fargate Pod execution role, see <>. [NOTE] ==== -If you created your cluster with `eksctl` using the `--fargate` option, your cluster already has a [.noloc]`Pod` execution role that you can find in the IAM console with the pattern `eksctl-my-cluster-FargatePodExecutionRole-ABCDEFGHIJKL`. Similarly, if you use `eksctl` to create your Fargate profiles, `eksctl` creates your [.noloc]`Pod` execution role if one isn't already created. +If you created your cluster with `eksctl` using the `--fargate` option, your cluster already has a Pod execution role that you can find in the IAM console with the pattern `eksctl-my-cluster-FargatePodExecutionRole-ABCDEFGHIJKL`. Similarly, if you use `eksctl` to create your Fargate profiles, `eksctl` creates your Pod execution role if one isn't already created. ==== -[[fargate-gs-create-profile,fargate-gs-create-profile.title]] +[#fargate-gs-create-profile] == Step 3: Create a Fargate profile for your cluster -Before you can schedule [.noloc]`Pods` that are running on Fargate in your cluster, you must define a Fargate profile that specifies which [.noloc]`Pods` use Fargate when they're launched. For more information, see <>. +Before you can schedule Pods that are running on Fargate in your cluster, you must define a Fargate profile that specifies which Pods use Fargate when they're launched. For more information, see <>. [NOTE] ==== -If you created your cluster with `eksctl` using the `--fargate` option, then a Fargate profile is already created for your cluster with selectors for all [.noloc]`Pods` in the `kube-system` and `default` namespaces. Use the following procedure to create Fargate profiles for any other namespaces you would like to use with Fargate. +If you created your cluster with `eksctl` using the `--fargate` option, then a Fargate profile is already created for your cluster with selectors for all Pods in the `kube-system` and `default` namespaces. Use the following procedure to create Fargate profiles for any other namespaces you would like to use with Fargate. ==== @@ -83,18 +76,18 @@ For instructions on how to install or upgrade `eksctl`, see https://eksctl.io/in *To create a Fargate profile with `eksctl`* -Create your Fargate profile with the following `eksctl` command, replacing every [.replaceable]`example value` with your own values. You're required to specify a namespace. However, the `--labels` option isn't required. +Create your Fargate profile with the following `eksctl` command, replacing every `` with your own values. You're required to specify a namespace. However, the `--labels` option isn't required. [source,bash,subs="verbatim,attributes"] ---- eksctl create fargateprofile \ - --cluster my-cluster \ - --name my-fargate-profile \ - --namespace my-kubernetes-namespace \ - --labels key=value + --cluster \ + --name \ + --namespace \ + --labels ---- -You can use certain wildcards for [.replaceable]`my-kubernetes-namespace` and [.replaceable]`key=value` labels. For more information, see <>. +You can use certain wildcards for `` and `` labels. For more information, see <>. === {aws-management-console} [[console_fargate_profile_create]] @@ -103,34 +96,34 @@ You can use certain wildcards for [.replaceable]`my-kubernetes-namespace` and [. . Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. . Choose the cluster to create a Fargate profile for. . Choose the *Compute* tab. -. Under *Fargate profiles*, choose *Add Fargate profile*. +. Under *Fargate profiles*, choose *Add Fargate profile*. . On the *Configure Fargate profile* page, do the following: + .. For *Name*, enter a name for your Fargate profile. The name must be unique. -.. For *Pod execution role*, choose the [.noloc]`Pod` execution role to use with your Fargate profile. Only the IAM roles with the `eks-fargate-pods.amazonaws.com` service principal are shown. If you don't see any roles listed, you must create one. For more information, see <>. +.. For *Pod execution role*, choose the Pod execution role to use with your Fargate profile. Only the IAM roles with the `eks-fargate-pods.amazonaws.com` service principal are shown. If you don't see any roles listed, you must create one. For more information, see <>. .. Modify the selected *Subnets* as needed. + -NOTE: Only private subnets are supported for [.noloc]`Pods` that are running on Fargate. -.. For *Tags*, you can optionally tag your Fargate profile. These tags don't propagate to other resources that are associated with the profile such as [.noloc]`Pods`. +NOTE: Only private subnets are supported for Pods that are running on Fargate. +.. For *Tags*, you can optionally tag your Fargate profile. These tags don't propagate to other resources that are associated with the profile such as Pods. .. Choose *Next*. -. On the *Configure [.noloc]`Pod` selection* page, do the following: +. On the *Configure Pod selection* page, do the following: + -.. For *Namespace*, enter a namespace to match for [.noloc]`Pods`. +.. For *Namespace*, enter a namespace to match for Pods. + *** You can use specific namespaces to match, such as `kube-system` or `default`. *** You can use certain wildcards (for example, `prod-*`) to match multiple namespaces (for example, `prod-deployment` and `prod-test`). For more information, see <>. -.. (Optional) Add [.noloc]`Kubernetes` labels to the selector. Specifically add them to the one that the [.noloc]`Pods` in the specified namespace need to match. +.. (Optional) Add Kubernetes labels to the selector. Specifically add them to the one that the Pods in the specified namespace need to match. + -*** You can add the label `infrastructure: fargate` to the selector so that only [.noloc]`Pods` in the specified namespace that also have the `infrastructure: fargate` [.noloc]`Kubernetes` label match the selector. +*** You can add the label `infrastructure: fargate` to the selector so that only Pods in the specified namespace that also have the `infrastructure: fargate` Kubernetes label match the selector. *** You can use certain wildcards (for example, `key?: value?`) to match multiple namespaces (for example, `keya: valuea` and `keyb: valueb`). For more information, see <>. .. Choose *Next*. -. On the *Review and create* page, review the information for your Fargate profile and choose *Create*. +. On the *Review and create* page, review the information for your Fargate profile and choose *Create*. -[[fargate-gs-coredns,fargate-gs-coredns.title]] -== Step 4: Update [.noloc]`CoreDNS` +[#fargate-gs-coredns] +== Step 4: Update CoreDNS -By default, [.noloc]`CoreDNS` is configured to run on Amazon EC2 infrastructure on Amazon EKS clusters. If you want to _only_ run your [.noloc]`Pods` on Fargate in your cluster, complete the following steps. +By default, CoreDNS is configured to run on Amazon EC2 infrastructure on Amazon EKS clusters. If you want to _only_ run your Pods on Fargate in your cluster, complete the following steps. [NOTE] ==== @@ -138,42 +131,38 @@ By default, [.noloc]`CoreDNS` is configured to run on Amazon EC2 infrastructure If you created your cluster with `eksctl` using the `--fargate` option, then you can skip to <>. ==== -. Create a Fargate profile for [.noloc]`CoreDNS` with the following command. Replace [.replaceable]`my-cluster` with your cluster name, [.replaceable]`111122223333` with your account ID, [.replaceable]`AmazonEKSFargatePodExecutionRole` with the name of your [.noloc]`Pod` execution role, and [.replaceable]`0000000000000001`, [.replaceable]`0000000000000002`, and [.replaceable]`0000000000000003` with the IDs of your private subnets. If you don't have a [.noloc]`Pod` execution role, you must create one first (see <>). +. Create a Fargate profile for CoreDNS with the following command. Replace `` with your cluster name, `<111122223333>` with your account ID, `` with the name of your Pod execution role, and `<000000000000000a>`, `<000000000000000b>`, and `<000000000000000c>` with the IDs of your private subnets. If you don't have a Pod execution role, you must create one first (see <>). + -IMPORTANT: The role ARN can't include a link:IAM/latest/UserGuide/reference_identifiers.html#identifiers-friendly-names[path,type="documentation"] other than `/`. For example, if the name of your role is `development/apps/my-role`, you need to change it to `my-role` when specifying the ARN for the role. The format of the role ARN must be `{arn-aws}iam::111122223333:role/role-name`. +IMPORTANT: The role ARN can't include a link:IAM/latest/UserGuide/reference_identifiers.html#identifiers-friendly-names[path,type="documentation"] other than `/`. For example, if the name of your role is `development/apps/AmazonEKSFargatePodExecutionRole`, you need to change it to `AmazonEKSFargatePodExecutionRole` when specifying the ARN for the role. The format of the role ARN must be `{arn-aws}iam::<111122223333>:role/`. + [source,bash,subs="verbatim,attributes"] ---- aws eks create-fargate-profile \ --fargate-profile-name coredns \ - --cluster-name my-cluster \ - --pod-execution-role-arn {arn-aws}iam::111122223333:role/AmazonEKSFargatePodExecutionRole \ + --cluster-name \ + --pod-execution-role-arn {arn-aws}iam::<111122223333>:role/ \ --selectors namespace=kube-system,labels={k8s-app=kube-dns} \ - --subnets subnet-0000000000000001 subnet-0000000000000002 subnet-0000000000000003 + --subnets subnet-<000000000000000a> subnet-<000000000000000b> subnet-<000000000000000c> ---- -. Run the following command to remove the `eks.amazonaws.com/compute-type : ec2` annotation from the [.noloc]`CoreDNS` [.noloc]`Pods`. +. Trigger a rollout of the `coredns` deployment. + [source,bash,subs="verbatim,attributes"] ---- -kubectl patch deployment coredns \ - -n kube-system \ - --type json \ - -p='[{"op": "remove", "path": "/spec/template/metadata/annotations/eks.amazonaws.com~1compute-type"}]' +kubectl rollout restart -n kube-system deployment coredns ---- - -[[fargate-gs-next-steps,fargate-gs-next-steps.title]] +[#fargate-gs-next-steps] == Next steps * You can start migrating your existing applications to run on Fargate with the following workflow. + -.. <> that matches your application's [.noloc]`Kubernetes` namespace and [.noloc]`Kubernetes` labels. -.. Delete and re-create any existing [.noloc]`Pods` so that they're scheduled on Fargate. For example, the following command triggers a rollout of the `coredns` deployment. You can modify the namespace and deployment type to update your specific [.noloc]`Pods`. +.. <> that matches your application's Kubernetes namespace and Kubernetes labels. +.. Delete and re-create any existing Pods so that they're scheduled on Fargate. Modify the `` and `` to update your specific Pods. + [source,bash,subs="verbatim,attributes"] ---- -kubectl rollout restart -n kube-system deployment coredns +kubectl rollout restart -n deployment ---- -* Deploy the <> to allow Ingress objects for your [.noloc]`Pods` running on Fargate. -* You can use the <> to set the initial correct size of CPU and memory for your Fargate [.noloc]`Pods`, and then use the <> to scale those [.noloc]`Pods`. If you want the Vertical Pod Autoscaler to automatically re-deploy [.noloc]`Pods` to Fargate with higher CPU and memory combinations, set the Vertical Pod Autoscaler's mode to either `Auto` or `Recreate`. This is to ensure correct functionality. For more information, see the https://github.com/kubernetes/autoscaler/tree/master/vertical-pod-autoscaler#quick-start[Vertical Pod Autoscaler] documentation on [.noloc]`GitHub`. -* You can set up the link:otel[{aws} Distro for OpenTelemetry,type="marketing"] (ADOT) collector for application monitoring by following link:AmazonCloudWatch/latest/monitoring/Container-Insights-EKS-otel.html[these instructions,type="documentation"]. +* Deploy the <> to allow Ingress objects for your Pods running on Fargate. +* You can use the <> to set the initial correct size of CPU and memory for your Fargate Pods, and then use the <> to scale those Pods. If you want the Vertical Pod Autoscaler to automatically re-deploy Pods to Fargate with higher CPU and memory combinations, set the Vertical Pod Autoscaler's mode to either `Auto` or `Recreate`. This is to ensure correct functionality. For more information, see the https://github.com/kubernetes/autoscaler/tree/master/vertical-pod-autoscaler#quick-start[Vertical Pod Autoscaler] documentation on GitHub. +* You can set up the link:otel[{aws} Distro for OpenTelemetry,type="marketing"] (ADOT) collector for application monitoring by following link:AmazonCloudWatch/latest/monitoring/Container-Insights-EKS-otel.html[these instructions,type="documentation"]. \ No newline at end of file diff --git a/latest/ug/nodes/fargate-logging.adoc b/latest/ug/nodes/fargate-logging.adoc index aedd924b0..72820dd34 100644 --- a/latest/ug/nodes/fargate-logging.adoc +++ b/latest/ug/nodes/fargate-logging.adoc @@ -1,43 +1,41 @@ -//!!NODE_ROOT
include::../attributes.txt[] + [.topic] -[[fargate-logging,fargate-logging.title]] +[#fargate-logging] = Start {aws} Fargate logging for your cluster :info_titleabbrev: Logging [abstract] -- -Amazon EKS on Fargate offers a built-in log router based on [.noloc]`Fluent Bit`. +Amazon EKS on Fargate offers a built-in log router based on Fluent Bit. -- -[IMPORTANT] -==== - -{aws} Fargate with Amazon EKS isn't available in {aws} GovCloud (US-East) and {aws} GovCloud (US-West). - -==== - -Amazon EKS on Fargate offers a built-in log router based on [.noloc]`Fluent Bit`. This means that you don't explicitly run a [.noloc]`Fluent Bit` container as a sidecar, but Amazon runs it for you. All that you have to do is configure the log router. The configuration happens through a dedicated `ConfigMap` that must meet the following criteria: +Amazon EKS on Fargate offers a built-in log router based on Fluent Bit. This means that you don't explicitly run a Fluent Bit container as a sidecar, but Amazon runs it for you. All that you have to do is configure the log router. The configuration happens through a dedicated `ConfigMap` that must meet the following criteria: * Named `aws-logging` * Created in a dedicated namespace called `aws-observability` * Can't exceed 5300 characters. -Once you've created the `ConfigMap`, Amazon EKS on Fargate automatically detects it and configures the log router with it. Fargate uses a version of {aws} for [.noloc]`Fluent Bit`, an upstream compliant distribution of [.noloc]`Fluent Bit` managed by {aws}. For more information, see https://github.com/aws/aws-for-fluent-bit[{aws} for Fluent Bit] on GitHub. +Once you've created the `ConfigMap`, Amazon EKS on Fargate automatically detects it and configures the log router with it. Fargate uses a version of {aws} for Fluent Bit, an upstream compliant distribution of Fluent Bit managed by {aws}. For more information, see https://github.com/aws/aws-for-fluent-bit[{aws} for Fluent Bit] on GitHub. The log router allows you to use the breadth of services at {aws} for log analytics and storage. You can stream logs from Fargate directly to Amazon CloudWatch, Amazon OpenSearch Service. You can also stream logs to destinations such as link:s3/[Amazon S3,type="marketing"], link:kinesis/data-streams/[Amazon Kinesis Data Streams,type="marketing"], and partner tools through link:kinesis/data-firehose/[Amazon Data Firehose,type="marketing"]. -* An existing Fargate profile that specifies an existing [.noloc]`Kubernetes` namespace that you deploy Fargate [.noloc]`Pods` to. For more information, see <>. -* An existing Fargate [.noloc]`Pod` execution role. For more information, see <>. +* An existing Fargate profile that specifies an existing Kubernetes namespace that you deploy Fargate Pods to. For more information, see <>. +* An existing Fargate Pod execution role. For more information, see <>. -[[fargate-logging-log-router-configuration,fargate-logging-log-router-configuration.title]] +[#fargate-logging-log-router-configuration] == Log router configuration +[IMPORTANT] +==== +For logs to be successfully published, there must be network access from the VPC that your cluster is in to the log destination. This mainly concerns users customizing egress rules for their VPC. For an example using CloudWatch, see link:AmazonCloudWatch/latest/logs/cloudwatch-logs-and-interface-VPC.html[Using CloudWatch Logs with interface VPC endpoints,type="documentation"] in the _Amazon CloudWatch Logs User Guide_. +==== + In the following steps, replace every [.replaceable]`example value` with your own values. -. Create a dedicated [.noloc]`Kubernetes` namespace named `aws-observability`. +. Create a dedicated Kubernetes namespace named `aws-observability`. + -.. Save the following contents to a file named `[.replaceable]``aws-observability-namespace``.yaml` on your computer. The value for `name` must be `aws-observability` and the `aws-observability: enabled` label is required. +.. Save the following contents to a file named `aws-observability-namespace.yaml` on your computer. The value for `name` must be `aws-observability` and the `aws-observability: enabled` label is required. + [source,yaml,subs="verbatim,attributes"] ---- @@ -50,11 +48,11 @@ metadata: ---- .. Create the namespace. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- -kubectl apply -f [.replaceable]`aws-observability-namespace`.yaml +kubectl apply -f aws-observability-namespace.yaml ---- -. Create a `ConfigMap` with a `Fluent Conf` data value to ship container logs to a destination. Fluent Conf is [.noloc]`Fluent Bit`, which is a fast and lightweight log processor configuration language that's used to route container logs to a log destination of your choice. For more information, see https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/classic-mode/configuration-file[Configuration File] in the [.noloc]`Fluent Bit` documentation. +. Create a `ConfigMap` with a `Fluent Conf` data value to ship container logs to a destination. Fluent Conf is Fluent Bit, which is a fast and lightweight log processor configuration language that's used to route container logs to a log destination of your choice. For more information, see https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/classic-mode/configuration-file[Configuration File] in the Fluent Bit documentation. + [IMPORTANT] ==== @@ -94,12 +92,12 @@ When creating the `ConfigMap`, take into account the following rules that Fargat * At least one supported `Output` plugin has to be provided in the `ConfigMap` to enable logging. `Filter` and `Parser` aren't required to enable logging. + -You can also run [.noloc]`Fluent Bit` on Amazon EC2 using the desired configuration to troubleshoot any issues that arise from validation. Create your `ConfigMap` using one of the following examples. +You can also run Fluent Bit on Amazon EC2 using the desired configuration to troubleshoot any issues that arise from validation. Create your `ConfigMap` using one of the following examples. + [IMPORTANT] ==== -Amazon EKS Fargate logging doesn't support dynamic configuration of a `ConfigMap`. Any changes to a `ConfigMap` are applied to new [.noloc]`Pods` only. Changes aren't applied to existing [.noloc]`Pods`. +Amazon EKS Fargate logging doesn't support dynamic configuration of a `ConfigMap`. Any changes to a `ConfigMap` are applied to new Pods only. Changes aren't applied to existing Pods. ==== + @@ -115,8 +113,6 @@ You can also use Amazon Kinesis Data Streams for your log destination. If you us ==== [role="tablist"] CloudWatch:: -*To create a `ConfigMap` for CloudWatch* - + You have two output options when using CloudWatch: + @@ -126,7 +122,7 @@ You have two output options when using CloudWatch: + The following example shows you how to use the `cloudwatch_logs` plugin to send logs to CloudWatch. -.. Save the following contents to a file named `[.replaceable]``aws-logging-cloudwatch-configmap``.yaml`. Replace [.replaceable]`region-code` with the {aws} Region that your cluster is in. The parameters under `[OUTPUT]` are required. +.. Save the following contents to a file named `aws-logging-cloudwatch-configmap.yaml`. Replace [.replaceable]`region-code` with the {aws} Region that your cluster is in. The parameters under `[OUTPUT]` are required. // Not using subs="quotes" here with [.replaceable]`region-code` because the ^ characters get dropped, even when using AsciiDoc's built-in {caret} character replacement attribute. + [source,yaml,subs="verbatim,attributes"] @@ -170,23 +166,16 @@ data: ---- .. Apply the manifest to your cluster. + -[source,bash,subs="verbatim,attributes,quotes"] ----- -kubectl apply -f [.replaceable]`aws-logging-cloudwatch-configmap`.yaml ----- -.. Download the CloudWatch IAM policy to your computer. You can also https://raw.githubusercontent.com/aws-samples/amazon-eks-fluent-logging-examples/mainline/examples/fargate/cloudwatchlogs/permissions.json[view the policy] on GitHub. -+ [source,bash,subs="verbatim,attributes"] ---- -curl -O https://raw.githubusercontent.com/aws-samples/amazon-eks-fluent-logging-examples/mainline/examples/fargate/cloudwatchlogs/permissions.json +kubectl apply -f aws-logging-cloudwatch-configmap.yaml ---- Amazon OpenSearch Service:: -*To create a `ConfigMap` for Amazon OpenSearch Service* + -If you want to send logs to Amazon OpenSearch Service, you can use https://docs.fluentbit.io/manual/v/1.5/pipeline/outputs/elasticsearch[es] output, which is a plugin written in [.noloc]`C`. The following example shows you how to use the plugin to send logs to OpenSearch. +If you want to send logs to Amazon OpenSearch Service, you can use https://docs.fluentbit.io/manual/v/1.5/pipeline/outputs/elasticsearch[es] output, which is a plugin written in C. The following example shows you how to use the plugin to send logs to OpenSearch. + -.. Save the following contents to a file named `[.replaceable]``aws-logging-opensearch-configmap``.yaml`. Replace every [.replaceable]`example value` with your own values. +.. Save the following contents to a file named `aws-logging-opensearch-configmap.yaml`. Replace every [.replaceable]`example value` with your own values. + [source,yaml,subs="verbatim,attributes,quotes"] ---- @@ -210,21 +199,12 @@ data: ---- .. Apply the manifest to your cluster. + -[source,bash,subs="verbatim,attributes,quotes"] ----- -kubectl apply -f [.replaceable]`aws-logging-opensearch-configmap`.yaml ----- -.. Download the OpenSearch IAM policy to your computer. You can also https://raw.githubusercontent.com/aws-samples/amazon-eks-fluent-logging-examples/mainline/examples/fargate/amazon-elasticsearch/permissions.json[view the policy] on GitHub. -+ [source,bash,subs="verbatim,attributes"] ---- -curl -O https://raw.githubusercontent.com/aws-samples/amazon-eks-fluent-logging-examples/mainline/examples/fargate/amazon-elasticsearch/permissions.json +kubectl apply -f aws-logging-opensearch-configmap.yaml ---- -+ -Make sure that OpenSearch Dashboards' access control is configured properly. The `all_access role` in OpenSearch Dashboards needs to have the Fargate [.noloc]`Pod` execution role and the IAM role mapped. The same mapping must be done for the `security_manager` role. You can add the previous mappings by selecting `Menu`, then `Security`, then `Roles`, and then select the respective roles. For more information, see link:tr/premiumsupport/knowledge-center/es-troubleshoot-cloudwatch-logs/[How do I troubleshoot CloudWatch Logs so that it streams to my Amazon ES domain?,type="marketing"]. Firehose:: -*To create a `ConfigMap` for Firehose* + You have two output options when sending logs to Firehose: + @@ -233,7 +213,7 @@ You have two output options when sending logs to Firehose: + The following example shows you how to use the `kinesis_firehose` plugin to send logs to Firehose. + -.. Save the following contents to a file named `[.replaceable]``aws-logging-firehose-configmap``.yaml`. Replace [.replaceable]`region-code` with the {aws} Region that your cluster is in. +.. Save the following contents to a file named `aws-logging-firehose-configmap.yaml`. Replace [.replaceable]`region-code` with the {aws} Region that your cluster is in. + [source,yaml,subs="verbatim,attributes,quotes"] ---- @@ -252,24 +232,53 @@ data: ---- .. Apply the manifest to your cluster. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- -kubectl apply -f [.replaceable]`aws-logging-firehose-configmap`.yaml +kubectl apply -f aws-logging-firehose-configmap.yaml ---- -.. Download the Firehose IAM policy to your computer. You can also https://raw.githubusercontent.com/aws-samples/amazon-eks-fluent-logging-examples/mainline/examples/fargate/kinesis-firehose/permissions.json[view the policy] on GitHub. +==== + +. Set up permissions for the Fargate Pod execution role to send logs to your destination. + +.. Download the IAM policy for your destination to your computer. ++ +==== +[role="tablist"] +CloudWatch:: +Download the CloudWatch IAM policy to your computer. You can also https://raw.githubusercontent.com/aws-samples/amazon-eks-fluent-logging-examples/mainline/examples/fargate/cloudwatchlogs/permissions.json[view the policy] on GitHub. ++ +[source,bash,subs="verbatim,attributes"] +---- +curl -O https://raw.githubusercontent.com/aws-samples/amazon-eks-fluent-logging-examples/mainline/examples/fargate/cloudwatchlogs/permissions.json +---- + +Amazon OpenSearch Service:: +Download the OpenSearch IAM policy to your computer. You can also https://raw.githubusercontent.com/aws-samples/amazon-eks-fluent-logging-examples/mainline/examples/fargate/amazon-elasticsearch/permissions.json[view the policy] on GitHub. ++ +[source,bash,subs="verbatim,attributes"] +---- +curl -O https://raw.githubusercontent.com/aws-samples/amazon-eks-fluent-logging-examples/mainline/examples/fargate/amazon-elasticsearch/permissions.json +---- ++ +Make sure that OpenSearch Dashboards' access control is configured properly. The `all_access role` in OpenSearch Dashboards needs to have the Fargate Pod execution role and the IAM role mapped. The same mapping must be done for the `security_manager` role. You can add the previous mappings by selecting `Menu`, then `Security`, then `Roles`, and then select the respective roles. For more information, see link:tr/premiumsupport/knowledge-center/es-troubleshoot-cloudwatch-logs/[How do I troubleshoot CloudWatch Logs so that it streams to my Amazon ES domain?,type="marketing"]. + +Firehose:: +Download the Firehose IAM policy to your computer. You can also https://raw.githubusercontent.com/aws-samples/amazon-eks-fluent-logging-examples/mainline/examples/fargate/kinesis-firehose/permissions.json[view the policy] on GitHub. + [source,bash,subs="verbatim,attributes"] ---- curl -O https://raw.githubusercontent.com/aws-samples/amazon-eks-fluent-logging-examples/mainline/examples/fargate/kinesis-firehose/permissions.json ---- ==== -. Create an IAM policy from the policy file you downloaded in a previous step. + +.. Create an IAM policy from the policy file that you downloaded. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- -aws iam create-policy --policy-name [.replaceable]`eks-fargate-logging-policy` --policy-document file://permissions.json +aws iam create-policy --policy-name eks-fargate-logging-policy --policy-document file://permissions.json ---- -. Attach the IAM policy to the pod execution role specified for your Fargate profile with the following command. Replace [.replaceable]`111122223333` with your account ID. Replace [.replaceable]`AmazonEKSFargatePodExecutionRole` with your [.noloc]`Pod` execution role (for more information, see <>). + +.. Attach the IAM policy to the pod execution role specified for your Fargate profile with the following command. Replace [.replaceable]`111122223333` with your account ID. Replace [.replaceable]`AmazonEKSFargatePodExecutionRole` with your Pod execution role (for more information, see <>). + [source,bash,subs="verbatim,attributes,quotes"] ---- @@ -278,22 +287,10 @@ aws iam attach-role-policy \ --role-name [.replaceable]`AmazonEKSFargatePodExecutionRole` ---- -[[fargate-logging-kubernetes-filter,fargate-logging-kubernetes-filter.title]] -=== [.noloc]`Kubernetes` filter support - -This feature requires the following minimum [.noloc]`Kubernetes` version and platform level, or later. - -[cols="1,1", options="header"] -|=== -|Kubernetes version -|Platform level - +[#fargate-logging-kubernetes-filter] +=== Kubernetes filter support -|1.23 and later -|eks.1 -|=== - -The [.noloc]`Fluent Bit` [.noloc]`Kubernetes` filter allows you to add [.noloc]`Kubernetes` metadata to your log files. For more information about the filter, see https://docs.fluentbit.io/manual/pipeline/filters/kubernetes[Kubernetes] in the [.noloc]`Fluent Bit` documentation. You can apply a filter using the API server endpoint. +The Fluent Bit Kubernetes filter allows you to add Kubernetes metadata to your log files. For more information about the filter, see https://docs.fluentbit.io/manual/pipeline/filters/kubernetes[Kubernetes] in the Fluent Bit documentation. You can apply a filter using the API server endpoint. [source,yaml,subs="verbatim,attributes,quotes"] ---- @@ -310,14 +307,14 @@ filters.conf: | ==== * `Kube_URL`, `Kube_CA_File`, `Kube_Token_Command`, and `Kube_Token_File` are service owned configuration parameters and must not be specified. Amazon EKS Fargate populates these values. -* `Kube_Meta_Cache_TTL` is the time [.noloc]`Fluent Bit` waits until it communicates with the API server for the latest metadata. If `Kube_Meta_Cache_TTL` isn't specified, Amazon EKS Fargate appends a default value of 30 minutes to lessen the load on the API server. +* `Kube_Meta_Cache_TTL` is the time Fluent Bit waits until it communicates with the API server for the latest metadata. If `Kube_Meta_Cache_TTL` isn't specified, Amazon EKS Fargate appends a default value of 30 minutes to lessen the load on the API server. ==== -[[ship-fluent-bit-process-logs,ship-fluent-bit-process-logs.title]] -=== To ship [.noloc]`Fluent Bit` process logs to your account +[#ship-fluent-bit-process-logs] +=== To ship Fluent Bit process logs to your account -You can optionally ship [.noloc]`Fluent Bit` process logs to Amazon CloudWatch using the following `ConfigMap`. Shipping Fluent Bit process logs to CloudWatch requires additional log ingestion and storage costs. Replace [.replaceable]`region-code` with the {aws} Region that your cluster is in. +You can optionally ship Fluent Bit process logs to Amazon CloudWatch using the following `ConfigMap`. Shipping Fluent Bit process logs to CloudWatch requires additional log ingestion and storage costs. Replace [.replaceable]`region-code` with the {aws} Region that your cluster is in. [source,yaml,subs="verbatim,attributes,quotes"] ---- @@ -342,31 +339,31 @@ data: auto_create_group true ---- -The logs are in the {aws} Region that the cluster resides in under CloudWatch. The log group name is `[.replaceable]``my-cluster``-fluent-bit-logs` and the [.noloc]`Fluent Bit` logstream name is `fluent-bit-[.replaceable]``podname``-[.replaceable]``pod-namespace```. +The logs are in CloudWatch in the same {aws} Region as the cluster. The log group name is `[.replaceable]``my-cluster``-fluent-bit-logs` and the Fluent Bit logstream name is `fluent-bit-[.replaceable]``podname``-[.replaceable]``pod-namespace```. [NOTE] ==== -* The process logs are shipped only when the [.noloc]`Fluent Bit` process successfully starts. If there is a failure while starting [.noloc]`Fluent Bit`, the process logs are missed. You can only ship process logs to CloudWatch. -* To debug shipping process logs to your account, you can apply the previous `ConfigMap` to get the process logs. [.noloc]`Fluent Bit` failing to start is usually due to your `ConfigMap` not being parsed or accepted by [.noloc]`Fluent Bit` while starting. +* The process logs are shipped only when the Fluent Bit process successfully starts. If there is a failure while starting Fluent Bit, the process logs are missed. You can only ship process logs to CloudWatch. +* To debug shipping process logs to your account, you can apply the previous `ConfigMap` to get the process logs. Fluent Bit failing to start is usually due to your `ConfigMap` not being parsed or accepted by Fluent Bit while starting. ==== -[[stop-fluent-bit-process-logs,stop-fluent-bit-process-logs.title]] -=== To stop shipping [.noloc]`Fluent Bit` process logs +[#stop-fluent-bit-process-logs] +=== To stop shipping Fluent Bit process logs -Shipping [.noloc]`Fluent Bit` process logs to CloudWatch requires additional log ingestion and storage costs. To exclude process logs in an existing `ConfigMap` setup, do the following steps. +Shipping Fluent Bit process logs to CloudWatch requires additional log ingestion and storage costs. To exclude process logs in an existing `ConfigMap` setup, do the following steps. -. Locate the CloudWatch log group automatically created for your Amazon EKS cluster's [.noloc]`Fluent Bit` process logs after enabling Fargate logging. It follows the format `{cluster_name}-fluent-bit-logs`. -. Delete the existing CloudWatch log streams created for each [.noloc]`Pod's` process logs in the CloudWatch log group. +. Locate the CloudWatch log group automatically created for your Amazon EKS cluster's Fluent Bit process logs after enabling Fargate logging. It follows the format `[.replaceable]``my-cluster``-fluent-bit-logs`. +. Delete the existing CloudWatch log streams created for each Pod's process logs in the CloudWatch log group. . Edit the `ConfigMap` and set `flb_log_cw: "false"`. -. Restart any existing [.noloc]`Pods` in the cluster. +. Restart any existing Pods in the cluster. -[[fargate-logging-test-application,fargate-logging-test-application.title]] +[#fargate-logging-test-application] == Test application -. Deploy a sample [.noloc]`Pod`. +. Deploy a sample Pod. + -.. Save the following contents to a file named `[.replaceable]``sample-app``.yaml` on your computer. +.. Save the following contents to a file named `sample-app.yaml` on your computer. + [source,yaml,subs="verbatim,attributes,quotes"] ---- @@ -394,28 +391,27 @@ spec: ---- .. Apply the manifest to the cluster. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- -kubectl apply -f [.replaceable]`sample-app`.yaml +kubectl apply -f sample-app.yaml ---- . View the NGINX logs using the destination(s) that you configured in the `ConfigMap`. -[[fargate-logging-size-considerations,fargate-logging-size-considerations.title]] +[#fargate-logging-size-considerations] == Size considerations We suggest that you plan for up to 50 MB of memory for the log router. If you expect your application to generate logs at very high throughput then you should plan for up to 100 MB. -[[fargate-logging-troubleshooting,fargate-logging-troubleshooting.title]] +[#fargate-logging-troubleshooting] == Troubleshooting -To confirm whether the logging feature is enabled or disabled for some reason, such as an invalid `ConfigMap`, and why it's invalid, check your [.noloc]`Pod` events with `kubectl describe pod [.replaceable]``pod-name```. The output might include [.noloc]`Pod` events that clarify whether logging is enabled or not, such as the following example output. +To confirm whether the logging feature is enabled or disabled for some reason, such as an invalid `ConfigMap`, and why it's invalid, check your Pod events with `kubectl describe pod [.replaceable]``pod-name```. The output might include Pod events that clarify whether logging is enabled or not, such as the following example output. [source,bash,subs="verbatim,attributes"] ---- [...] Annotations: CapacityProvisioned: 0.25vCPU 0.5GB Logging: LoggingDisabled: LOGGING_CONFIGMAP_NOT_FOUND - kubernetes.io/psp: eks.privileged [...] Events: Type Reason Age From Message @@ -423,4 +419,4 @@ Events: Warning LoggingDisabled fargate-scheduler Disabled logging because aws-logging configmap was not found. configmap "aws-logging" not found ---- -The [.noloc]`Pod` events are ephemeral with a time period depending on the settings. You can also view a [.noloc]`Pod's` annotations using `kubectl describe pod [.replaceable]``pod-name```. In the [.noloc]`Pod` annotation, there is information about whether the logging feature is enabled or disabled and the reason. +The Pod events are ephemeral with a time period depending on the settings. You can also view a Pod's annotations using `kubectl describe pod [.replaceable]``pod-name```. In the Pod annotation, there is information about whether the logging feature is enabled or disabled and the reason. diff --git a/latest/ug/nodes/fargate-pod-configuration.adoc b/latest/ug/nodes/fargate-pod-configuration.adoc index cbbbfac8c..9ee3cd69b 100644 --- a/latest/ug/nodes/fargate-pod-configuration.adoc +++ b/latest/ug/nodes/fargate-pod-configuration.adoc @@ -1,53 +1,45 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[fargate-pod-configuration,fargate-pod-configuration.title]] -= Understand Fargate [.noloc]`Pod` configuration details +[#fargate-pod-configuration] += Understand Fargate Pod configuration details :info_titleabbrev: Pod configuration details [abstract] -- -This section describes some of the unique [.noloc]`Pod` configuration details for running [.noloc]`Kubernetes` [.noloc]`Pods` on {aws} Fargate. +This section describes some of the unique Pod configuration details for running Kubernetes Pods on {aws} Fargate. -- -[IMPORTANT] -==== - -{aws} Fargate with Amazon EKS isn't available in {aws} GovCloud (US-East) and {aws} GovCloud (US-West). - -==== - -This section describes some of the unique [.noloc]`Pod` configuration details for running [.noloc]`Kubernetes` [.noloc]`Pods` on {aws} Fargate. +This section describes some of the unique Pod configuration details for running Kubernetes Pods on {aws} Fargate. -[[fargate-cpu-and-memory,fargate-cpu-and-memory.title]] -== [.noloc]`Pod` CPU and memory +[#fargate-cpu-and-memory] +== Pod CPU and memory -With [.noloc]`Kubernetes`, you can define requests, a minimum vCPU amount, and memory resources that are allocated to each container in a [.noloc]`Pod`. [.noloc]`Pods` are scheduled by [.noloc]`Kubernetes` to ensure that at least the requested resources for each [.noloc]`Pod` are available on the compute resource. For more information, see https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/[Managing compute resources for containers] in the [.noloc]`Kubernetes` documentation. +With Kubernetes, you can define requests, a minimum vCPU amount, and memory resources that are allocated to each container in a Pod. Pods are scheduled by Kubernetes to ensure that at least the requested resources for each Pod are available on the compute resource. For more information, see https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/[Managing compute resources for containers] in the Kubernetes documentation. [NOTE] ==== -Since Amazon EKS Fargate runs only one [.noloc]`Pod` per node, the scenario of evicting [.noloc]`Pods` in case of fewer resources doesn't occur. All Amazon EKS Fargate [.noloc]`Pods` run with guaranteed priority, so the requested CPU and memory must be equal to the limit for all of the containers. For more information, see https://kubernetes.io/docs/tasks/configure-pod-container/quality-service-pod/[Configure Quality of Service for Pods] in the [.noloc]`Kubernetes` documentation. +Since Amazon EKS Fargate runs only one Pod per node, the scenario of evicting Pods in case of fewer resources doesn't occur. All Amazon EKS Fargate Pods run with guaranteed priority, so the requested CPU and memory must be equal to the limit for all of the containers. For more information, see https://kubernetes.io/docs/tasks/configure-pod-container/quality-service-pod/[Configure Quality of Service for Pods] in the Kubernetes documentation. ==== -When [.noloc]`Pods` are scheduled on Fargate, the vCPU and memory reservations within the [.noloc]`Pod` specification determine how much CPU and memory to provision for the [.noloc]`Pod`. +When Pods are scheduled on Fargate, the vCPU and memory reservations within the Pod specification determine how much CPU and memory to provision for the Pod. * The maximum request out of any Init containers is used to determine the Init request vCPU and memory requirements. * Requests for all long-running containers are added up to determine the long-running request vCPU and memory requirements. -* The larger of the previous two values is chosen for the vCPU and memory request to use for your [.noloc]`Pod`. -* Fargate adds 256 MB to each [.noloc]`Pod's` memory reservation for the required [.noloc]`Kubernetes` components (``kubelet``, `kube-proxy`, and `containerd`). +* The larger of the previous two values is chosen for the vCPU and memory request to use for your Pod. +* Fargate adds 256 MB to each Pod's memory reservation for the required Kubernetes components (``kubelet``, `kube-proxy`, and `containerd`). -Fargate rounds up to the following compute configuration that most closely matches the sum of vCPU and memory requests in order to ensure [.noloc]`Pods` always have the resources that they need to run. +Fargate rounds up to the following compute configuration that most closely matches the sum of vCPU and memory requests in order to ensure Pods always have the resources that they need to run. If you don't specify a vCPU and memory combination, then the smallest available combination is used (.25 vCPU and 0.5 GB memory). -The following table shows the vCPU and memory combinations that are available for [.noloc]`Pods` running on Fargate. +The following table shows the vCPU and memory combinations that are available for Pods running on Fargate. -[cols="1,1", options="header"] +[%header,cols="2"] |=== |vCPU value |Memory value @@ -75,9 +67,9 @@ The following table shows the vCPU and memory combinations that are available fo |Between 32 GB and 120 GB in 8-GB increments |=== -The additional memory reserved for the [.noloc]`Kubernetes` components can cause a Fargate task with more vCPUs than requested to be provisioned. For example, a request for 1 vCPU and 8 GB memory will have 256 MB added to its memory request, and will provision a Fargate task with 2 vCPUs and 9 GB memory, since no task with 1 vCPU and 9 GB memory is available. +The additional memory reserved for the Kubernetes components can cause a Fargate task with more vCPUs than requested to be provisioned. For example, a request for 1 vCPU and 8 GB memory will have 256 MB added to its memory request, and will provision a Fargate task with 2 vCPUs and 9 GB memory, since no task with 1 vCPU and 9 GB memory is available. -There is no correlation between the size of the [.noloc]`Pod` running on Fargate and the node size reported by [.noloc]`Kubernetes` with `kubectl get nodes`. The reported node size is often larger than the [.noloc]`Pod's` capacity. You can verify [.noloc]`Pod` capacity with the following command. Replace [.replaceable]`default` with your [.noloc]`Pod's` namespace and [.replaceable]`pod-name` with the name of your [.noloc]`Pod`. +There is no correlation between the size of the Pod running on Fargate and the node size reported by Kubernetes with `kubectl get nodes`. The reported node size is often larger than the Pod's capacity. You can verify Pod capacity with the following command. Replace [.replaceable]`default` with your Pod's namespace and [.replaceable]`pod-name` with the name of your Pod. [source,bash,subs="verbatim,attributes"] ---- @@ -94,22 +86,22 @@ annotations: [...] ---- -The `CapacityProvisioned` annotation represents the enforced [.noloc]`Pod` capacity and it determines the cost of your [.noloc]`Pod` running on Fargate. For pricing information for the compute configurations, see link:fargate/pricing/[{aws} Fargate Pricing,type="marketing"]. +The `CapacityProvisioned` annotation represents the enforced Pod capacity and it determines the cost of your Pod running on Fargate. For pricing information for the compute configurations, see link:fargate/pricing/[{aws} Fargate Pricing,type="marketing"]. -[[fargate-storage,fargate-storage.title]] +[#fargate-storage] == Fargate storage -A [.noloc]`Pod` running on Fargate automatically mounts an Amazon EFS file system, without needing manual driver installation steps. You can't use dynamic persistent volume provisioning with Fargate nodes, but you can use static provisioning. For more information, see https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/docs/README.md[Amazon EFS CSI Driver] on GitHub. +A Pod running on Fargate automatically mounts an Amazon EFS file system, without needing manual driver installation steps. You can't use dynamic persistent volume provisioning with Fargate nodes, but you can use static provisioning. For more information, see https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/docs/README.md[Amazon EFS CSI Driver] on GitHub. -When provisioned, each [.noloc]`Pod` running on Fargate receives a default 20 GiB of ephemeral storage. This type of storage is deleted after a [.noloc]`Pod` stops. New [.noloc]`Pods` launched onto Fargate have encryption of the ephemeral storage volume enabled by default. The ephemeral [.noloc]`Pod` storage is encrypted with an AES-256 encryption algorithm using {aws} Fargate managed keys. +When provisioned, each Pod running on Fargate receives a default 20 GiB of ephemeral storage. This type of storage is deleted after a Pod stops. New Pods launched onto Fargate have encryption of the ephemeral storage volume enabled by default. The ephemeral Pod storage is encrypted with an AES-256 encryption algorithm using {aws} Fargate managed keys. [NOTE] ==== -The default usable storage for Amazon EKS [.noloc]`Pods` that run on Fargate is less than 20 GiB. This is because some space is used by the `kubelet` and other [.noloc]`Kubernetes` modules that are loaded inside the [.noloc]`Pod`. +The default usable storage for Amazon EKS Pods that run on Fargate is less than 20 GiB. This is because some space is used by the `kubelet` and other Kubernetes modules that are loaded inside the Pod. ==== -You can increase the total amount of ephemeral storage up to a maximum of 175 GiB. To configure the size with [.noloc]`Kubernetes`, specify the requests of `ephemeral-storage` resource to each container in a [.noloc]`Pod`. When [.noloc]`Kubernetes` schedules [.noloc]`Pods`, it ensures that the sum of the resource requests for each [.noloc]`Pod` is less than the capacity of the Fargate task. For more information, see https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/[Resource Management for Pods and Containers] in the [.noloc]`Kubernetes` documentation. +You can increase the total amount of ephemeral storage up to a maximum of 175 GiB. To configure the size with Kubernetes, specify the requests of `ephemeral-storage` resource to each container in a Pod. When Kubernetes schedules Pods, it ensures that the sum of the resource requests for each Pod is less than the capacity of the Fargate task. For more information, see https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/[Resource Management for Pods and Containers] in the Kubernetes documentation. -Amazon EKS Fargate provisions more ephemeral storage than requested for the purposes of system use. For example, a request of 100 GiB will provision a Fargate task with 115 GiB ephemeral storage. +Amazon EKS Fargate provisions more ephemeral storage than requested for the purposes of system use. For example, a request of 100 GiB will provision a Fargate task with 115 GiB ephemeral storage. \ No newline at end of file diff --git a/latest/ug/nodes/fargate-pod-patching.adoc b/latest/ug/nodes/fargate-pod-patching.adoc index 4d854ce45..55a6a381c 100644 --- a/latest/ug/nodes/fargate-pod-patching.adoc +++ b/latest/ug/nodes/fargate-pod-patching.adoc @@ -1,7 +1,7 @@ -//!!NODE_ROOT
include::../attributes.txt[] + [.topic] -[[fargate-pod-patching,fargate-pod-patching.title]] +[#fargate-pod-patching] = Set actions for {aws} Fargate OS patching events :info_titleabbrev: OS patching events @@ -10,31 +10,24 @@ include::../attributes.txt[] Amazon EKS periodically patches the OS for {aws} Fargate nodes to keep them secure. -- -[IMPORTANT] -==== - -{aws} Fargate with Amazon EKS isn't available in {aws} GovCloud (US-East) and {aws} GovCloud (US-West). - -==== - -Amazon EKS periodically patches the OS for {aws} Fargate nodes to keep them secure. As part of the patching process, we recycle the nodes to install OS patches. Updates are attempted in a way that creates the least impact on your services. However, if [.noloc]`Pods` aren't successfully evicted, there are times when they must be deleted. The following are actions that you can take to minimize potential disruptions: +Amazon EKS periodically patches the OS for {aws} Fargate nodes to keep them secure. As part of the patching process, we recycle the nodes to install OS patches. Updates are attempted in a way that creates the least impact on your services. However, if Pods aren't successfully evicted, there are times when they must be deleted. The following are actions that you can take to minimize potential disruptions: -* Set appropriate [.noloc]`Pod` disruption budgets (PDBs) to control the number of [.noloc]`Pods` that are down simultaneously. -* Create Amazon EventBridge rules to handle failed evictions before the [.noloc]`Pods` are deleted. +* Set appropriate Pod disruption budgets (PDBs) to control the number of Pods that are down simultaneously. +* Create Amazon EventBridge rules to handle failed evictions before the Pods are deleted. * Manually restart your affected pods before the eviction date posted in the notification you receive. * Create a notification configuration in {aws} User Notifications. -Amazon EKS works closely with the [.noloc]`Kubernetes` community to make bug fixes and security patches available as quickly as possible. All Fargate [.noloc]`Pods` start on the most recent [.noloc]`Kubernetes` patch version, which is available from Amazon EKS for the [.noloc]`Kubernetes` version of your cluster. If you have a [.noloc]`Pod` with an older patch version, Amazon EKS might recycle it to update it to the latest version. This ensures that your [.noloc]`Pods` are equipped with the latest security updates. That way, if there's a critical https://cve.mitre.org/[Common Vulnerabilities and Exposures] (CVE) issue, you're kept up to date to reduce security risks. +Amazon EKS works closely with the Kubernetes community to make bug fixes and security patches available as quickly as possible. All Fargate Pods start on the most recent Kubernetes patch version, which is available from Amazon EKS for the Kubernetes version of your cluster. If you have a Pod with an older patch version, Amazon EKS might recycle it to update it to the latest version. This ensures that your Pods are equipped with the latest security updates. That way, if there's a critical https://cve.mitre.org/[Common Vulnerabilities and Exposures] (CVE) issue, you're kept up to date to reduce security risks. When the {aws} Fargate OS is updated, Amazon EKS will send you a notification that includes your affected resources and the date of upcoming pod evictions. If the provided eviction date is inconvenient, you have the option to manually restart your affected pods before the eviction date posted in the notification. Any pods created before the time at which you receive the notification are subject to eviction. Refer to the https://kubernetes.io/docs/reference/kubectl/generated/kubectl_rollout/kubectl_rollout_restart[Kubernetes Documentation] for further instructions on how to manually restart your pods. -To limit the number of [.noloc]`Pods` that are down at one time when [.noloc]`Pods` are recycled, you can set [.noloc]`Pod` disruption budgets (PDBs). You can use PDBs to define minimum availability based on the requirements of each of your applications while still allowing updates to occur. Your PDB's minimum availability must be less than 100%. For more information, see https://kubernetes.io/docs/tasks/run-application/configure-pdb/[Specifying a Disruption Budget for your Application] in the [.noloc]`Kubernetes` Documentation. +To limit the number of Pods that are down at one time when Pods are recycled, you can set Pod disruption budgets (PDBs). You can use PDBs to define minimum availability based on the requirements of each of your applications while still allowing updates to occur. Your PDB's minimum availability must be less than 100%. For more information, see https://kubernetes.io/docs/tasks/run-application/configure-pdb/[Specifying a Disruption Budget for your Application] in the Kubernetes Documentation. -Amazon EKS uses the https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/#eviction-api[Eviction API] to safely drain the [.noloc]`Pod` while respecting the PDBs that you set for the application. Pods are evicted by Availability Zone to minimize impact. If the eviction succeeds, the new [.noloc]`Pod` gets the latest patch and no further action is required. +Amazon EKS uses the https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/#eviction-api[Eviction API] to safely drain the Pod while respecting the PDBs that you set for the application. Pods are evicted by Availability Zone to minimize impact. If the eviction succeeds, the new Pod gets the latest patch and no further action is required. -When the eviction for a [.noloc]`Pod` fails, Amazon EKS sends an event to your account with details about the [.noloc]`Pods` that failed eviction. You can act on the message before the scheduled termination time. The specific time varies based on the urgency of the patch. When it's time, Amazon EKS attempts to evict the [.noloc]`Pods` again. However, this time a new event isn't sent if the eviction fails. If the eviction fails again, your existing [.noloc]`Pods` are deleted periodically so that the new [.noloc]`Pods` can have the latest patch. +When the eviction for a Pod fails, Amazon EKS sends an event to your account with details about the Pods that failed eviction. You can act on the message before the scheduled termination time. The specific time varies based on the urgency of the patch. When it's time, Amazon EKS attempts to evict the Pods again. However, this time a new event isn't sent if the eviction fails. If the eviction fails again, your existing Pods are deleted periodically so that the new Pods can have the latest patch. -The following is a sample event received when the [.noloc]`Pod` eviction fails. It contains details about the cluster, [.noloc]`Pod` name, [.noloc]`Pod` namespace, Fargate profile, and the scheduled termination time. +The following is a sample event received when the Pod eviction fails. It contains details about the cluster, Pod name, Pod namespace, Fargate profile, and the scheduled termination time. [source,json,subs="verbatim,attributes"] ---- @@ -60,14 +53,14 @@ The following is a sample event received when the [.noloc]`Pod` eviction fails. } ---- -In addition, having multiple PDBs associated with a [.noloc]`Pod` can cause an eviction failure event. This event returns the following error message. +In addition, having multiple PDBs associated with a Pod can cause an eviction failure event. This event returns the following error message. [source,json,subs="verbatim,attributes"] ---- "evictErrorMessage": "This pod has multiple PodDisruptionBudget, which the eviction subresource does not support", ---- -You can create a desired action based on this event. For example, you can adjust your [.noloc]`Pod` disruption budget (PDB) to control how the [.noloc]`Pods` are evicted. More specifically, suppose that you start with a PDB that specifies the target percentage of [.noloc]`Pods` that are available. Before your [.noloc]`Pods` are force terminated during an upgrade, you can adjust the PDB to a different percentage of [.noloc]`Pods`. To receive this event, you must create an Amazon EventBridge rule in the {aws} account and {aws} Region that the cluster belongs to. The rule must use the following *Custom pattern*. For more information, see link:eventbridge/latest/userguide/eb-create-rule.html[Creating Amazon EventBridge rules that react to events,type="documentation"] in the _Amazon EventBridge User Guide_. +You can create a desired action based on this event. For example, you can adjust your Pod disruption budget (PDB) to control how the Pods are evicted. More specifically, suppose that you start with a PDB that specifies the target percentage of Pods that are available. Before your Pods are force terminated during an upgrade, you can adjust the PDB to a different percentage of Pods. To receive this event, you must create an Amazon EventBridge rule in the {aws} account and {aws} Region that the cluster belongs to. The rule must use the following *Custom pattern*. For more information, see link:eventbridge/latest/userguide/eb-create-rule.html[Creating Amazon EventBridge rules that react to events,type="documentation"] in the _Amazon EventBridge User Guide_. [source,json,subs="verbatim,attributes"] ---- @@ -77,6 +70,6 @@ You can create a desired action based on this event. For example, you can adjust } ---- -A suitable target can be set for the event to capture it. For a complete list of available targets, see link:eventbridge/latest/userguide/eb-targets.html[Amazon EventBridge targets,type="documentation"] in the _Amazon EventBridge User Guide_. You can also create a notification configuration in {aws} User Notifications. When using the {aws-management-console} to create the notification, under *Event Rules*, choose *Elastic Kubernetes Service (EKS)* for *{aws} service name* and *EKS Fargate Pod Scheduled Termination* for *Event type*. For more information, see link:notifications/latest/userguide/getting-started.html[Getting started with {aws} User Notifications,type="documentation"] in the {aws} User Notifications User Guide. +A suitable target can be set for the event to capture it. For a complete list of available targets, see link:eventbridge/latest/userguide/eb-targets.html[Amazon EventBridge targets,type="documentation"] in the _Amazon EventBridge User Guide_. You can also create a notification configuration in {aws} User Notifications. When using the {aws-management-console} to create the notification, under *Event Rules*, choose *Elastic Kubernetes Service (EKS)* for *{aws} service name* and *EKS Fargate Pod Scheduled Termination* for *Event type*. For more information, see link:notifications/latest/userguide/getting-started.html[Getting started with {aws} User Notifications,type="documentation"] in the {aws} User Notifications User Guide. -See https://repost.aws/knowledge-center/fargate-pod-eviction-notice[FAQs: Fargate Pod eviction notice] in _{aws} re:Post_ for frequently asked questions regarding EKS Pod Evictions. +See https://repost.aws/knowledge-center/fargate-pod-eviction-notice[FAQs: Fargate Pod eviction notice] in _{aws} re:Post_ for frequently asked questions regarding EKS Pod Evictions. \ No newline at end of file diff --git a/latest/ug/nodes/fargate-profile.adoc b/latest/ug/nodes/fargate-profile.adoc index cae05666c..48a7ff4a5 100644 --- a/latest/ug/nodes/fargate-profile.adoc +++ b/latest/ug/nodes/fargate-profile.adoc @@ -1,36 +1,29 @@ -//!!NODE_ROOT
include::../attributes.txt[] + [.topic] -[[fargate-profile,fargate-profile.title]] -= Define which [.noloc]`Pods` use {aws} Fargate when launched +[#fargate-profile] += Define which Pods use {aws} Fargate when launched :info_titleabbrev: Define profiles [abstract] -- -Before you schedule [.noloc]`Pods` on Fargate in your cluster, you must define at least one Fargate profile that specifies which [.noloc]`Pods` use Fargate when launched. +Before you schedule Pods on Fargate in your cluster, you must define at least one Fargate profile that specifies which Pods use Fargate when launched. -- -[IMPORTANT] -==== - -{aws} Fargate with Amazon EKS isn't available in {aws} GovCloud (US-East) and {aws} GovCloud (US-West). - -==== - -Before you schedule [.noloc]`Pods` on Fargate in your cluster, you must define at least one Fargate profile that specifies which [.noloc]`Pods` use Fargate when launched. +Before you schedule Pods on Fargate in your cluster, you must define at least one Fargate profile that specifies which Pods use Fargate when launched. -As an administrator, you can use a Fargate profile to declare which [.noloc]`Pods` run on Fargate. You can do this through the profile's selectors. You can add up to five selectors to each profile. Each selector must contain a namespace. The selector can also include labels. The label field consists of multiple optional key-value pairs. Pods that match a selector are scheduled on Fargate. Pods are matched using a namespace and the labels that are specified in the selector. If a namespace selector is defined without labels, Amazon EKS attempts to schedule all the [.noloc]`Pods` that run in that namespace onto Fargate using the profile. If a to-be-scheduled [.noloc]`Pod` matches any of the selectors in the Fargate profile, then that [.noloc]`Pod` is scheduled on Fargate. +As an administrator, you can use a Fargate profile to declare which Pods run on Fargate. You can do this through the profile's selectors. You can add up to five selectors to each profile. Each selector must contain a namespace. The selector can also include labels. The label field consists of multiple optional key-value pairs. Pods that match a selector are scheduled on Fargate. Pods are matched using a namespace and the labels that are specified in the selector. If a namespace selector is defined without labels, Amazon EKS attempts to schedule all the Pods that run in that namespace onto Fargate using the profile. If a to-be-scheduled Pod matches any of the selectors in the Fargate profile, then that Pod is scheduled on Fargate. -If a [.noloc]`Pod` matches multiple Fargate profiles, you can specify which profile a [.noloc]`Pod` uses by adding the following [.noloc]`Kubernetes` label to the [.noloc]`Pod` specification: `eks.amazonaws.com/fargate-profile: [.replaceable]``my-fargate-profile```. The [.noloc]`Pod` must match a selector in that profile to be scheduled onto Fargate. [.noloc]`Kubernetes` affinity/anti-affinity rules do not apply and aren't necessary with Amazon EKS Fargate [.noloc]`Pods`. +If a Pod matches multiple Fargate profiles, you can specify which profile a Pod uses by adding the following Kubernetes label to the Pod specification: `eks.amazonaws.com/fargate-profile: my-fargate-profile`. The Pod must match a selector in that profile to be scheduled onto Fargate. Kubernetes affinity/anti-affinity rules do not apply and aren't necessary with Amazon EKS Fargate Pods. -When you create a Fargate profile, you must specify a [.noloc]`Pod` execution role. This execution role is for the Amazon EKS components that run on the Fargate infrastructure using the profile. It's added to the cluster's [.noloc]`Kubernetes` https://kubernetes.io/docs/reference/access-authn-authz/rbac/[Role Based Access Control] (RBAC) for authorization. That way, the `kubelet` that runs on the Fargate infrastructure can register with your Amazon EKS cluster and appear in your cluster as a node. The [.noloc]`Pod` execution role also provides IAM permissions to the Fargate infrastructure to allow read access to Amazon ECR image repositories. For more information, see <>. +When you create a Fargate profile, you must specify a Pod execution role. This execution role is for the Amazon EKS components that run on the Fargate infrastructure using the profile. It's added to the cluster's Kubernetes https://kubernetes.io/docs/reference/access-authn-authz/rbac/[Role Based Access Control] (RBAC) for authorization. That way, the `kubelet` that runs on the Fargate infrastructure can register with your Amazon EKS cluster and appear in your cluster as a node. The Pod execution role also provides IAM permissions to the Fargate infrastructure to allow read access to Amazon ECR image repositories. For more information, see <>. Fargate profiles can't be changed. However, you can create a new updated profile to replace an existing profile, and then delete the original. [NOTE] ==== -Any [.noloc]`Pods` that are running using a Fargate profile are stopped and put into a pending state when the profile is deleted. +Any Pods that are running using a Fargate profile are stopped and put into a pending state when the profile is deleted. ==== @@ -39,13 +32,13 @@ If any Fargate profiles in a cluster are in the `DELETING` status, you must wait [NOTE] ==== -Fargate does not currently support [.noloc]`Kubernetes` https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/[topologySpreadConstraints]. +Fargate does not currently support Kubernetes https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/[topologySpreadConstraints]. ==== -Amazon EKS and Fargate spread [.noloc]`Pods` across each of the subnets that's defined in the Fargate profile. However, you might end up with an uneven spread. If you must have an even spread, use two Fargate profiles. Even spread is important in scenarios where you want to deploy two replicas and don't want any downtime. We recommend that each profile has only one subnet. +Amazon EKS and Fargate spread Pods across each of the subnets that's defined in the Fargate profile. However, you might end up with an uneven spread. If you must have an even spread, use two Fargate profiles. Even spread is important in scenarios where you want to deploy two replicas and don't want any downtime. We recommend that each profile has only one subnet. -[[fargate-profile-components,fargate-profile-components.title]] +[#fargate-profile-components] == Fargate profile components The following components are contained in a Fargate profile. @@ -53,26 +46,26 @@ The following components are contained in a Fargate profile. *Pod execution role*:: -When your cluster creates [.noloc]`Pods` on {aws} Fargate, the `kubelet` that's running on the Fargate infrastructure must make calls to {aws} APIs on your behalf. For example, it needs to make calls to pull container images from Amazon ECR. The Amazon EKS [.noloc]`Pod` execution role provides the IAM permissions to do this. +When your cluster creates Pods on {aws} Fargate, the `kubelet` that's running on the Fargate infrastructure must make calls to {aws} APIs on your behalf. For example, it needs to make calls to pull container images from Amazon ECR. The Amazon EKS Pod execution role provides the IAM permissions to do this. + -When you create a Fargate profile, you must specify a [.noloc]`Pod` execution role to use with your [.noloc]`Pods`. This role is added to the cluster's [.noloc]`Kubernetes` https://kubernetes.io/docs/reference/access-authn-authz/rbac/[Role-based access control] (RBAC) for authorization. This is so that the `kubelet` that's running on the Fargate infrastructure can register with your Amazon EKS cluster and appear in your cluster as a node. For more information, see <>. +When you create a Fargate profile, you must specify a Pod execution role to use with your Pods. This role is added to the cluster's Kubernetes https://kubernetes.io/docs/reference/access-authn-authz/rbac/[Role-based access control] (RBAC) for authorization. This is so that the `kubelet` that's running on the Fargate infrastructure can register with your Amazon EKS cluster and appear in your cluster as a node. For more information, see <>. *Subnets*:: -The IDs of subnets to launch [.noloc]`Pods` into that use this profile. At this time, [.noloc]`Pods` that are running on Fargate aren't assigned public IP addresses. Therefore, only private subnets with no direct route to an Internet Gateway are accepted for this parameter. +The IDs of subnets to launch Pods into that use this profile. At this time, Pods that are running on Fargate aren't assigned public IP addresses. Therefore, only private subnets with no direct route to an Internet Gateway are accepted for this parameter. *Selectors*:: -The selectors to match for [.noloc]`Pods` to use this Fargate profile. You might specify up to five selectors in a Fargate profile. The selectors have the following components: +The selectors to match for Pods to use this Fargate profile. You might specify up to five selectors in a Fargate profile. The selectors have the following components: + -** *Namespace* – You must specify a namespace for a selector. The selector only matches [.noloc]`Pods` that are created in this namespace. However, you can create multiple selectors to target multiple namespaces. -** *Labels* – You can optionally specify [.noloc]`Kubernetes` labels to match for the selector. The selector only matches [.noloc]`Pods` that have all of the labels that are specified in the selector. +** *Namespace* – You must specify a namespace for a selector. The selector only matches Pods that are created in this namespace. However, you can create multiple selectors to target multiple namespaces. +** *Labels* – You can optionally specify Kubernetes labels to match for the selector. The selector only matches Pods that have all of the labels that are specified in the selector. -[[fargate-profile-wildcards,fargate-profile-wildcards.title]] +[#fargate-profile-wildcards] == Fargate profile wildcards -In addition to characters allowed by [.noloc]`Kubernetes`, you're allowed to use `{asterisk}` and `?` in the selector criteria for namespaces, label keys, and label values: +In addition to characters allowed by Kubernetes, you're allowed to use `{asterisk}` and `?` in the selector criteria for namespaces, label keys, and label values: @@ -81,20 +74,18 @@ In addition to characters allowed by [.noloc]`Kubernetes`, you're allowed to use These wildcard characters can be used in any position and in combination (for example, `prod*`, `\*dev`, and `frontend*?`). Other wildcards and forms of pattern matching, such as regular expressions, aren't supported. -If there are multiple matching profiles for the namespace and labels in the [.noloc]`Pod` spec, Fargate picks up the profile based on alphanumeric sorting by profile name. For example, if both profile A (with the name `beta-workload`) and profile B (with the name `prod-workload`) have matching selectors for the [.noloc]`Pods` to be launched, Fargate picks profile A (`beta-workload`) for the [.noloc]`Pods`. The [.noloc]`Pods` have labels with profile A on the [.noloc]`Pods` (for example, `eks.amazonaws.com/fargate-profile=beta-workload`). - -If you want to migrate existing Fargate [.noloc]`Pods` to new profiles that use wildcards, there are two ways to do so: - +If there are multiple matching profiles for the namespace and labels in the Pod spec, Fargate picks up the profile based on alphanumeric sorting by profile name. For example, if both profile A (with the name `beta-workload`) and profile B (with the name `prod-workload`) have matching selectors for the Pods to be launched, Fargate picks profile A (`beta-workload`) for the Pods. The Pods have labels with profile A on the Pods (for example, `eks.amazonaws.com/fargate-profile=beta-workload`). +If you want to migrate existing Fargate Pods to new profiles that use wildcards, there are two ways to do so: * Create a new profile with matching selectors, then delete the old profiles. Pods labeled with old profiles are rescheduled to new matching profiles. -* If you want to migrate workloads but aren't sure what Fargate labels are on each Fargate [.noloc]`Pod`, you can use the following method. Create a new profile with a name that sorts alphanumerically first among the profiles on the same cluster. Then, recycle the Fargate [.noloc]`Pods` that need to be migrated to new profiles. +* If you want to migrate workloads but aren't sure what Fargate labels are on each Fargate Pod, you can use the following method. Create a new profile with a name that sorts alphanumerically first among the profiles on the same cluster. Then, recycle the Fargate Pods that need to be migrated to new profiles. -[[create-fargate-profile,create-fargate-profile.title]] +[#create-fargate-profile] == Create a Fargate profile -This section describes how to create a Fargate profile. You also must have created a [.noloc]`Pod` execution role to use for your Fargate profile. For more information, see <>. [.noloc]`Pods` that are running on Fargate are only supported on private subnets with link:vpc/latest/userguide/vpc-nat-gateway.html[NAT gateway,type="documentation"] access to {aws} services, but not a direct route to an Internet Gateway. This is so that your cluster's VPC must have private subnets available. +This section describes how to create a Fargate profile. You also must have created a Pod execution role to use for your Fargate profile. For more information, see <>. Pods that are running on Fargate are only supported on private subnets with link:vpc/latest/userguide/vpc-nat-gateway.html[NAT gateway,type="documentation"] access to {aws} services, but not a direct route to an Internet Gateway. This is so that your cluster's VPC must have private subnets available. You can create a profile with the following: @@ -105,7 +96,7 @@ You can create a profile with the following: *To create a Fargate profile with `eksctl`* -Create your Fargate profile with the following `eksctl` command, replacing every [.replaceable]`example value` with your own values. You're required to specify a namespace. However, the `--labels` option isn't required. +Create your Fargate profile with the following `eksctl` command, replacing every example value with your own values. You're required to specify a namespace. However, the `--labels` option isn't required. [source,bash,subs="verbatim,attributes"] ---- @@ -116,7 +107,7 @@ eksctl create fargateprofile \ --labels key=value ---- -You can use certain wildcards for [.replaceable]`my-kubernetes-namespace` and [.replaceable]`key=value` labels. For more information, see <>. +You can use certain wildcards for `my-kubernetes-namespace` and `key=value` labels. For more information, see <>. == {aws-management-console} [[console_create_a_fargate_profile]] @@ -126,25 +117,25 @@ You can use certain wildcards for [.replaceable]`my-kubernetes-namespace` and [. . Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. . Choose the cluster to create a Fargate profile for. . Choose the *Compute* tab. -. Under *Fargate profiles*, choose *Add Fargate profile*. +. Under *Fargate profiles*, choose *Add Fargate profile*. . On the *Configure Fargate profile* page, do the following: + -.. For *Name*, enter a unique name for your Fargate profile, such as [.replaceable]`my-profile`. -.. For *Pod execution role*, choose the [.noloc]`Pod` execution role to use with your Fargate profile. Only the IAM roles with the `eks-fargate-pods.amazonaws.com` service principal are shown. If you don't see any roles listed, you must create one. For more information, see <>. +.. For *Name*, enter a unique name for your Fargate profile, such as `my-profile`. +.. For *Pod execution role*, choose the Pod execution role to use with your Fargate profile. Only the IAM roles with the `eks-fargate-pods.amazonaws.com` service principal are shown. If you don't see any roles listed, you must create one. For more information, see <>. .. Modify the selected *Subnets* as needed. + -NOTE: Only private subnets are supported for [.noloc]`Pods` that are running on Fargate. -.. For *Tags*, you can optionally tag your Fargate profile. These tags don't propagate to other resources that are associated with the profile, such as [.noloc]`Pods`. +NOTE: Only private subnets are supported for Pods that are running on Fargate. +.. For *Tags*, you can optionally tag your Fargate profile. These tags don't propagate to other resources that are associated with the profile, such as Pods. .. Choose *Next*. -. On the *Configure [.noloc]`Pod` selection* page, do the following: +. On the *Configure Pod selection* page, do the following: + -.. For *Namespace*, enter a namespace to match for [.noloc]`Pods`. +.. For *Namespace*, enter a namespace to match for Pods. + *** You can use specific namespaces to match, such as `kube-system` or `default`. *** You can use certain wildcards (for example, `prod-*`) to match multiple namespaces (for example, `prod-deployment` and `prod-test`). For more information, see <>. -.. (Optional) Add [.noloc]`Kubernetes` labels to the selector. Specifically, add them to the one that the [.noloc]`Pods` in the specified namespace need to match. +.. (Optional) Add Kubernetes labels to the selector. Specifically, add them to the one that the Pods in the specified namespace need to match. + -*** You can add the label `infrastructure: fargate` to the selector so that only [.noloc]`Pods` in the specified namespace that also have the `infrastructure: fargate` [.noloc]`Kubernetes` label match the selector. +*** You can add the label `infrastructure: fargate` to the selector so that only Pods in the specified namespace that also have the `infrastructure: fargate` Kubernetes label match the selector. *** You can use certain wildcards (for example, `key?: value?`) to match multiple namespaces (for example, `keya: valuea` and `keyb: valueb`). For more information, see <>. .. Choose *Next*. -. On the *Review and create* page, review the information for your Fargate profile and choose *Create*. +. On the *Review and create* page, review the information for your Fargate profile and choose *Create*. \ No newline at end of file diff --git a/latest/ug/nodes/fargate.adoc b/latest/ug/nodes/fargate.adoc index 0a39b29a9..cf8ae7174 100644 --- a/latest/ug/nodes/fargate.adoc +++ b/latest/ug/nodes/fargate.adoc @@ -1,67 +1,53 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[fargate,fargate.title]] +[#fargate] = Simplify compute management with {aws} Fargate -:info_doctype: section -:info_title: Simplify compute management with {aws} Fargate :info_titleabbrev: {aws} Fargate -:keywords: Fargate, nodes -:info_abstract: This topic discusses using Amazon EKS to run Kubernetes Pods on {aws} Fargate. [abstract] -- -This topic discusses using Amazon EKS to run [.noloc]`Kubernetes` [.noloc]`Pods` on {aws} Fargate. +This topic discusses using Amazon EKS to run Kubernetes Pods on {aws} Fargate. -- -[IMPORTANT] -==== +This topic discusses using Amazon EKS to run Kubernetes Pods on {aws} Fargate. Fargate is a technology that provides on-demand, right-sized compute capacity for link:what-are-containers[containers,type="marketing"]. With Fargate, you don't have to provision, configure, or scale groups of virtual machines on your own to run containers. You also don't need to choose server types, decide when to scale your node groups, or optimize cluster packing. -{aws} Fargate with Amazon EKS isn't available in {aws} GovCloud (US-East) and {aws} GovCloud (US-West). +You can control which Pods start on Fargate and how they run with <>. Fargate profiles are defined as part of your Amazon EKS cluster. Amazon EKS integrates Kubernetes with Fargate by using controllers that are built by {aws} using the upstream, extensible model provided by Kubernetes. These controllers run as part of the Amazon EKS managed Kubernetes control plane and are responsible for scheduling native Kubernetes Pods onto Fargate. The Fargate controllers include a new scheduler that runs alongside the default Kubernetes scheduler in addition to several mutating and validating admission controllers. When you start a Pod that meets the criteria for running on Fargate, the Fargate controllers that are running in the cluster recognize, update, and schedule the Pod onto Fargate. -==== +This topic describes the different components of Pods that run on Fargate, and calls out special considerations for using Fargate with Amazon EKS. -This topic discusses using Amazon EKS to run [.noloc]`Kubernetes` [.noloc]`Pods` on {aws} Fargate. Fargate is a technology that provides on-demand, right-sized compute capacity for link:what-are-containers[containers,type="marketing"]. With Fargate, you don't have to provision, configure, or scale groups of virtual machines on your own to run containers. You also don't need to choose server types, decide when to scale your node groups, or optimize cluster packing. - -You can control which [.noloc]`Pods` start on Fargate and how they run with <>. Fargate profiles are defined as part of your Amazon EKS cluster. Amazon EKS integrates [.noloc]`Kubernetes` with Fargate by using controllers that are built by {aws} using the upstream, extensible model provided by [.noloc]`Kubernetes`. These controllers run as part of the Amazon EKS managed [.noloc]`Kubernetes` control plane and are responsible for scheduling native [.noloc]`Kubernetes` [.noloc]`Pods` onto Fargate. The Fargate controllers include a new scheduler that runs alongside the default [.noloc]`Kubernetes` scheduler in addition to several mutating and validating admission controllers. When you start a [.noloc]`Pod` that meets the criteria for running on Fargate, the Fargate controllers that are running in the cluster recognize, update, and schedule the [.noloc]`Pod` onto Fargate. - -This topic describes the different components of [.noloc]`Pods` that run on Fargate, and calls out special considerations for using Fargate with Amazon EKS. - -[[fargate-considerations,fargate-considerations.title]] +[#fargate-considerations] == {aws} Fargate considerations Here are some things to consider about using Fargate on Amazon EKS. - - -* Each [.noloc]`Pod` that runs on Fargate has its own isolation boundary. They don't share the underlying kernel, CPU resources, memory resources, or elastic network interface with another [.noloc]`Pod`. +* Each Pod that runs on Fargate has its own compute boundary. They don't share the underlying kernel, CPU resources, memory resources, or elastic network interface with another Pod. * Network Load Balancers and Application Load Balancers (ALBs) can be used with Fargate with IP targets only. For more information, see <> and <>. * Fargate exposed services only run on target type IP mode, and not on node IP mode. The recommended way to check the connectivity from a service running on a managed node and a service running on Fargate is to connect via service name. -* Pods must match a Fargate profile at the time that they're scheduled to run on Fargate. Pods that don't match a Fargate profile might be stuck as `Pending`. If a matching Fargate profile exists, you can delete pending [.noloc]`Pods` that you have created to reschedule them onto Fargate. -* Daemonsets aren't supported on Fargate. If your application requires a daemon, reconfigure that daemon to run as a sidecar container in your [.noloc]`Pods`. +* Pods must match a Fargate profile at the time that they're scheduled to run on Fargate. Pods that don't match a Fargate profile might be stuck as `Pending`. If a matching Fargate profile exists, you can delete pending Pods that you have created to reschedule them onto Fargate. +* Daemonsets aren't supported on Fargate. If your application requires a daemon, reconfigure that daemon to run as a sidecar container in your Pods. * Privileged containers aren't supported on Fargate. -* Pods running on Fargate can't specify `HostPort` or `HostNetwork` in the [.noloc]`Pod` manifest. -* The default `nofile` and `nproc` soft limit is 1024 and the hard limit is 65535 for Fargate [.noloc]`Pods`. +* Pods running on Fargate can't specify `HostPort` or `HostNetwork` in the Pod manifest. +* The default `nofile` and `nproc` soft limit is 1024 and the hard limit is 65535 for Fargate Pods. * GPUs aren't currently available on Fargate. * Pods that run on Fargate are only supported on private subnets (with NAT gateway access to {aws} services, but not a direct route to an Internet Gateway), so your cluster's VPC must have private subnets available. For clusters without outbound internet access, see <>. -* You can use the <> to set the initial correct size of CPU and memory for your Fargate [.noloc]`Pods`, and then use the <> to scale those [.noloc]`Pods`. If you want the Vertical Pod Autoscaler to automatically re-deploy [.noloc]`Pods` to Fargate with larger CPU and memory combinations, set the mode for the Vertical Pod Autoscaler to either `Auto` or `Recreate` to ensure correct functionality. For more information, see the https://github.com/kubernetes/autoscaler/tree/master/vertical-pod-autoscaler#quick-start[Vertical Pod Autoscaler] documentation on [.noloc]`GitHub`. -* DNS resolution and DNS hostnames must be enabled for your VPC. For more information, see link:vpc/latest/userguide/vpc-dns.html#vpc-dns-updating[Viewing and updating DNS support for your VPC,type="documentation"]. -* Amazon EKS Fargate adds defense-in-depth for [.noloc]`Kubernetes` applications by isolating each Pod within a Virtual Machine (VM). This VM boundary prevents access to host-based resources used by other Pods in the event of a container escape, which is a common method of attacking containerized applications and gain access to resources outside of the container. +* You can use the <> to set the initial correct size of CPU and memory for your Fargate Pods, and then use the <> to scale those Pods. If you want the Vertical Pod Autoscaler to automatically re-deploy Pods to Fargate with larger CPU and memory combinations, set the mode for the Vertical Pod Autoscaler to either `Auto` or `Recreate` to ensure correct functionality. For more information, see the https://github.com/kubernetes/autoscaler/tree/master/vertical-pod-autoscaler#quick-start[Vertical Pod Autoscaler] documentation on GitHub. +* DNS resolution and DNS hostnames must be enabled for your VPC. For more information, see link:vpc/latest/userguide/vpc-dns.html#vpc-dns-updating[Viewing and updating DNS support for your VPC,type="documentation"]. +* Amazon EKS Fargate adds defense-in-depth for Kubernetes applications by isolating each Pod within a Virtual Machine (VM). This VM boundary prevents access to host-based resources used by other Pods in the event of a container escape, which is a common method of attacking containerized applications and gain access to resources outside of the container. + Using Amazon EKS doesn't change your responsibilities under the <>. You should carefully consider the configuration of cluster security and governance controls. The safest way to isolate an application is always to run it in a separate cluster. -* Fargate profiles support specifying subnets from VPC secondary CIDR blocks. You might want to specify a secondary CIDR block. This is because there's a limited number of IP addresses available in a subnet. As a result, there's also a limited number of [.noloc]`Pods` that can be created in the cluster. By using different subnets for [.noloc]`Pods`, you can increase the number of available IP addresses. For more information, see link:vpc/latest/userguide/VPC_Subnets.html#vpc-resize[Adding IPv4 CIDR blocks to a VPC.,type="documentation"] -* The Amazon EC2 instance metadata service (IMDS) isn't available to [.noloc]`Pods` that are deployed to Fargate nodes. If you have [.noloc]`Pods` that are deployed to Fargate that need IAM credentials, assign them to your [.noloc]`Pods` using <>. If your [.noloc]`Pods` need access to other information available through IMDS, then you must hard code this information into your [.noloc]`Pod` spec. This includes the {aws} Region or Availability Zone that a [.noloc]`Pod` is deployed to. -* You can't deploy Fargate [.noloc]`Pods` to {aws} Outposts, {aws} Wavelength, or {aws} Local Zones. -* Amazon EKS must periodically patch Fargate [.noloc]`Pods` to keep them secure. We attempt the updates in a way that reduces impact, but there are times when [.noloc]`Pods` must be deleted if they aren't successfully evicted. There are some actions you can take to minimize disruption. For more information, see <>. +* Fargate profiles support specifying subnets from VPC secondary CIDR blocks. You might want to specify a secondary CIDR block. This is because there's a limited number of IP addresses available in a subnet. As a result, there's also a limited number of Pods that can be created in the cluster. By using different subnets for Pods, you can increase the number of available IP addresses. For more information, see link:vpc/latest/userguide/VPC_Subnets.html#vpc-resize[Adding IPv4 CIDR blocks to a VPC.,type="documentation"] +* The Amazon EC2 instance metadata service (IMDS) isn't available to Pods that are deployed to Fargate nodes. If you have Pods that are deployed to Fargate that need IAM credentials, assign them to your Pods using <>. If your Pods need access to other information available through IMDS, then you must hard code this information into your Pod spec. This includes the {aws} Region or Availability Zone that a Pod is deployed to. +* You can't deploy Fargate Pods to {aws} Outposts, {aws} Wavelength, or {aws} Local Zones. +* Amazon EKS must periodically patch Fargate Pods to keep them secure. We attempt the updates in a way that reduces impact, but there are times when Pods must be deleted if they aren't successfully evicted. There are some actions you can take to minimize disruption. For more information, see <>. * The https://github.com/aws/amazon-vpc-cni-plugins[Amazon VPC CNI plugin for Amazon EKS] is installed on Fargate nodes. You can't use <> with Fargate nodes. -* A [.noloc]`Pod` running on Fargate automatically mounts an Amazon EFS file system, without needing manual driver installation steps. You can't use dynamic persistent volume provisioning with Fargate nodes, but you can use static provisioning. +* A Pod running on Fargate automatically mounts an Amazon EFS file system, without needing manual driver installation steps. You can't use dynamic persistent volume provisioning with Fargate nodes, but you can use static provisioning. * Amazon EKS doesn't support Fargate Spot. -* You can't mount Amazon EBS volumes to Fargate [.noloc]`Pods`. -* You can run the Amazon EBS CSI controller on Fargate nodes, but the Amazon EBS CSI node [.noloc]`DaemonSet` can only run on Amazon EC2 instances. -* After a https://kubernetes.io/docs/concepts/workloads/controllers/job/[Kubernetes Job] is marked `Completed` or `Failed`, the [.noloc]`Pods` that the [.noloc]`Job` creates normally continue to exist. This behavior allows you to view your logs and results, but with Fargate you will incur costs if you don't clean up the [.noloc]`Job` afterwards. +* You can't mount Amazon EBS volumes to Fargate Pods. +* You can run the Amazon EBS CSI controller on Fargate nodes, but the Amazon EBS CSI node DaemonSet can only run on Amazon EC2 instances. +* After a https://kubernetes.io/docs/concepts/workloads/controllers/job/[Kubernetes Job] is marked `Completed` or `Failed`, the Pods that the Job creates normally continue to exist. This behavior allows you to view your logs and results, but with Fargate you will incur costs if you don't clean up the Job afterwards. + -To automatically delete the related [.noloc]`Pods` after a [.noloc]`Job` completes or fails, you can specify a time period using the time-to-live (TTL) controller. The following example shows specifying `.spec.ttlSecondsAfterFinished` in your [.noloc]`Job` manifest. +To automatically delete the related Pods after a Job completes or fails, you can specify a time period using the time-to-live (TTL) controller. The following example shows specifying `.spec.ttlSecondsAfterFinished` in your Job manifest. + [source,yaml,subs="verbatim,attributes"] ---- @@ -82,7 +68,7 @@ spec: == Fargate Comparison Table -[cols="1,1", options="header"] +[%header,cols="2"] |=== |Criteria |{aws} Fargate @@ -93,10 +79,10 @@ spec: |Can be deployed to an <> |No -|Can run containers that require [.noloc]`Windows` +|Can run containers that require Windows |No -|Can run containers that require [.noloc]`Linux` +|Can run containers that require Linux |Yes |Can run workloads that require the Inferentia chip @@ -111,14 +97,14 @@ spec: |Can run {aws} link:bottlerocket/[Bottlerocket,type="marketing"] |No -|Pods share a kernel runtime environment with other [.noloc]`Pods` -|No – Each [.noloc]`Pod` has a dedicated kernel +|Pods share a kernel runtime environment with other Pods +|No – Each Pod has a dedicated kernel -|Pods share CPU, memory, storage, and network resources with other [.noloc]`Pods`. -|No – Each [.noloc]`Pod` has dedicated resources and can be sized independently to maximize resource utilization. +|Pods share CPU, memory, storage, and network resources with other Pods. +|No – Each Pod has dedicated resources and can be sized independently to maximize resource utilization. -|Pods can use more hardware and memory than requested in [.noloc]`Pod` specs -|No – The [.noloc]`Pod` can be re-deployed using a larger vCPU and memory configuration though. +|Pods can use more hardware and memory than requested in Pod specs +|No – The Pod can be re-deployed using a larger vCPU and memory configuration though. |Must deploy and manage Amazon EC2 instances |No @@ -129,7 +115,7 @@ spec: |Can provide bootstrap arguments at deployment of a node, such as extra https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/[kubelet] arguments. |No -|Can assign IP addresses to [.noloc]`Pods` from a different CIDR block than the IP address assigned to the node. +|Can assign IP addresses to Pods from a different CIDR block than the IP address assigned to the node. |No |Can SSH into node @@ -144,16 +130,16 @@ spec: |Must update node AMI on your own |No -|Must update node [.noloc]`Kubernetes` version on your own +|Must update node Kubernetes version on your own |No – You don't manage nodes. -|Can use Amazon EBS storage with [.noloc]`Pods` +|Can use Amazon EBS storage with Pods |No -|Can use Amazon EFS storage with [.noloc]`Pods` +|Can use Amazon EFS storage with Pods |<> -|Can use Amazon FSx for Lustre storage with [.noloc]`Pods` +|Can use Amazon FSx for Lustre storage with Pods |No |Can use Network Load Balancer for services @@ -162,23 +148,23 @@ spec: |Pods can run in a public subnet |No -|Can assign different VPC security groups to individual [.noloc]`Pods` +|Can assign different VPC security groups to individual Pods |Yes -|Can run [.noloc]`Kubernetes` [.noloc]`DaemonSets` +|Can run Kubernetes DaemonSets |No -|Support `HostPort` and `HostNetwork` in the [.noloc]`Pod` manifest +|Support `HostPort` and `HostNetwork` in the Pod manifest |No |{aws} Region availability -|<> +|link:general/latest/gr/eks.html[Some Amazon EKS supported regions,type="documentation"] |Can run containers on Amazon EC2 dedicated hosts |No |Pricing -|Cost of an individual Fargate memory and CPU configuration. Each [.noloc]`Pod` has its own cost. For more information, see link:fargate/pricing/[{aws} Fargate pricing,type="marketing"]. +|Cost of an individual Fargate memory and CPU configuration. Each Pod has its own cost. For more information, see link:fargate/pricing/[{aws} Fargate pricing,type="marketing"]. |=== diff --git a/latest/ug/nodes/hybrid-nodes-add-ons.adoc b/latest/ug/nodes/hybrid-nodes-add-ons.adoc index 317b2d471..67f0b6085 100644 --- a/latest/ug/nodes/hybrid-nodes-add-ons.adoc +++ b/latest/ug/nodes/hybrid-nodes-add-ons.adoc @@ -1,27 +1,22 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[hybrid-nodes-add-ons,hybrid-nodes-add-ons.title]] +[#hybrid-nodes-add-ons] = Configure add-ons for hybrid nodes -:info_doctype: section -:info_title: Configure common add-ons for hybrid nodes :info_titleabbrev: Configure add-ons -:keywords: add-ons for on-premises nodes, add-ons for hybrid nodes -:info_abstract: Configure common add-ons for hybrid nodes - -include::../attributes.txt[] [abstract] -- Configure common add-ons for hybrid nodes -- -This page describes considerations for running Amazon EKS add-ons from {aws} on Amazon EKS Hybrid Nodes. To learn more about the Amazon EKS add-ons from {aws} and the processes for creating, upgrading, and removing add-ons from your cluster, see <>. The processes for creating, upgrading, and removing Amazon EKS add-ons is the same for Amazon EKS clusters with hybrid nodes as it is for Amazon EKS clusters with nodes running in {aws} Cloud unless otherwise noted on this page. +This page describes considerations for running {aws} add-ons and community add-ons on Amazon EKS Hybrid Nodes. To learn more about Amazon EKS add-ons and the processes for creating, upgrading, and removing add-ons from your cluster, see <>. Unless otherwise noted on this page, the processes for creating, upgrading, and removing Amazon EKS add-ons is the same for Amazon EKS clusters with hybrid nodes as it is for Amazon EKS clusters with nodes running in {aws} Cloud. Only the add-ons included on this page have been validated for compatibility with Amazon EKS Hybrid Nodes. -The following Amazon EKS add-ons from {aws} are compatible with Amazon EKS Hybrid Nodes. +The following {aws} add-ons are compatible with Amazon EKS Hybrid Nodes. -[cols="1,1", options="header"] +[%header,cols="2"] |=== -|EKS add-on +|{aws} add-on |Compatible add-on versions |kube-proxy @@ -33,38 +28,72 @@ The following Amazon EKS add-ons from {aws} are compatible with Amazon EKS Hybri |{aws} Distro for OpenTelemetry (ADOT) |v0.102.1-eksbuild.2 and above -|CloudWatch Observability Agent +|CloudWatch Observability agent |v2.2.1-eksbuild.1 and above |EKS Pod Identity Agent -|v1.3.3-eksbuild.1 and above +a|* v1.3.3-eksbuild.1 and above, except for Bottlerocket +* v1.3.7-eksbuild.2 and above for Bottlerocket + +|Node monitoring agent +|v1.2.0-eksbuild.1 and above |CSI snapshot controller |v8.1.0-eksbuild.1 and above + +|{aws} Private CA Connector for Kubernetes +|v1.6.0-eksbuild.1 and above |=== -In addition to the Amazon EKS add-ons in the table above, the <>, and the <> for <> (HTTP) and <> (TCP/UDP) are compatible with hybrid nodes. +The following community add-ons are compatible with Amazon EKS Hybrid Nodes. To learn more about community add-ons, see <>. -Amazon EKS add-ons from {aws} that are not compatible with Amazon EKS Hybrid Nodes have been updated with an affinity rule for the default eks.amazonaws.com/compute-type: hybrid label applied to hybrid nodes. This prevents them from running on hybrid nodes when deployed in your clusters. If you have clusters with both hybrid nodes and nodes running in {aws} Cloud, Amazon EKS add-ons that are not compatible with hybrid nodes can still be deployed in your cluster to nodes running in {aws} Cloud. The Amazon VPC CNI is not compatible with hybrid nodes, and Cilium and Calico are supported as the Container Networking Interfaces (CNIs) for Amazon EKS Hybrid Nodes. See <> for more information. +[%header,cols="2"] +|=== +|Community add-on +|Compatible add-on versions + +|Kubernetes Metrics Server +|v0.7.2-eksbuild.1 and above -The rest of this page describes differences between running compatible Amazon EKS add-ons from {aws} on hybrid nodes, compared to the other Amazon EKS compute types. +|cert-manager +|v1.17.2-eksbuild.1 and above -[[hybrid-nodes-add-ons-core,hybrid-nodes-add-ons-core.title]] +|Prometheus Node Exporter +|v1.9.1-eksbuild.2 and above + +|kube-state-metrics +|v2.15.0-eksbuild.4 and above + +|External DNS +|v0.19.0-eksbuild.1 and above +|=== + +In addition to the Amazon EKS add-ons in the tables above, the <>, and the <> for <> (HTTP) and <> (TCP/UDP) are compatible with hybrid nodes. + +There are {aws} add-ons and community add-ons that aren't compatible with Amazon EKS Hybrid Nodes. The latest versions of these add-ons have an anti-affinity rule for the default `eks.amazonaws.com/compute-type: hybrid` label applied to hybrid nodes. This prevents them from running on hybrid nodes when deployed in your clusters. If you have clusters with both hybrid nodes and nodes running in {aws} Cloud, you can deploy these add-ons in your cluster to nodes running in {aws} Cloud. The Amazon VPC CNI is not compatible with hybrid nodes, and Cilium and Calico are supported as the Container Networking Interfaces (CNIs) for Amazon EKS Hybrid Nodes. See <> for more information. + +[#hybrid-nodes-add-ons-aws-add-ons] +== {aws} add-ons + +The sections that follow describe differences between running compatible {aws} add-ons on hybrid nodes compared to other Amazon EKS compute types. + +[#hybrid-nodes-add-ons-core] == kube-proxy and CoreDNS -Kube-proxy and CoreDNS are installed as unmanaged add-ons by default when an EKS cluster is created. These add-ons can be managed as Amazon EKS add-ons after cluster creation. Reference the EKS documentation for details on <> and <>. If you are running a cluster with hybrid nodes and nodes in {aws} Cloud, it is recommended to have at least one CoreDNS replica on hybrid nodes and at least one CoreDNS replica on your nodes in {aws} Cloud. +EKS installs kube-proxy and CoreDNS as self-managed add-ons by default when you create an EKS cluster with the {aws} API and {aws} SDKs, including from the {aws} CLI. You can overwrite these add-ons with Amazon EKS add-ons after cluster creation. Reference the EKS documentation for details on <> and <>. If you are running a mixed mode cluster with both hybrid nodes and nodes in {aws} Cloud, {aws} recommends to have at least one CoreDNS replica on hybrid nodes and at least one CoreDNS replica on your nodes in {aws} Cloud. See <> for configuration steps. -[[hybrid-nodes-add-ons-cw,hybrid-nodes-add-ons-cw.title]] -== CloudWatch Observability Agent add-on +[#hybrid-nodes-add-ons-cw] +== CloudWatch Observability agent +The CloudWatch Observability agent operator uses https://kubernetes.io/docs/reference/access-authn-authz/webhook/[webhooks]. If you run the operator on hybrid nodes, your on-premises pod CIDR must be routable on your on-premises network and you must configure your EKS cluster with your remote pod network. For more information, see <>. Node-level metrics are not available for hybrid nodes because link:AmazonCloudWatch/latest/monitoring/ContainerInsights.html[CloudWatch Container Insights,type="documentation"] depends on the availability of link:AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html[Instance Metadata Service,type="documentation"] (IMDS) for node-level metrics. Cluster, workload, pod, and container-level metrics are available for hybrid nodes. After installing the add-on by following the steps described in link:AmazonCloudWatch/latest/monitoring/install-CloudWatch-Observability-EKS-addon.html[Install the CloudWatch agent with the Amazon CloudWatch Observability,type="documentation"], the add-on manifest must be updated before the agent can run successfully on hybrid nodes. Edit the `amazoncloudwatchagents` resource on the cluster to add the `RUN_WITH_IRSA` environment variable as shown below. -[source,yaml,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- kubectl edit amazoncloudwatchagents -n amazon-cloudwatch cloudwatch-agent ---- -[source,yaml,subs="verbatim,attributes,quotes"] +[source,yaml,subs="verbatim,attributes"] ---- apiVersion: v1 items: @@ -87,7 +116,7 @@ items: ... ---- -[[hybrid-nodes-add-ons-amp,hybrid-nodes-add-ons-amp.title]] +[#hybrid-nodes-add-ons-amp] == Amazon Managed Prometheus managed collector for hybrid nodes An Amazon Managed Service for Prometheus (AMP) managed collector consists of a scraper that discovers and collects metrics from the resources in an Amazon EKS cluster. AMP manages the scraper for you, removing the need to manage any instances, agents, or scrapers yourself. @@ -96,42 +125,48 @@ You can use AMP managed collectors without any additional configuration specific Follow the steps in link:prometheus/latest/userguide/AMP-collector-how-to.html[Using an {aws} managed collector,type="documentation"] in the Amazon Managed Service for Prometheus User Guide. -[[hybrid-nodes-add-ons-adot,hybrid-nodes-add-ons-adot.title]] -== {aws} Distro for OpenTelemetry (ADOT) add-on +[#hybrid-nodes-add-ons-adot] +== {aws} Distro for OpenTelemetry (ADOT) -You can use the {aws} Distro for OpenTelemetry (ADOT) Amazon EKS add-on to collect metrics, logs, and tracing data from your applications running on hybrid nodes. Note, ADOT uses admission https://kubernetes.io/docs/reference/access-authn-authz/webhook/[webhooks] to mutate and validate the Collector Custom Resource requests and you must configure your remote pod network when creating your Amazon EKS cluster. +You can use the {aws} Distro for OpenTelemetry (ADOT) add-on to collect metrics, logs, and tracing data from your applications running on hybrid nodes. ADOT uses admission https://kubernetes.io/docs/reference/access-authn-authz/webhook/[webhooks] to mutate and validate the Collector Custom Resource requests. If you run the ADOT operator on hybrid nodes, your on-premises pod CIDR must be routable on your on-premises network and you must configure your EKS cluster with your remote pod network. For more information, see <>. Follow the steps in https://aws-otel.github.io/docs/getting-started/adot-eks-add-on[Getting Started with {aws} Distro for OpenTelemetry using EKS Add-Ons] in the _{aws} Distro for OpenTelemetry_ documentation. -[[hybrid-nodes-add-ons-lbc,hybrid-nodes-add-ons-lbc.title]] +[#hybrid-nodes-add-ons-lbc] == {aws} Load Balancer Controller -You can use the <> and Application Load Balancer (ALB) or Network Load Balancer (NLB) with the target type ip for workloads on hybrid nodes connected with {aws} Direct Connect or {aws} Site-to-Site VPN. As the {aws} Load Balancer Controller uses https://kubernetes.io/docs/reference/access-authn-authz/webhook/[webhooks], you must configure your remote pod network when creating your Amazon EKS cluster. +You can use the <> and Application Load Balancer (ALB) or Network Load Balancer (NLB) with the target type `ip` for workloads on hybrid nodes The IP target(s) used with the ALB or NLB must be routable from {aws}. The {aws} Load Balancer controller also uses https://kubernetes.io/docs/reference/access-authn-authz/webhook/[webhooks]. If you run the {aws} Load Balancer Controller operator on hybrid nodes, your on-premises pod CIDR must be routable on your on-premises network and you must configure your EKS cluster with your remote pod network. For more information, see <>. -To install the {aws} Load Balancer Controller, follow the steps at <> or <>. +To install the {aws} Load Balancer Controller, follow the steps at <> or <>. -For ingress with ALB, you must specify the annotations below. See <> for instructions. -[source,yaml,subs="verbatim,attributes,quotes"] +For ingress with ALB, you must specify the annotations below. See <> for more information. +[source,yaml,subs="verbatim,attributes"] ---- -alb.ingress.kubernetes.io/scheme: internal alb.ingress.kubernetes.io/target-type: ip ---- -For load balancing with NLB, you must specify the annotations below. See <> for instructions. -[source,yaml,subs="verbatim,attributes,quotes"] +For load balancing with NLB, you must specify the annotations below. See <> for more information. +[source,yaml,subs="verbatim,attributes"] ---- service.beta.kubernetes.io/aws-load-balancer-type: "external" service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip" ---- -[[hybrid-nodes-add-ons-pod-id,hybrid-nodes-add-ons-pod-id.title]] -== EKS Pod Identity Agent add-on +[#hybrid-nodes-add-ons-pod-id] +== EKS Pod Identity Agent + +[NOTE] +==== +To successfully deploy the EKS Pod Identity Agent add-on on hybrid nodes running Bottlerocket, ensure your Bottlerocket version is at least v1.39.0. The Pod Identity Agent is not supported on earlier Bottlerocket versions in hybrid node environments. +==== -The original Amazon EKS Pod Identity Agent [.noloc]`DaemonSet` relies on the availability of EC2 IMDS on the node to obtain the required {aws} credentials. As IMDS isn't available on hybrid nodes, starting in add-on version `1.3.3-eksbuild.1`, the Pod Identity Agent add-on optionally deploys a second [.noloc]`DaemonSet` that specifically targets hybrid nodes. This [.noloc]`DaemonSet` mounts the required credentials to the pods created by the Pod Identity Agent add-on. +The original Amazon EKS Pod Identity Agent DaemonSet relies on the availability of EC2 IMDS on the node to obtain the required {aws} credentials. As IMDS isn't available on hybrid nodes, starting with version 1.3.3-eksbuild.1, the Pod Identity Agent add-on optionally deploys a DaemonSet that mounts the required credentials. Hybrid nodes running Bottlerocket require a different method to mount the credentials, and starting in version 1.3.7-eksbuild.2, the Pod Identity Agent add-on optionally deploys a DaemonSet that specifically targets Bottlerocket hybrid nodes. The following sections describe the process for enabling the optional DaemonSets. -. To use the Pod Identity agent on hybrid nodes, set `enableCredentialsFile: true` in the hybrid section of `nodeadm` config as shown below: +=== Ubuntu/RHEL/AL2023 + +. To use the Pod Identity agent on Ubuntu/RHEL/Al2023 hybrid nodes, set `enableCredentialsFile: true` in the hybrid section of `nodeadm` config as shown below: + -[source,yaml,subs="verbatim,attributes,quotes"] +[source,yaml,subs="verbatim,attributes"] ---- apiVersion: node.eks.aws/v1alpha1 kind: NodeConfig @@ -144,31 +179,83 @@ This will configure `nodeadm` to create a credentials file to be configured on t + . After you update the `nodeadm` config on _each_ node, run the following `nodeadm init` command with your `nodeConfig.yaml` to join your hybrid nodes to your Amazon EKS cluster. If your nodes have joined the cluster previous, still run the `init` command again. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- nodeadm init -c file://nodeConfig.yaml ---- + -. Install `eks-pod-identity-agent` with support for hybrid nodes enabled, by either using the {cli} or {aws-management-console}. +. Install `eks-pod-identity-agent` with support for hybrid nodes enabled, by using either the {aws} CLI or {aws-management-console}. + -.. {cli}: From the machine that you're using to administer the cluster, run the following command to install `eks-pod-identity-agent` with support for hybrid nodes enabled. +.. {aws} CLI: From the machine that you're using to administer the cluster, run the following command to install `eks-pod-identity-agent` with support for hybrid nodes enabled. Replace `my-cluster` with the name of your cluster. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- aws eks create-addon \ - --cluster-name [.replaceable]`cluster-name` \ + --cluster-name my-cluster \ --addon-name eks-pod-identity-agent \ --configuration-values '{"daemonsets":{"hybrid":{"create": true}}}' ---- + -.. {aws-management-console}: If you are installing the Pod Identity Agent add-on through the {aws} console, add the following to the optional configuration to deploy the daemonset that targets hybrid nodes. +.. {aws-management-console}: If you are installing the Pod Identity Agent add-on through the {aws} console, add the following to the optional configuration to deploy the DaemonSet that targets hybrid nodes. + -[source,yaml,subs="verbatim,attributes,quotes"] +[source,yaml,subs="verbatim,attributes"] ---- {"daemonsets":{"hybrid":{"create": true}}} ---- -[[hybrid-nodes-add-ons-csi-snapshotter,hybrid-nodes-add-ons-csi-snapshotter.title]] -== CSI snapshot controller add-on +=== Bottlerocket + +. To use the Pod Identity agent on Bottlerocket hybrid nodes, add the `--enable-credentials-file=true` flag to the command used for the Bottlerocket bootstrap container user data, as described in <>. ++ +.. If you are using the SSM credential provider, your command should look like this: ++ +[source,bash,subs="verbatim,attributes"] +---- +eks-hybrid-ssm-setup --activation-id= --activation-code= --region= --enable-credentials-file=true +---- ++ +.. If you are using the IAM Roles Anywhere credential provider, your command should look like this: ++ +[source,bash,subs="verbatim,attributes"] +---- +eks-hybrid-iam-ra-setup --certificate= --key= --enable-credentials-file=true +---- ++ +This will configure the bootstrap script to create a credentials file on the node under `/var/eks-hybrid/.aws/credentials`, which will be used by `eks-pod-identity-agent` pods. This credentials file will contain temporary {aws} credentials that will be refreshed periodically. ++ +. Install `eks-pod-identity-agent` with support for Bottlerocket hybrid nodes enabled, by using either the {aws} CLI or {aws-management-console}. ++ +.. {aws} CLI: From the machine that you're using to administer the cluster, run the following command to install `eks-pod-identity-agent` with support for Bottlerocket hybrid nodes enabled. Replace `my-cluster` with the name of your cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks create-addon \ + --cluster-name my-cluster \ + --addon-name eks-pod-identity-agent \ + --configuration-values '{"daemonsets":{"hybrid-bottlerocket":{"create": true}}}' +---- ++ +.. {aws-management-console}: If you are installing the Pod Identity Agent add-on through the {aws} console, add the following to the optional configuration to deploy the DaemonSet that targets Bottlerocket hybrid nodes. ++ +[source,yaml,subs="verbatim,attributes"] +---- +{"daemonsets":{"hybrid-bottlerocket":{"create": true}}} +---- + +[#hybrid-nodes-add-ons-csi-snapshotter] +== CSI snapshot controller Starting with version `v8.1.0-eksbuild.2`, the <> applies a soft anti-affinity rule for hybrid nodes, preferring the controller `deployment` to run on EC2 in the same {aws} Region as the Amazon EKS control plane. Co-locating the `deployment` in the same {aws} Region as the Amazon EKS control plane improves latency. + +[#hybrid-nodes-add-ons-community] +== Community add-ons + +The sections that follow describe differences between running compatible community add-ons on hybrid nodes compared to other Amazon EKS compute types. + +[#hybrid-nodes-add-ons-metrics-server] +== Kubernetes Metrics Server +The control plane needs to reach Metrics Server's pod IP (or node IP if hostNetwork is enabled). Therefore, unless you run Metrics Server in hostNetwork mode, you must configure a remote pod network when creating your Amazon EKS cluster, and you must make your pod IP addresses routable. Implementing Border Gateway Protocol (BGP) with the CNI is one common way to make your pod IP addresses routable. + +[#hybrid-nodes-add-ons-cert-manager] +== cert-manager +`cert-manager` uses https://kubernetes.io/docs/reference/access-authn-authz/webhook/[webhooks]. If you run `cert-manager` on hybrid nodes, your on-premises pod CIDR must be routable on your on-premises network and you must configure your EKS cluster with your remote pod network. For more information, see <>. diff --git a/latest/ug/nodes/hybrid-nodes-bottlerocket.adoc b/latest/ug/nodes/hybrid-nodes-bottlerocket.adoc new file mode 100644 index 000000000..5f1b2e706 --- /dev/null +++ b/latest/ug/nodes/hybrid-nodes-bottlerocket.adoc @@ -0,0 +1,223 @@ +include::../attributes.txt[] + +[.topic] +[#hybrid-nodes-bottlerocket] += Connect hybrid nodes with Bottlerocket +:info_titleabbrev: Connect hybrid nodes with Bottlerocket + +[abstract] +-- +Connect hybrid nodes running Bottlerocket to an Amazon EKS cluster. +-- + +This topic describes how to connect hybrid nodes running Bottlerocket to an Amazon EKS cluster. link:bottlerocket/[Bottlerocket,type="marketing"] is an open source Linux distribution that is sponsored and supported by {aws}. Bottlerocket is purpose-built for hosting container workloads. With Bottlerocket, you can improve the availability of containerized deployments and reduce operational costs by automating updates to your container infrastructure. Bottlerocket includes only the essential software to run containers, which improves resource usage, reduces security threats, and lowers management overhead. + +Only VMware variants of Bottlerocket version v1.37.0 and above are supported with EKS Hybrid Nodes. VMware variants of Bottlerocket are available for Kubernetes versions v1.28 and above. The OS images for these variants include the kubelet, containerd, aws-iam-authenticator and other software prerequisites for EKS Hybrid Nodes. You can configure these components using a Bottlerocket https://github.com/bottlerocket-os/bottlerocket?tab=readme-ov-file#settings[settings] file that includes base64 encoded user-data for the Bottlerocket bootstrap and admin containers. Configuring these settings enables Bottlerocket to use your hybrid nodes credentials provider to authenticate hybrid nodes to your cluster. After your hybrid nodes join the cluster, they will appear with status `Not Ready` in the Amazon EKS console and in Kubernetes-compatible tooling such as `kubectl`. After completing the steps on this page, proceed to <> to make your hybrid nodes ready to run applications. + +== Prerequisites + +Before connecting hybrid nodes to your Amazon EKS cluster, make sure you have completed the prerequisite steps. + +* You have network connectivity from your on-premises environment to the {aws} Region hosting your Amazon EKS cluster. See <> for more information. +* You have created your Hybrid Nodes IAM role and set up your on-premises credential provider ({aws} Systems Manager hybrid activations or {aws} IAM Roles Anywhere). See <> for more information. +* You have created your hybrid nodes-enabled Amazon EKS cluster. See <> for more information. +* You have associated your Hybrid Nodes IAM role with Kubernetes Role-Based Access Control (RBAC) permissions. See <> for more information. + +== Step 1: Create the Bottlerocket settings TOML file + +To configure Bottlerocket for hybrid nodes, you need to create a `settings.toml` file with the necessary configuration. The contents of the TOML file will differ based on the credential provider you are using (SSM or IAM Roles Anywhere). This file will be passed as user data when provisioning the Bottlerocket instance. + +=== SSM + +If you are using {aws} Systems Manager as your credential provider, create a `settings.toml` file with the following content: + +[source,toml,subs="verbatim,attributes,quotes"] +---- +[settings.kubernetes] +cluster-name = "" +api-server = "" +cluster-certificate = "" +hostname-override = "" +provider-id = "eks-hybrid://///" +authentication-mode = "aws" +cloud-provider = "" + +[settings.network] +hostname = "" + +[settings.aws] +region = "" + +[settings.kubernetes.node-labels] +"eks.amazonaws.com/compute-type" = "hybrid" +"eks.amazonaws.com/hybrid-credential-provider" = "ssm" + +[settings.host-containers.admin] +enabled = true +user-data = "" + +[settings.bootstrap-containers.eks-hybrid-setup] +mode = "always" +user-data = "" + +[settings.host-containers.control] +enabled = true +---- + +Replace the placeholders with the following values: + +* ``: The name of your Amazon EKS cluster. +* ``: The API server endpoint of your cluster. +* ``: The base64-encoded CA bundle of your cluster. +* ``: The {aws} Region hosting your cluster, for example "us-east-1". +* ``: The hostname of the Bottlerocket instance, which will also be configured as the node name. This can be any unique value of your choice, but must follow the https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names[Kubernetes Object naming conventions]. In addition, the hostname you use cannot be longer than 64 characters. NOTE: When using SSM provider, this hostname and node name will be replaced by the managed instance ID (for example, `mi-*` ID) after the instance has been registered with SSM. +* ``: The base64-encoded contents of the Bottlerocket admin container configuration. Enabling the admin container allows you to connect to your Bottlerocket instance with SSH for system exploration and debugging. While this is not a required setting, we recommend enabling it for ease of troubleshooting. Refer to the https://github.com/bottlerocket-os/bottlerocket-admin-container#authenticating-with-the-admin-container[Bottlerocket admin container documentation] for more information on authenticating with the admin container. The admin container takes SSH user and key input in JSON format, for example, + +[source,json,subs="verbatim,attributes"] +---- +{ + "user": "", + "ssh": { + "authorized-keys": [ + "" + ] + } +} +---- + +* ``: The base64-encoded contents of the Bottlerocket bootstrap container configuration. Refer to the https://github.com/bottlerocket-os/bottlerocket-bootstrap-container[Bottlerocket bootstrap container documentation] for more information on its configuration. The bootstrap container is responsible for registering the instance as an {aws} SSM Managed Instance and joining it as a Kubernetes node on your Amazon EKS Cluster. The user data passed into the bootstrap container takes the form of a command invocation which accepts as input the SSM hybrid activation code and ID you previously created: + +[source,bash,subs="verbatim,attributes"] +---- +eks-hybrid-ssm-setup --activation-id= --activation-code= --region= +---- + +=== IAM Roles Anywhere + +If you are using {aws} IAM Roles Anywhere as your credential provider, create a `settings.toml` file with the following content: + +[source,toml,subs="verbatim,attributes,quotes"] +---- +[settings.kubernetes] +cluster-name = "" +api-server = "" +cluster-certificate = "" +hostname-override = "" +provider-id = "eks-hybrid://///" +authentication-mode = "aws" +cloud-provider = "" + +[settings.network] +hostname = "" + +[settings.aws] +region = "" +config = "" + +[settings.kubernetes.node-labels] +"eks.amazonaws.com/compute-type" = "hybrid" +"eks.amazonaws.com/hybrid-credential-provider" = "iam-ra" + +[settings.host-containers.admin] +enabled = true +user-data = "" + +[settings.bootstrap-containers.eks-hybrid-setup] +mode = "always" +user-data = "" +---- + +Replace the placeholders with the following values: + +* ``: The name of your Amazon EKS cluster. +* ``: The API server endpoint of your cluster. +* ``: The base64-encoded CA bundle of your cluster. +* ``: The {aws} Region hosting your cluster, e.g., "us-east-1" +* ``: The hostname of the Bottlerocket instance, which will also be configured as the node name. This can be any unique value of your choice, but must follow the https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names[Kubernetes Object naming conventions]. In addition, the hostname you use cannot be longer than 64 characters. NOTE: When using IAM-RA provider, the node name must match the CN of the certificate on the host if you configured the trust policy of your Hybrid Nodes IAM role with the `"sts:RoleSessionName": "${aws:PrincipalTag/x509Subject/CN}"` resource condition. +* ``: The base64-encoded contents of your {aws} config file. The contents of the file should be as follows: +[source,bash,subs="verbatim,attributes"] +---- +[default] +credential_process = aws_signing_helper credential-process --certificate /root/.aws/node.crt --private-key /root/.aws/node.key --profile-arn --role-arn --trust-anchor-arn --role-session-name +---- +* ``: The base64-encoded contents of the Bottlerocket admin container configuration. Enabling the admin container allows you to connect to your Bottlerocket instance with SSH for system exploration and debugging. While this is not a required setting, we recommend enabling it for ease of troubleshooting. Refer to the https://github.com/bottlerocket-os/bottlerocket-admin-container#authenticating-with-the-admin-container[Bottlerocket admin container documentation] for more information on authenticating with the admin container. The admin container takes SSH user and key input in JSON format, for example, + +[source,json,subs="verbatim,attributes"] +---- +{ + "user": "", + "ssh": { + "authorized-keys": [ + "" + ] + } +} +---- + +* ``: The base64-encoded contents of the Bottlerocket bootstrap container configuration. Refer to the https://github.com/bottlerocket-os/bottlerocket-bootstrap-container[Bottlerocket bootstrap container documentation] for more information on its configuration. The bootstrap container is responsible for creating the IAM Roles Anywhere host certificate and certificate private key files on the instance. These will then be consumed by the `aws_signing_helper` to obtain temporary credentials for authenticating with your Amazon EKS cluster. The user data passed into the bootstrap container takes the form of a command invocation which accepts as input the contents of the certificate and private key you previously created: + +[source,bash,subs="verbatim,attributes"] +---- +eks-hybrid-iam-ra-setup --certificate= --key= +---- + +== Step 2: Provision the Bottlerocket vSphere VM with user data + +Once you have constructed the TOML file, pass it as user data during vSphere VM creation. Keep in mind that the user data must be configured before the VM is powered on for the first time. As such, you will need to supply it when creating the instance, or if you wish to create the VM ahead of time, the VM must be in poweredOff state until you configure the user data for it. For example, if using the `govc` CLI: + +=== Creating VM for the first time + +[source,bash,subs="verbatim,attributes,quotes"] +---- +govc vm.create \ + -on=true \ + -c=2 \ + -m=4096 \ + -net.adapter= \ + -net= \ + -e guestinfo.userdata.encoding="base64" \ + -e guestinfo.userdata="$(base64 -w0 settings.toml)" \ + -template= \ + +---- + +=== Updating user data for an existing VM + +[source,bash,subs="verbatim,attributes,quotes"] +---- +govc vm.create \ + -on=false \ + -c=2 \ + -m=4096 \ + -net.adapter= \ + -net= \ + -template= \ + + +govc vm.change + -vm \ + -e guestinfo.userdata="$(base64 -w0 settings.toml)" \ + -e guestinfo.userdata.encoding="base64" + +govc vm.power -on +---- + +In the above sections, the `-e guestinfo.userdata.encoding="base64"` option specifies that the user data is base64-encoded. The `-e guestinfo.userdata` option passes the base64-encoded contents of the `settings.toml` file as user data to the Bottlerocket instance. Replace the placeholders with your specific values, such as the Bottlerocket OVA template and networking details. + +== Step 3: Verify the hybrid node connection + +After the Bottlerocket instance starts, it will attempt to join your Amazon EKS cluster. You can verify the connection in the Amazon EKS console by navigating to the Compute tab for your cluster or by running the following command: + +[source,bash,subs="verbatim,attributes,quotes"] +---- +kubectl get nodes +---- + +[IMPORTANT] +==== +Your nodes will have status `Not Ready`, which is expected and is due to the lack of a CNI running on your hybrid nodes. If your nodes did not join the cluster, see <>. +==== + +== Step 4: Configure a CNI for hybrid nodes + +To make your hybrid nodes ready to run applications, continue with the steps on <>. diff --git a/latest/ug/nodes/hybrid-nodes-cilium-bgp.adoc b/latest/ug/nodes/hybrid-nodes-cilium-bgp.adoc new file mode 100644 index 000000000..045c7ae98 --- /dev/null +++ b/latest/ug/nodes/hybrid-nodes-cilium-bgp.adoc @@ -0,0 +1,184 @@ +include::../attributes.txt[] + +[.topic] +[#hybrid-nodes-cilium-bgp] += Configure Cilium BGP for hybrid nodes +:info_titleabbrev: Configure BGP + +[abstract] +-- +Configure Cilium Border Gateway Protocol (BGP) for hybrid nodes +-- + +This topic describes how to configure Cilium Border Gateway Protocol (BGP) for Amazon EKS Hybrid Nodes. Cilium's BGP functionality is called link:https://docs.cilium.io/en/stable/network/bgp-control-plane/bgp-control-plane/[Cilium BGP Control Plane] and can be used to advertise pod and service addresses to your on-premises network. For alternative methods to make pod CIDRs routable on your on-premises network, see <>. + +[#hybrid-nodes-cilium-bgp-configure] +== Configure Cilium BGP + +=== Prerequisites + +- Cilium installed following the instructions in <>. + +=== Procedure + +. To use BGP with Cilium to advertise pod or service addresses with your on-premises network, Cilium must be installed with `bgpControlPlane.enabled: true`. If you are enabling BGP for an existing Cilium deployment, you must restart the Cilium operator to apply the BGP configuration if BGP was not previously enabled. You can set `operator.rollOutPods` to `true` in your Helm values to restart the Cilium operator as part of the Helm install/upgrade process. ++ +[source,yaml,subs="verbatim,attributes,quotes"] +---- +helm upgrade cilium oci://public.ecr.aws/eks/cilium/cilium \ + --namespace kube-system \ + --reuse-values \ + --set operator.rollOutPods=true \ + --set bgpControlPlane.enabled=true +---- ++ +. Confirm that the Cilium operator and agents were restarted and are running. ++ +[source,bash] +---- +kubectl -n kube-system get pods --selector=app.kubernetes.io/part-of=cilium +---- ++ +[source,bash] +---- +NAME READY STATUS RESTARTS AGE +cilium-grwlc 1/1 Running 0 4m12s +cilium-operator-68f7766967-5nnbl 1/1 Running 0 4m20s +cilium-operator-68f7766967-7spfz 1/1 Running 0 4m20s +cilium-pnxcv 1/1 Running 0 6m29s +cilium-r7qkj 1/1 Running 0 4m12s +cilium-wxhfn 1/1 Running 0 4m1s +cilium-z7hlb 1/1 Running 0 6m30s +---- + +. Create a file called `cilium-bgp-cluster.yaml` with a `CiliumBGPClusterConfig` definition. You may need to obtain the following information from your network administrator. ++ +- Configure `localASN` with the ASN for the nodes running Cilium. +- Configure `peerASN` with the ASN for your on-premises router. +- Configure the `peerAddress` with the on-premises router IP that each node running Cilium will peer with. ++ +[source,yaml,subs="verbatim,attributes,quotes"] +---- +apiVersion: cilium.io/v2alpha1 +kind: CiliumBGPClusterConfig +metadata: + name: cilium-bgp +spec: + nodeSelector: + matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: In + values: + - hybrid + bgpInstances: + - name: "rack0" + localASN: [.replaceable]`NODES_ASN` + peers: + - name: "onprem-router" + peerASN: [.replaceable]`ONPREM_ROUTER_ASN` + peerAddress: [.replaceable]`ONPREM_ROUTER_IP` + peerConfigRef: + name: "cilium-peer" +---- ++ +. Apply the Cilium BGP cluster configuration to your cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl apply -f cilium-bgp-cluster.yaml +---- ++ +. Create a file named `cilium-bgp-peer.yaml` with the `CiliumBGPPeerConfig` resource that defines a BGP peer configuration. Multiple peers can share the same configuration and provide reference to the common `CiliumBGPPeerConfig` resource. See the https://docs.cilium.io/en/latest/network/bgp-control-plane/bgp-control-plane-v2/#bgp-peer-configuration[BGP Peer configuration] in the Cilium documentation for a full list of configuration options. ++ +The values for the following Cilium peer settings must match those of the on-premises router you are peering with. ++ +- Configure `holdTimeSeconds` which determines how long a BGP peer waits for a keepalive or update message before declaring the session down. The default is 90 seconds. +- Configure `keepAliveTimeSeconds` which determines if a BGP peer is still reachable and the BGP session is active. The default is 30 seconds. +- Configure `restartTimeSeconds` which determines the time that Cilium's BGP control plane is expected to re-establish the BGP session after a restart. The default is 120 seconds. ++ +[source,yaml,subs="verbatim,attributes,quotes"] +---- +apiVersion: cilium.io/v2alpha1 +kind: CiliumBGPPeerConfig +metadata: + name: cilium-peer +spec: + timers: + holdTimeSeconds: [.replaceable]`90` + keepAliveTimeSeconds: [.replaceable]`30` + gracefulRestart: + enabled: true + restartTimeSeconds: [.replaceable]`120` + families: + - afi: ipv4 + safi: unicast + advertisements: + matchLabels: + advertise: "bgp" +---- + +. Apply the Cilium BGP peer configuration to your cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl apply -f cilium-bgp-peer.yaml +---- + +. Create a file named `cilium-bgp-advertisement-pods.yaml` with a `CiliumBGPAdvertisement` resource to advertise the pod CIDRs to your on-premises network. ++ +- The `CiliumBGPAdvertisement` resource is used to define advertisement types and attributes associated with them. The example below configures Cilium to advertise only pod CIDRs. See the examples in <> and <> for more information on configuring Cilium to advertise service addresses. +- Each hybrid node running the Cilium agent peers with the upstream BGP-enabled router. Each node advertises the pod CIDR range that it owns when Cilium’s `advertisementType` is set to `PodCIDR` like in the example below. See the link:https://docs.cilium.io/en/stable/network/bgp-control-plane/bgp-control-plane-v2/#bgp-advertisements[BGP Advertisements configuration] in the Cilium documentation for more information. ++ +[source,yaml,subs="verbatim,attributes"] +---- +apiVersion: cilium.io/v2alpha1 +kind: CiliumBGPAdvertisement +metadata: + name: bgp-advertisement-pods + labels: + advertise: bgp +spec: + advertisements: + - advertisementType: "PodCIDR" +---- + +. Apply the Cilium BGP Advertisement configuration to your cluster. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl apply -f cilium-bgp-advertisement-pods.yaml +---- ++ +. You can confirm the BGP peering worked with the https://docs.cilium.io/en/stable/gettingstarted/k8s-install-default/#install-the-cilium-cli[Cilium CLI] by using the `cilium bgp peers` command. You should see the correct values in the output for your environment and the Session State as `established`. See the https://docs.cilium.io/en/latest/network/bgp-control-plane/bgp-control-plane/#troubleshooting-and-operation-guide[Troubleshooting and Operations Guide] in the Cilium documentation for more information on troubleshooting. ++ +In the examples below, there are five hybrid nodes running the Cilium agent and each node is advertising the Pod CIDR range that it owns. ++ +[source,bash] +---- +cilium bgp peers +---- ++ +[source,bash,subs="verbatim,attributes,quotes"] +---- +Node Local AS Peer AS Peer Address Session State Uptime Family Received Advertised +mi-026d6a261e355fba7 [.replaceable]`NODES_ASN` [.replaceable]`ONPREM_ROUTER_ASN` [.replaceable]`ONPREM_ROUTER_IP` established 1h18m58s ipv4/unicast 1 2 +mi-082f73826a163626e [.replaceable]`NODES_ASN` [.replaceable]`ONPREM_ROUTER_ASN` [.replaceable]`ONPREM_ROUTER_IP` established 1h19m12s ipv4/unicast 1 2 +mi-09183e8a3d755abf6 [.replaceable]`NODES_ASN` [.replaceable]`ONPREM_ROUTER_ASN` [.replaceable]`ONPREM_ROUTER_IP` established 1h18m47s ipv4/unicast 1 2 +mi-0d78d815980ed202d [.replaceable]`NODES_ASN` [.replaceable]`ONPREM_ROUTER_ASN` [.replaceable]`ONPREM_ROUTER_IP` established 1h19m12s ipv4/unicast 1 2 +mi-0daa253999fe92daa [.replaceable]`NODES_ASN` [.replaceable]`ONPREM_ROUTER_ASN` [.replaceable]`ONPREM_ROUTER_IP` established 1h18m58s ipv4/unicast 1 2 +---- ++ +[source,bash] +---- +cilium bgp routes +---- ++ +[source,bash,subs="verbatim,attributes,quotes"] +---- +Node VRouter Prefix NextHop Age Attrs +mi-026d6a261e355fba7 [.replaceable]`NODES_ASN` 10.86.2.0/26 0.0.0.0 1h16m46s [{Origin: i} {Nexthop: 0.0.0.0}] +mi-082f73826a163626e [.replaceable]`NODES_ASN` 10.86.2.192/26 0.0.0.0 1h16m46s [{Origin: i} {Nexthop: 0.0.0.0}] +mi-09183e8a3d755abf6 [.replaceable]`NODES_ASN` 10.86.2.64/26 0.0.0.0 1h16m46s [{Origin: i} {Nexthop: 0.0.0.0}] +mi-0d78d815980ed202d [.replaceable]`NODES_ASN` 10.86.2.128/26 0.0.0.0 1h16m46s [{Origin: i} {Nexthop: 0.0.0.0}] +mi-0daa253999fe92daa [.replaceable]`NODES_ASN` 10.86.3.0/26 0.0.0.0 1h16m46s [{Origin: i} {Nexthop: 0.0.0.0}] +---- \ No newline at end of file diff --git a/latest/ug/nodes/hybrid-nodes-cluster-create.adoc b/latest/ug/nodes/hybrid-nodes-cluster-create.adoc index 1549c5e17..e3e742c93 100644 --- a/latest/ug/nodes/hybrid-nodes-cluster-create.adoc +++ b/latest/ug/nodes/hybrid-nodes-cluster-create.adoc @@ -1,24 +1,21 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[hybrid-nodes-cluster-create,hybrid-nodes-cluster-create.title]] +[#hybrid-nodes-cluster-create] = Create an Amazon EKS cluster with hybrid nodes -:info_doctype: section -:info_title: Create an Amazon EKS cluster with hybrid nodes :info_titleabbrev: Create cluster -:keywords: on-premises, hybrid -:info_abstract: Create an Amazon EKS cluster with hybrid nodes - -include::../attributes.txt[] [abstract] -- Create hybrid nodes cluster -- -This topic provides an overview of the available options and describes what to consider when you create a hybrid nodes-enabled Amazon EKS cluster. If you are not planning to use hybrid nodes, see <>. +This topic provides an overview of the available options and describes what to consider when you create a hybrid nodes-enabled Amazon EKS cluster. EKS Hybrid Nodes have the same link:eks/latest/userguide/kubernetes-versions.html[Kubernetes version support,type="documentation"] as Amazon EKS clusters with cloud nodes, including standard and extended support. + +If you are not planning to use EKS Hybrid Nodes, see the primary Amazon EKS create cluster documentation at <>. -[[hybrid-nodes-cluster-create-prep,hybrid-nodes-cluster-create-prep.title]] +[#hybrid-nodes-cluster-create-prep] == Prerequisites * The <> completed. Before you create your hybrid nodes-enabled cluster, you must have your on-premises node and optionally pod CIDRs identified, your VPC and subnets created according to the EKS requirements, and hybrid nodes requirements, and your security group with inbound rules for your on-premises and optionally pod CIDRs. For more information on these prerequisites, see <>. @@ -26,43 +23,31 @@ This topic provides an overview of the available options and describes what to c * An link:IAM/latest/UserGuide/id_roles#iam-term-principal[IAM principal,type="documentation"] with permissions to create IAM roles and attach policies, and create and describe EKS clusters -[[hybrid-nodes-cluster-create-consider,hybrid-nodes-cluster-create-consider.title]] +[#hybrid-nodes-cluster-create-consider] == Considerations * Your cluster must use either `API` or `API_AND_CONFIG_MAP` for the cluster authentication mode. * Your cluster must use IPv4 address family. * Your cluster must use either Public or Private cluster endpoint connectivity. Your cluster cannot use “Public and Private” cluster endpoint connectivity, because the Amazon EKS Kubernetes API server endpoint will resolve to the public IPs for hybrid nodes running outside of your VPC. -* Currently, hybrid nodes must be enabled during cluster creation. You cannot change your `RemoteNodeNetwork` or `RemotePodNetwork` after cluster creation. +* OIDC authentication is supported for EKS clusters with hybrid nodes. +* You can add, change, or remove the hybrid nodes configuration of an existing cluster. For more information, see <>. -[[hybrid-nodes-cluster-create-iam,hybrid-nodes-cluster-create-iam.title]] +[#hybrid-nodes-cluster-create-iam] == Step 1: Create cluster IAM role If you already have a cluster IAM role, or you're going to create your cluster with `eksctl` or {aws} CloudFormation, then you can skip this step. By default, `eksctl` and the {aws} CloudFormation template create the cluster IAM role for you. . Run the following command to create an IAM trust policy JSON file. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- -cat >eks-cluster-role-trust-policy.json <>. Attach the Amazon EKS managed policy named `AmazonEKSClusterPolicy` to the role. To attach an IAM policy to an link:IAM/latest/UserGuide/id_roles#iam-term-principal[IAM principal,type="documentation"], the principal that is attaching the policy must be assigned one of the following IAM actions (permissions): `iam:AttachUserPolicy` or `iam:AttachRolePolicy`. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- aws iam attach-role-policy \ - --policy-arn arn:aws:iam::aws:policy/AmazonEKSClusterPolicy \ + --policy-arn {arn-aws}iam::aws:policy/AmazonEKSClusterPolicy \ --role-name myAmazonEKSClusterRole ---- -[[hybrid-nodes-cluster-create-cluster,hybrid-nodes-cluster-create-cluster.title]] +[#hybrid-nodes-cluster-create-cluster] == Step 2: Create hybrid nodes-enabled cluster You can create a cluster by using: * <> * <> -* <> +* <> * <> -[[hybrid-nodes-cluster-create-eksctl,hybrid-nodes-cluster-create-eksctl.title]] +[#hybrid-nodes-cluster-create-eksctl] === Create hybrid nodes-enabled cluster - eksctl You need to install the latest version of the `eksctl` command line tool. To install or update `eksctl`, see https://eksctl.io/installation[Installation] in the `eksctl` documentation. @@ -98,17 +83,17 @@ You need to install the latest version of the `eksctl` command line tool. To ins . Create `cluster-config.yaml` to define a hybrid nodes-enabled Amazon EKS IPv4 cluster. Make the following replacements in your `cluster-config.yaml`. For a full list of settings, see the https://eksctl.io/getting-started/[eksctl documentation]. .. Replace `CLUSTER_NAME` with a name for your cluster. The name can contain only alphanumeric characters (case-sensitive) and hyphens. It must start with an alphanumeric character and can't be longer than 100 characters. The name must be unique within the {aws} Region and {aws} account that you're creating the cluster in. .. Replace `AWS_REGION` with the {aws} Region that you want to create your cluster in. -.. Replace `K8S_VERSION` with any <>. +.. Replace `K8S_VERSION` with any link:eks/latest/userguide/kubernetes-versions.html[Amazon EKS supported version,type="documentation"]. .. Replace `CREDS_PROVIDER` with `ssm` or `ira` based on the credential provider you configured in the steps for <>. .. Replace `CA_BUNDLE_CERT` if your credential provider is set to `ira`, which uses {aws} IAM Roles Anywhere as the credential provider. The CA_BUNDLE_CERT is the certificate authority (CA) certificate body and depends on your choice of CA. The certificate must be in Privacy Enhanced Mail (PEM) format. .. Replace `GATEWAY_ID` with the ID of your virtual private gateway or transit gateway to be attached to your VPC. .. Replace `REMOTE_NODE_CIDRS` with the on-premises node CIDR for your hybrid nodes. -.. Replace `REMOTE_POD_CIDRS` with the on-premises pod CIDR for workloads running on hybrid nodes or remove the line from your configuration if you are not running webhooks on hybrid nodes. You must configure your `REMOTE_POD_CIDRS` if your CNI does not use Network Address Translation (NAT) or masquerading for pod IP addresses when pod traffic leaves your on-premises hosts. You must configure `REMOTE_POD_CIDRS` if you are running webhooks on hybrid nodes. +.. Replace `REMOTE_POD_CIDRS` with the on-premises pod CIDR for workloads running on hybrid nodes or remove the line from your configuration if you are not running webhooks on hybrid nodes. You must configure your `REMOTE_POD_CIDRS` if your CNI does not use Network Address Translation (NAT) or masquerading for pod IP addresses when pod traffic leaves your on-premises hosts. You must configure `REMOTE_POD_CIDRS` if you are running webhooks on hybrid nodes, see <> for more information. .. Your on-premises node and pod CIDR blocks must meet the following requirements: ... Be within one of the IPv4 RFC-1918 ranges: `10.0.0.0/8`, `172.16.0.0/12`, or `192.168.0.0/16`. ... Not overlap with each other, the `VPC CIDR` for your cluster, or your Kubernetes service IPv4 CIDR + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- apiVersion: eksctl.io/v1alpha5 kind: ClusterConfig @@ -122,7 +107,7 @@ remoteNetworkConfig: iam: provider: CREDS_PROVIDER # default SSM, can also be set to IRA # caBundleCert: CA_BUNDLE_CERT - vpcGatewayID: GATEWAY_ID + vpcGatewayID: GATEWAY_ID remoteNodeNetworks: - cidrs: ["REMOTE_NODE_CIDRS"] remotePodNetworks: @@ -131,14 +116,14 @@ remoteNetworkConfig: . Run the following command: + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- eksctl create cluster -f cluster-config.yaml ---- + Cluster provisioning takes several minutes. While the cluster is being created, several lines of output appear. The last line of output is similar to the following example line. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- [✓] EKS cluster "CLUSTER_NAME" in "REGION" region is ready ---- @@ -146,14 +131,14 @@ Cluster provisioning takes several minutes. While the cluster is being created, . Continue with <>. -[[hybrid-nodes-cluster-create-cfn,hybrid-nodes-cluster-create-cfn.title]] +[#hybrid-nodes-cluster-create-cfn] === Create hybrid nodes-enabled cluster - {aws} CloudFormation The CloudFormation stack creates the EKS cluster IAM role and an EKS cluster with the `RemoteNodeNetwork` and `RemotePodNetwork` you specify. Modify the CloudFormation template If you need to customize settings for your EKS cluster that are not exposed in the CloudFormation template. . Download the CloudFormation template. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- curl -OL 'https://raw.githubusercontent.com/aws/eks-hybrid/refs/heads/main/example/hybrid-eks-cfn.yaml' ---- @@ -165,15 +150,15 @@ curl -OL 'https://raw.githubusercontent.com/aws/eks-hybrid/refs/heads/main/examp .. `SUBNET2_ID`: the ID of the second subnet you created in the prerequisite steps .. `SG_ID`: the security group ID you created in the prerequisite steps .. `REMOTE_NODE_CIDRS`: the on-premises node CIDR for your hybrid nodes -.. `REMOTE_POD_CIDRS`: the on-premises pod CIDR for workloads running on hybrid nodes. You must configure your `REMOTE_POD_CIDRS` if your CNI does not use Network Address Translation (NAT) or masquerading for pod IP addresses when pod traffic leaves your on-premises hosts. You must configure `REMOTE_POD_CIDRS` if you are running webhooks on hybrid nodes. +.. `REMOTE_POD_CIDRS`: the on-premises pod CIDR for workloads running on hybrid nodes. You must configure your `REMOTE_POD_CIDRS` if your CNI does not use Network Address Translation (NAT) or masquerading for pod IP addresses when pod traffic leaves your on-premises hosts. You must configure `REMOTE_POD_CIDRS` if you are running webhooks on hybrid nodes, see <> for more information. .. Your on-premises node and pod CIDR blocks must meet the following requirements: ... Be within one of the IPv4 RFC-1918 ranges: `10.0.0.0/8`, `172.16.0.0/12`, or `192.168.0.0/16`. ... Not overlap with each other, the `VPC CIDR` for your cluster, or your Kubernetes service IPv4 CIDR. .. `CLUSTER_AUTH`: the cluster authentication mode for your cluster. Valid values are `API` and `API_AND_CONFIG_MAP`. The default in the template is `API_AND_CONFIG_MAP`. .. `CLUSTER_ENDPOINT`: the cluster endpoint connectivity for your cluster. Valid values are “Public” and “Private”. The default in the template is Private, which means you will only be able to connect to the Kubernetes API endpoint from within your VPC. -.. `K8S_VERSION`: the Kubernetes version to use for your cluster. See <>. +.. `K8S_VERSION`: the Kubernetes version to use for your cluster. See link:eks/latest/userguide/kubernetes-versions.html[Amazon EKS supported versions,type="documentation"]. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,json,subs="verbatim,attributes"] ---- { "Parameters": { @@ -191,9 +176,9 @@ curl -OL 'https://raw.githubusercontent.com/aws/eks-hybrid/refs/heads/main/examp } ---- -. Deploy the CloudFormation stack. Replace `STACK_NAME` with your name for the CloudFormation stack and AWS_REGION with your desired {aws} Region where the cluster will be created. +. Deploy the CloudFormation stack. Replace `STACK_NAME` with your name for the CloudFormation stack and `AWS_REGION` with your {aws} Region where the cluster will be created. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- aws cloudformation deploy \ --stack-name STACK_NAME \ @@ -203,9 +188,9 @@ aws cloudformation deploy \ --capabilities CAPABILITY_NAMED_IAM ---- + -Cluster provisioning takes several minutes. You can check the status of your stack with the following command. Replace `STACK_NAME` with your name for the CloudFormation stack and `AWS_REGION` with your desired {aws} Region where the cluster will be created. +Cluster provisioning takes several minutes. You can check the status of your stack with the following command. Replace `STACK_NAME` with your name for the CloudFormation stack and `AWS_REGION` with your {aws} Region where the cluster will be created. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- aws cloudformation describe-stacks \ --stack-name STACK_NAME \ @@ -216,12 +201,12 @@ aws cloudformation describe-stacks \ . Continue with <>. -[[hybrid-nodes-cluster-create-cli,hybrid-nodes-cluster-create-cli.title]] +[#hybrid-nodes-cluster-create-cli] === Create hybrid nodes-enabled cluster - {aws} CLI -. Run the following command to create a hybrid nodes-enabled EKS cluster. Before running the command, replace the following with your desired settings. For a full list of settings, see the <> documentation. +. Run the following command to create a hybrid nodes-enabled EKS cluster. Before running the command, replace the following with your settings. For a full list of settings, see the <> documentation. .. `CLUSTER_NAME`: name of the EKS cluster to be created -.. `AWS_REGION`: {aws} Region where the cluster will be created. +.. `AWS_REGION`: {aws} Region where the cluster will be created. .. `K8S_VERSION`: the Kubernetes version to use for your cluster. See Amazon EKS supported versions. .. `ROLE_ARN`: the Amazon EKS cluster role you configured for your cluster. See Amazon EKS cluster IAM role for more information. .. `SUBNET1_ID`: the ID of the first subnet you created in the prerequisite steps @@ -229,27 +214,27 @@ aws cloudformation describe-stacks \ .. `SG_ID`: the security group ID you created in the prerequisite steps .. You can use `API` and `API_AND_CONFIG_MAP` for your cluster access authentication mode. In the command below, the cluster access authentication mode is set to `API_AND_CONFIG_MAP`. .. You can use the `endpointPublicAccess` and `endpointPrivateAccess` parameters to enable or disable public and private access to your cluster's Kubernetes API server endpoint. In the command below `endpointPublicAccess` is set to false and `endpointPrivateAccess` is set to true. -.. `REMOTE_NODE_CIDRS`: the on-premises node CIDR for your hybrid nodes. +.. `REMOTE_NODE_CIDRS`: the on-premises node CIDR for your hybrid nodes. .. `REMOTE_POD_CIDRS` (optional): the on-premises pod CIDR for workloads running on hybrid nodes. .. Your on-premises node and pod CIDR blocks must meet the following requirements: ... Be within one of the IPv4 RFC-1918 ranges: `10.0.0.0/8`, `172.16.0.0/12`, or `192.168.0.0/16`. ... Not overlap with each other, the `VPC CIDR` for your Amazon EKS cluster, or your Kubernetes service IPv4 CIDR. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- aws eks create-cluster \ --name CLUSTER_NAME \ --region AWS_REGION \ --kubernetes-version K8S_VERSION \ - --role-arn ROLE_ARN \ + --role-arn ROLE_ARN \ --resources-vpc-config subnetIds=SUBNET1_ID,SUBNET2_ID,securityGroupIds=SG_ID,endpointPrivateAccess=true,endpointPublicAccess=false \ --access-config authenticationMode=API_AND_CONFIG_MAP \ - --remote-network-config '{"remoteNodeNetworks":[{"cidrs":["REMOTE_NODE_CIDRS"]}],"remotePodNetworks":[{"cidrs":["REMOTE_POD_CIDRS"]}]}' + --remote-network-config '{"remoteNodeNetworks":[{"cidrs":["REMOTE_NODE_CIDRS"]}],"remotePodNetworks":[{"cidrs":["REMOTE_POD_CIDRS"]}]}' ---- . It takes several minutes to provision the cluster. You can query the status of your cluster with the following command. Replace `CLUSTER_NAME` with the name of the cluster you are creating and `AWS_REGION` with the {aws} Region where the cluster is creating. Don't proceed to the next step until the output returned is `ACTIVE`. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- aws eks describe-cluster \ --name CLUSTER_NAME \ @@ -259,8 +244,8 @@ aws eks describe-cluster \ . Continue with <>. -[[hybrid-nodes-cluster-create-console,hybrid-nodes-cluster-create-console.title]] -=== Create hybrid nodes-enabled cluster - {aws} Management Console +[#hybrid-nodes-cluster-create-console] +=== Create hybrid nodes-enabled cluster - {aws-management-console} . Open the Amazon EKS console at link:eks/home#/clusters[Amazon EKS console,type="console"]. . Choose Add cluster and then choose Create. @@ -275,7 +260,7 @@ aws eks describe-cluster \ ... *EKS API*: The cluster will source authenticated IAM principals only from EKS access entry APIs. ... *EKS API and ConfigMap*: The cluster will source authenticated IAM principals from both EKS access entry APIs and the `aws-auth` ConfigMap. .. *Secrets encryption* – (Optional) Choose to enable secrets encryption of Kubernetes secrets using a KMS key. You can also enable this after you create your cluster. Before you enable this capability, make sure that you're familiar with the information in <>. -.. *ARC Zonal Shift* - If enabled, EKS will register your cluster with ARC zonal shift to enable you to use zonal shift to shift application traffic away from an AZ. +.. *ARC zonal shift* - If enabled, EKS will register your cluster with ARC zonal shift to enable you to use zonal shift to shift application traffic away from an AZ. .. *Tags* – (Optional) Add any tags to your cluster. For more information, see <>. .. When you're done with this page, choose *Next*. . On the *Specify networking* page, select values for the following fields: @@ -299,39 +284,39 @@ aws eks describe-cluster \ .. You can choose as many *Amazon EKS add-ons* and *{aws} Marketplace add-ons* as you require. Amazon EKS add-ons that are not compatible with hybrid nodes are marked with “Not compatible with Hybrid Nodes” and the add-ons have an anti-affinity rule to prevent them from running on hybrid nodes. See Configuring add-ons for hybrid nodes for more information. If the *{aws} Marketplace* add-ons that you want to install isn't listed, you can search for available *{aws} Marketplace add-ons* by entering text in the search box. You can also search by *category*, *vendor*, or *pricing model* and then choose the add-ons from the search results. .. Some add-ons, such as CoreDNS and kube-proxy, are installed by default. If you disable any of the default add-ons, this may affect your ability to run Kubernetes applications. .. When you're done with this page, choose `Next`. -. On the *Configure selected add-ons settings* page, select the version that you want to install. -.. You can always update to a later version after cluster creation. You can update the configuration of each add-on after cluster creation. For more information about configuring add-ons, see <>. For the add-ons versions that are compatible with hybrid nodes, see <>. +. On the *Configure selected add-ons settings* page, select the version that you want to install. +.. You can always update to a later version after cluster creation. You can update the configuration of each add-on after cluster creation. For more information about configuring add-ons, see <>. For the add-ons versions that are compatible with hybrid nodes, see <>. .. When you're done with this page, choose Next. . On the *Review and create* page, review the information that you entered or selected on the previous pages. If you need to make changes, choose *Edit*. When you're satisfied, choose *Create*. The *Status* field shows *CREATING* while the cluster is provisioned. Cluster provisioning takes several minutes. . Continue with <>. -[[hybrid-nodes-cluster-create-kubeconfig,hybrid-nodes-cluster-create-kubeconfig.title]] +[#hybrid-nodes-cluster-create-kubeconfig] == Step 3: Update kubeconfig If you created your cluster using `eksctl`, then you can skip this step. This is because `eksctl` already completed this step for you. Enable `kubectl` to communicate with your cluster by adding a new context to the `kubectl` config file. For more information about how to create and update the file, see <>. -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- aws eks update-kubeconfig --name CLUSTER_NAME --region AWS_REGION ---- An example output is as follows. -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- -Added new context arn:aws:eks:AWS_REGION:111122223333:cluster/CLUSTER_NAME to /home/username/.kube/config +Added new context {arn-aws}eks:AWS_REGION:111122223333:cluster/CLUSTER_NAME to /home/username/.kube/config ---- Confirm communication with your cluster by running the following command. -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- kubectl get svc ---- An example output is as follows. -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE kubernetes ClusterIP 10.100.0.1 443/TCP 28h diff --git a/latest/ug/nodes/hybrid-nodes-cluster-prep.adoc b/latest/ug/nodes/hybrid-nodes-cluster-prep.adoc index 793c196c0..cd7b5952c 100644 --- a/latest/ug/nodes/hybrid-nodes-cluster-prep.adoc +++ b/latest/ug/nodes/hybrid-nodes-cluster-prep.adoc @@ -1,18 +1,15 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[hybrid-nodes-cluster-prep,hybrid-nodes-cluster-prep.title]] +[#hybrid-nodes-cluster-prep] = Prepare cluster access for hybrid nodes -:info_title: Prepare cluster access for hybrid nodes :info_titleabbrev: Prepare cluster access -:info_abstract: Prepare cluster access for hybrid nodes [abstract] -- Prepare cluster access for Amazon EKS hybrid nodes -- -include::../attributes.txt[] - Before connecting hybrid nodes to your Amazon EKS cluster, you must enable your Hybrid Nodes IAM Role with Kubernetes permissions to join the cluster. See <> for information on how to create the Hybrid Nodes IAM role. Amazon EKS supports two ways to associate IAM principals with Kubernetes Role-Based Access Control (RBAC), Amazon EKS access entries and the `aws-auth` ConfigMap. For more information on Amazon EKS access management, see <>. Use the procedures below to associate your Hybrid Nodes IAM role with Kubernetes permissions. To use Amazon EKS access entries, your cluster must have been created with the `API` or `API_AND_CONFIG_MAP` authentication modes. To use the `aws-auth` ConfigMap, your cluster must have been created with the `API_AND_CONFIG_MAP` authentication mode. The `CONFIG_MAP`-only authentication mode is not supported for hybrid nodes-enabled Amazon EKS clusters. @@ -27,14 +24,15 @@ There is an Amazon EKS access entry type for hybrid nodes named HYBRID_LINUX tha . Create your access entry with the following command. Replace CLUSTER_NAME with the name of your cluster and HYBRID_NODES_ROLE_ARN with the ARN of the role you created in the steps for <>. + -[source,shell,subs="verbatim,attributes,quotes"] +[source,shell,subs="verbatim,attributes"] ---- aws eks create-access-entry --cluster-name CLUSTER_NAME \ --principal-arn HYBRID_NODES_ROLE_ARN \ --type HYBRID_LINUX ---- -=== {aws} Management Console +[#hybrid-nodes-cluster-prep-console] +=== {aws-management-console} . Open the Amazon EKS console at link:eks/home#/clusters[Amazon EKS console,type="console"]. @@ -61,7 +59,7 @@ In the following steps, you will create or update the `aws-auth` ConfigMap with . Check to see if you have an existing `aws-auth` ConfigMap for your cluster. Note that if you are using a specific `kubeconfig` file, use the `--kubeconfig` flag. + -[source,shell,subs="verbatim,attributes,quotes"] +[source,shell,subs="verbatim,attributes"] ---- kubectl describe configmap -n kube-system aws-auth ---- @@ -70,14 +68,14 @@ kubectl describe configmap -n kube-system aws-auth + .. Open the ConfigMap for editing. + -[source,shell,subs="verbatim,attributes,quotes"] +[source,shell,subs="verbatim,attributes"] ---- kubectl edit -n kube-system configmap/aws-auth ---- .. Add a new `mapRoles` entry as needed. Replace `HYBRID_NODES_ROLE_ARN` with the ARN of your Hybrid Nodes IAM role. Note, `{{SessionName}}` is the correct template format to save in the ConfigMap. Do not replace it with other values. + -[source,yaml,subs="verbatim,attributes,quotes"] +[source,yaml,subs="verbatim,attributes"] ---- data: mapRoles: | @@ -92,7 +90,7 @@ data: . If there is not an existing `aws-auth` ConfigMap for your cluster, create it with the following command. Replace `HYBRID_NODES_ROLE_ARN` with the ARN of your Hybrid Nodes IAM role. Note that `{{SessionName}}` is the correct template format to save in the ConfigMap. Do not replace it with other values. + -[source,shell,subs="verbatim,attributes,quotes"] +[source,shell,subs="verbatim,attributes"] ---- kubectl apply -f=/dev/stdin <<-EOF apiVersion: v1 @@ -108,4 +106,4 @@ data: rolearn: HYBRID_NODES_ROLE_ARN username: system:node:{{SessionName}} EOF ----- +---- \ No newline at end of file diff --git a/latest/ug/nodes/hybrid-nodes-cluster-update.adoc b/latest/ug/nodes/hybrid-nodes-cluster-update.adoc new file mode 100644 index 000000000..c95a801ea --- /dev/null +++ b/latest/ug/nodes/hybrid-nodes-cluster-update.adoc @@ -0,0 +1,236 @@ +include::../attributes.txt[] + +[.topic] +[#hybrid-nodes-cluster-update] += Enable hybrid nodes on an existing Amazon EKS cluster or modify configuration +:info_titleabbrev: Existing cluster + +[abstract] +-- +Modify hybrid nodes configuration on an existing cluster +-- + +This topic provides an overview of the available options and describes what to consider when you add, change, or remove the hybrid nodes configuration for an Amazon EKS cluster. + +To enable an Amazon EKS cluster to use hybrid nodes, add the IP address CIDR ranges of your on-premises node and optionally pod network in the `RemoteNetworkConfig` configuration. EKS uses this list of CIDRs to enable connectivity between the cluster and your on-premises networks. For a full list of options when updating your cluster configuration, see the link:eks/latest/APIReference/API_UpdateClusterConfig.html[UpdateClusterConfig,type="documentation"] in the _Amazon EKS API Reference_. + +You can do any of the following actions to the EKS Hybrid Nodes networking configuration in a cluster: + +* <> +* <> +* <> + +[#hybrid-nodes-cluster-enable-prep] +== Prerequisites + +* Before enabling your Amazon EKS cluster for hybrid nodes, ensure your environment meets the requirements outlined at <>, and detailed at <>, <>, and <>. +* Your cluster must use IPv4 address family. +* Your cluster must use either `API` or `API_AND_CONFIG_MAP` for the cluster authentication mode. The process for modifying the cluster authentication mode is described at <>. +* We recommend that you use either public or private endpoint access for the Amazon EKS Kubernetes API server endpoint, but not both. If you choose “Public and Private”, the Amazon EKS Kubernetes API server endpoint will always resolve to the public IPs for hybrid nodes running outside of your VPC, which can prevent your hybrid nodes from joining the cluster. The process for modifying network access to your cluster is described at <>. +* The latest version of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device. To check your current version, use `aws --version`. Package managers such yum, apt-get, or Homebrew for macOS are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/getting-started-install.html[Installing or updating to the latest version of the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Configuring settings for the {aws} CLI,type="documentation"] in the {aws} Command Line Interface User Guide. +* An link:IAM/latest/UserGuide/id_roles#iam-term-principal[IAM principal,type="documentation"] with permission to call link:eks/latest/APIReference/API_UpdateClusterConfig.html[UpdateClusterConfig,type="documentation"] on your Amazon EKS cluster. +* Update add-ons to versions that are compatible with hybrid nodes. For the add-ons versions that are compatible with hybrid nodes, see <>. +* If you are running add-ons that are not compatible with hybrid nodes, ensure that the add-on https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/[DaemonSet] or https://kubernetes.io/docs/concepts/workloads/controllers/deployment/[Deployment] has the following affinity rule to prevent deployment to hybrid nodes. Add the following affinity rule if it is not already present. ++ +[source,yaml] +---- +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - hybrid +---- + + + +[#hybrid-nodes-cluster-enable-consider] +== Considerations +The `remoteNetworkConfig` JSON object has the following behavior during an update: + +* Any existing part of the configuration that you don't specify is unchanged. If you don't specify either of the `remoteNodeNetworks` or `remotePodNetworks`, that part will remain the same. +* If you are modifying either the `remoteNodeNetworks` or `remotePodNetworks` lists of CIDRs, you must specify the complete list of CIDRs that you want in your final configuration. When you specify a change to either the `remoteNodeNetworks` or `remotePodNetworks` CIDR list, EKS replaces the original list during the update. +* Your on-premises node and pod CIDR blocks must meet the following requirements: +. Be within one of the IPv4 RFC-1918 ranges: 10.0.0.0/8, 172.16.0.0/12, or 192.168.0.0/16. +. Not overlap with each other, all CIDRs of the VPC for your Amazon EKS cluster, or your Kubernetes service IPv4 CIDR. + + +[#hybrid-nodes-cluster-enable-existing] +== Enable hybrid nodes on an existing cluster + +You can enable EKS Hybrid Nodes in an existing cluster by using: + +* <> +* <> +* <> + + +[#hybrid-nodes-cluster-enable-cfn] +=== Enable EKS Hybrid Nodes in an existing cluster - {aws} CloudFormation + +. To enable EKS Hybrid Nodes in your cluster, add the `RemoteNodeNetwork` and (optional) `RemotePodNetwork` to your CloudFormation template and update the stack. Note that `RemoteNodeNetwork` is a list with a maximum of one `Cidrs` item and the `Cidrs` is a list of multiple IP CIDR ranges. ++ +[source,yaml,subs="verbatim,attributes"] +---- +RemoteNetworkConfig: + RemoteNodeNetworks: + - Cidrs: [RemoteNodeCIDR] + RemotePodNetworks: + - Cidrs: [RemotePodCIDR] +---- + +. Continue to <>. + +[#hybrid-nodes-cluster-enable-cli] +=== Enable EKS Hybrid Nodes in an existing cluster - {aws} CLI + +. Run the following command to enable `RemoteNetworkConfig` for EKS Hybrid Nodes for your EKS cluster. Before running the command, replace the following with your settings. For a full list of settings, see the link:eks/latest/APIReference/API_UpdateClusterConfig.html[UpdateClusterConfig,type="documentation"] in the _Amazon EKS API Reference_. +.. `CLUSTER_NAME`: name of the EKS cluster to update. +.. `AWS_REGION`: {aws} Region where the EKS cluster is running. +.. `REMOTE_NODE_CIDRS`: the on-premises node CIDR for your hybrid nodes. +.. `REMOTE_POD_CIDRS` (optional): the on-premises pod CIDR for workloads running on hybrid nodes. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks update-cluster-config \ + --name CLUSTER_NAME \ + --region AWS_REGION \ + --remote-network-config '{"remoteNodeNetworks":[{"cidrs":["REMOTE_NODE_CIDRS"]}],"remotePodNetworks":[{"cidrs":["REMOTE_POD_CIDRS"]}]}' +---- + +. It takes several minutes to update the cluster. You can query the status of your cluster with the following command. Replace `CLUSTER_NAME` with the name of the cluster you are modifying and `AWS_REGION` with the {aws} Region where the cluster is running. Don't proceed to the next step until the output returned is `ACTIVE`. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks describe-cluster \ + --name CLUSTER_NAME \ + --region AWS_REGION \ + --query "cluster.status" +---- + +. Continue to <>. + + +[#hybrid-nodes-cluster-enable-console] +=== Enable EKS Hybrid Nodes in an existing cluster - {aws-management-console} + +. Open the Amazon EKS console at link:eks/home#/clusters[Amazon EKS console,type="console"]. +. Choose the name of the cluster to display your cluster information. +. Choose the *Networking* tab and choose *Manage*. +. In the dropdown, choose *Remote networks*. +. *Choose Configure remote networks to enable hybrid nodes* and specify your on-premises node and pod CIDRs for hybrid nodes. +. Choose *Save changes* to finish. Wait for the cluster status to return to *Active*. + +. Continue to <>. + +[#hybrid-nodes-cluster-update-config] +== Update hybrid nodes configuration in an existing cluster + +You can modify `remoteNetworkConfig` in an existing hybrid cluster by using any of the following: + +* <> +* <> +* <> + +[#hybrid-nodes-cluster-update-cfn] +=== Update hybrid configuration in an existing cluster - {aws} CloudFormation + +. Update your CloudFormation template with the new network CIDR values. ++ +[source,yaml,subs="verbatim,attributes"] +---- +RemoteNetworkConfig: + RemoteNodeNetworks: + - Cidrs: [NEW_REMOTE_NODE_CIDRS] + RemotePodNetworks: + - Cidrs: [NEW_REMOTE_POD_CIDRS] +---- +NOTE: When updating `RemoteNodeNetworks` or `RemotePodNetworks` CIDR lists, include all CIDRs (new and existing). EKS replaces the entire list during updates. +Omitting these fields from the update request retains their existing configurations. + +. Update your CloudFormation stack with the modified template and wait for the stack update to complete. + +[#hybrid-nodes-cluster-update-cli] +=== Update hybrid configuration in an existing cluster - {aws} CLI + +. To modify the remote network CIDRs, run the following command. Replace the values with your settings: ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks update-cluster-config +--name CLUSTER_NAME +--region AWS_REGION +--remote-network-config '{"remoteNodeNetworks":[{"cidrs":["NEW_REMOTE_NODE_CIDRS"]}],"remotePodNetworks":[{"cidrs":["NEW_REMOTE_POD_CIDRS"]}]}' +---- +NOTE: When updating `remoteNodeNetworks` or `remotePodNetworks` CIDR lists, include all CIDRs (new and existing). EKS replaces the entire list during updates. +Omitting these fields from the update request retains their existing configurations. + +. Wait for the cluster status to return to ACTIVE before proceeding. + +[#hybrid-nodes-cluster-update-console] +=== Update hybrid configuration in an existing cluster - {aws-management-console} + +. Open the Amazon EKS console at link:eks/home#/clusters[Amazon EKS console,type="console"]. +. Choose the name of the cluster to display your cluster information. +. Choose the *Networking* tab and choose *Manage*. +. In the dropdown, choose *Remote networks*. +. Update the CIDRs under `Remote node networks` and `Remote pod networks - Optional` as needed. +. Choose *Save changes* and wait for the cluster status to return to *Active*. + + +[#hybrid-nodes-cluster-disable] +== Disable Hybrid nodes in an existing cluster + +You can disable EKS Hybrid Nodes in an existing cluster by using: + +* <> +* <> +* <> + +[#hybrid-nodes-cluster-disable-cfn] +=== Disable EKS Hybrid Nodes in an existing cluster - {aws} CloudFormation + +. To disable EKS Hybrid Nodes in your cluster, set `RemoteNodeNetworks` and `RemotePodNetworks` to empty arrays in your CloudFormation template and update the stack. ++ +[source,yaml,subs="verbatim,attributes"] +---- +RemoteNetworkConfig: + RemoteNodeNetworks: [] + RemotePodNetworks: [] +---- + +[#hybrid-nodes-cluster-disable-cli] +=== Disable EKS Hybrid Nodes in an existing cluster - {aws} CLI +. Run the following command to remove `RemoteNetworkConfig` from your EKS cluster. Before running the command, replace the following with your settings. For a full list of settings, see the link:eks/latest/APIReference/API_UpdateClusterConfig.html[UpdateClusterConfig,type="documentation"] in the _Amazon EKS API Reference_. +.. `CLUSTER_NAME`: name of the EKS cluster to update. +.. `AWS_REGION`: {aws} Region where the EKS cluster is running. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks update-cluster-config \ + --name CLUSTER_NAME \ + --region AWS_REGION \ + --remote-network-config '{"remoteNodeNetworks":[],"remotePodNetworks":[]}' +---- +. It takes several minutes to update the cluster. You can query the status of your cluster with the following command. Replace `CLUSTER_NAME` with the name of the cluster you are modifying and `AWS_REGION` with the {aws} Region where the cluster is running. Don't proceed to the next step until the output returned is `ACTIVE`. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks describe-cluster \ + --name CLUSTER_NAME \ + --region AWS_REGION \ + --query "cluster.status" +---- + +[#hybrid-nodes-cluster-disable-console] +=== Disable EKS Hybrid Nodes in an existing cluster - {aws-management-console} + +. Open the Amazon EKS console at link:eks/home#/clusters[Amazon EKS console,type="console"]. +. Choose the name of the cluster to display your cluster information. +. Choose the *Networking* tab and choose *Manage*. +. In the dropdown, choose *Remote networks*. +. Choose *Configure remote networks to enable hybrid nodes* and remove all the CIDRs under `Remote node networks` and `Remote pod networks - Optional`. +. Choose *Save changes* to finish. Wait for the cluster status to return to *Active*. diff --git a/latest/ug/nodes/hybrid-nodes-cni.adoc b/latest/ug/nodes/hybrid-nodes-cni.adoc index 465a94f16..20dd19f2e 100644 --- a/latest/ug/nodes/hybrid-nodes-cni.adoc +++ b/latest/ug/nodes/hybrid-nodes-cni.adoc @@ -1,120 +1,114 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[hybrid-nodes-cni,hybrid-nodes-cni.title]] -= Configure a CNI for hybrid nodes -:info_doctype: section -:info_title: Configure a CNI for hybrid nodes +[#hybrid-nodes-cni] += Configure CNI for hybrid nodes :info_titleabbrev: Configure CNI -:keywords: on-premises CNI, hybrid CNI -:info_abstract: Configure a CNI for Amazon EKS hybrid nodes - -include::../attributes.txt[] [abstract] -- -Configure a CNI for Amazon EKS hybrid nodes +Configure the Cilium CNI for hybrid nodes -- -Cilium and Calico are supported as the Container Networking Interfaces (CNIs) for Amazon EKS Hybrid Nodes. You must install a CNI for hybrid nodes to become ready to serve workloads. Hybrid nodes appear with status `Not Ready` until a CNI is running. You can manage these CNIs with your choice of tooling such as Helm. The Amazon VPC CNI is not compatible with hybrid nodes and the VPC CNI is configured with anti-affinity for the `eks.amazonaws.com/compute-type: hybrid` label. - -== Version compatibility - -The table below represents the Cilium and Calico versions that are compatible and validated for each Kubernetes version supported in Amazon EKS. +Cilium is the {aws}-supported Container Networking Interface (CNI) for Amazon EKS Hybrid Nodes. You must install a CNI for hybrid nodes to become ready to serve workloads. Hybrid nodes appear with status `Not Ready` until a CNI is running. You can manage the CNI with your choice of tools such as Helm. The instructions on this page cover Cilium lifecycle management (install, upgrade, delete). See <>, <>, and <> for how to configure Cilium for ingress, load balancing, and network policies. -[cols="1,1,1", options="header"] -|=== -|Kubernetes version -|Cilium version -|Calico version +Cilium is not supported by {aws} when running on nodes in {aws} Cloud. The Amazon VPC CNI is not compatible with hybrid nodes and the VPC CNI is configured with anti-affinity for the `eks.amazonaws.com/compute-type: hybrid` label. -|1.31 -|1.16.x -|3.29.x +The Calico documentation previously on this page has been moved to the link:https://github.com/aws-samples/eks-hybrid-examples[EKS Hybrid Examples Repository]. -|1.30 -|1.16.x -|3.29.x +[#hybrid-nodes-cilium-version-compatibility] +== Version compatibility -|1.29 -|1.16.x -|3.29.x +Cilium versions `v1.17.x` and `v1.18.x` are supported for EKS Hybrid Nodes for every Kubernetes version supported in Amazon EKS. -|1.28 -|1.16.x -|3.29.x +[NOTE] +*Cilium v1.18.3 kernel requirement*: Due to the kernel requirement (Linux kernel >= 5.10), Cilium v1.18.3 is not supported on: -|1.27 -|1.16.x -|3.29.x +- Ubuntu 20.04 +- Red Hat Enterprise Linux (RHEL) 8 -|1.26 -|1.16.x -|3.29.x +For system requirements, see link:https://docs.cilium.io/en/stable/operations/system_requirements/[Cilium system requirements]. -|1.25 -|1.16.x -|3.29.x -|=== +See link:eks/latest/userguide/kubernetes-versions.html[Kubernetes version support,type="documentation"] for the Kubernetes versions supported by Amazon EKS. EKS Hybrid Nodes have the same Kubernetes version support as Amazon EKS clusters with cloud nodes. +[#hybrid-nodes-cilium-support] == Supported capabilities -{aws} supports the following capabilities of Cilium and Calico for use with hybrid nodes. If you plan to use functionality outside the scope of {aws} support, we recommend that you obtain commercial support for the plugin or have the in-house expertise to troubleshoot and contribute fixes to the CNI plugin project. +{aws} maintains builds of Cilium for EKS Hybrid Nodes that are based on the open source link:https://github.com/cilium/cilium[Cilium project]. To receive support from {aws} for Cilium, you must be using the {aws}-maintained Cilium builds and supported Cilium versions. +{aws} provides technical support for the default configurations of the following capabilities of Cilium for use with EKS Hybrid Nodes. If you plan to use functionality outside the scope of {aws} support, it is recommended to obtain alternative commercial support for Cilium or have the in-house expertise to troubleshoot and contribute fixes to the Cilium project. -[cols="1,1,1", options="header"] +[%header,cols="2,3"] |=== -|Feature -|Cilium -|Calico +|Cilium Feature +^|Supported by {aws} |Kubernetes network conformance -|Yes -|Yes +^|Yes -|Control plane to node connectivity -|Yes -|Yes +|Core cluster connectivity +^|Yes -|Control plane to pod connectivity -|Yes -|Yes +|IP family +^|IPv4 |Lifecycle Management -|Install, Upgrade, Delete -|Install, Upgrade, Delete +^|Helm -|Networking Mode -|VXLAN -|VXLAN +|Networking Mode +^|VXLAN encapsulation |IP Address Management (IPAM) -|Cluster Scope (Cilium IPAM) -|Calico IPAM +^|Cilium IPAM Cluster Scope -|IP family -|IPv4 -|IPv4 +|Network Policy +^|Kubernetes Network Policy + +|Border Gateway Protocol (BGP) +^|Cilium BGP Control Plane + +|Kubernetes Ingress +^|Cilium Ingress, Cilium Gateway + +|Service LoadBalancer IP Allocation +^|Cilium Load Balancer IPAM + +|Service LoadBalancer IP Address Advertisement +^|Cilium BGP Control Plane + +|kube-proxy replacement +^|Yes -|BGP -|Yes (Cilium Control Plane) -|Yes |=== -== Install Cilium on hybrid nodes +[#hybrid-nodes-cilium-considerations] +== Cilium considerations -. Ensure that you have installed the helm CLI on your command-line environment. See the https://helm.sh/docs/intro/quickstart/[Helm documentation] for installation instructions. -. Install the Cilium Helm repo. +- *Helm repository* - {aws} hosts the Cilium Helm chart in the Amazon Elastic Container Registry Public (Amazon ECR Public) at link:https://gallery.ecr.aws/eks/cilium/cilium[Amazon EKS Cilium/Cilium]. The available versions include: + * Cilium v1.17.9: `oci://public.ecr.aws/eks/cilium/cilium:1.17.9-0` + * Cilium v1.18.3: `oci://public.ecr.aws/eks/cilium/cilium:1.18.3-0` + -[source,bash,subs="verbatim,attributes,quotes"] ----- -helm repo add cilium https://helm.cilium.io/ ----- +The commands in this topic use this repository. Note that certain `helm repo` commands aren't valid for Helm repositores in Amazon ECR Public, so you can't refer to this repository from a local Helm repo name. Instead, use the full URI in most commands. +- By default, Cilium is configured to run in overlay / tunnel mode with VXLAN as the link:https://docs.cilium.io/en/stable/network/concepts/routing/#encapsulation[encapsulation method]. This mode has the fewest requirements on the underlying physical network. +- By default, Cilium link:https://docs.cilium.io/en/stable/network/concepts/masquerading/[masquerades] the source IP address of all pod traffic leaving the cluster to the IP address of the node. If you disable masquerading, then your pod CIDRs must be routable on your on-premises network. +- If you are running webhooks on hybrid nodes, your pod CIDRs must be routable on your on-premises network. If your pod CIDRs are not routable on your on-premises network, then it is recommended to run webhooks on cloud nodes in the same cluster. See <> and <> for more information. +- {aws} recommends using Cilium's built-in BGP functionality to make your pod CIDRs routable on your on-premises network. For more information on how to configure Cilium BGP with hybrid nodes, see <>. +- The default IP Address Management (IPAM) in Cilium is called link:https://docs.cilium.io/en/stable/network/concepts/ipam/cluster-pool/[Cluster Scope], where the Cilium operator allocates IP addresses for each node based on user-configured pod CIDRs. -. Create a yaml file called `cilium-values.yaml`. If you configured at least one _remote pod network_, configure the same pod CIDRs for your `clusterPoolIPv4PodCIDRList`. You shouldn't change your `clusterPoolIPv4PodCIDRList` after deploying Cilium on your cluster. You can configure `clusterPoolIPv4MaskSize` based on your required pods per node, see https://docs.cilium.io/en/stable/network/concepts/ipam/cluster-pool/#expanding-the-cluster-pool[Expanding the cluster pool] in the Cilium documentation. For a full list of Helm values for Cilium, see the the https://docs.cilium.io/en/stable/helm-reference/[Helm reference] in the Cilium documentation. The following example configures all of the Cilium components to run on only the hybrid nodes, since they have the the `eks.amazonaws.com/compute-type: hybrid` label. -+ -By default, Cilium masquerades the source IP address of all pod traffic leaving the cluster to the IP address of the node. This makes it possible for Cilium to run with Amazon EKS clusters that have remote pod networks configured and with clusters that don't have remote pod networks configured. If you disable masquerading for your Cilium deployment, then you must configure your Amazon EKS cluster with your remote pod networks and you must advertise your pod addresses with your on-premises network. If you are running webhooks on your hybrid nodes, you must configure your cluster with your remote pod networks and you must advertise your pod addresses with your on-premises network. -+ -A common way to advertise pod addresses with your on-premises network is by using BGP. To use BGP with Cilium, you must set `bgpControlPlane.enabled: true`. For more information on Cilium's BGP support, see https://docs.cilium.io/en/stable/network/bgp-control-plane/bgp-control-plane/[Cilium BGP Control Plane] in the Cilium documentation. +[#hybrid-nodes-cilium-install] +== Install Cilium on hybrid nodes + +=== Procedure + +. Create a YAML file called `cilium-values.yaml`. The following example configures Cilium to run only on hybrid nodes by setting affinity for the `eks.amazonaws.com/compute-type: hybrid` label for the Cilium agent and operator. + +- Configure `clusterPoolIpv4PodCIDRList` with the same pod CIDRs you configured for your EKS cluster's _remote pod networks_. For example, `10.100.0.0/24`. The Cilium operator allocates IP address slices from within the configured `clusterPoolIpv4PodCIDRList` IP space. Your pod CIDR must not overlap with your on-premises node CIDR, your VPC CIDR, or your Kubernetes service CIDR. +- Configure `clusterPoolIpv4MaskSize` based on your required pods per node. For example, `25` for a /25 segment size of 128 pods per node. +- Do not change `clusterPoolIpv4PodCIDRList` or `clusterPoolIpv4MaskSize` after deploying Cilium on your cluster, see https://docs.cilium.io/en/stable/network/concepts/ipam/cluster-pool/#expanding-the-cluster-pool[Expanding the cluster pool] for more information. +- If you are running Cilium in kube-proxy replacement mode, set `kubeProxyReplacement: "true"` in your Helm values and ensure you do not have an existing kube-proxy deployment running on the same nodes as Cilium. +- The example below disables the Envoy Layer 7 (L7) proxy that Cilium uses for L7 network policies and ingress. For more information, see <> and <>. +- The example below configures `loadBalancer.serviceTopology`: `true` for Service Traffic Distribution to function correctly if you configure it for your services. For more information, see <>. +- For a full list of Helm values for Cilium, see the https://docs.cilium.io/en/stable/helm-reference/[Helm reference] in the Cilium documentation. + [source,bash,subs="verbatim,attributes,quotes"] ---- @@ -130,188 +124,108 @@ affinity: ipam: mode: cluster-pool operator: - clusterPoolIPv4MaskSize: 25 + clusterPoolIPv4MaskSize: [.replaceable]`25` clusterPoolIPv4PodCIDRList: - - POD_CIDR + - [.replaceable]`POD_CIDR` +loadBalancer: + serviceTopology: true operator: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: In + values: + - hybrid unmanagedPodWatcher: restart: false +loadBalancer: + serviceTopology: true +envoy: + enabled: false +kubeProxyReplacement: "false" ---- -. Install Cilium on your cluster. Replace `CILIUM_VERSION` with your desired Cilium version. It is recommended to run the latest patch version for your Cilium minor version. You can find the latest patch release for a given minor Cilium release in the https://github.com/cilium/cilium#stable-releases[Stable Releases section] of the Cilium documentation. If you are enabling BGP for your deployment, add the `--set bgpControlPlane.enabled=true` flag in the command below. If you are using a specific kubeconfig file, use the `--kubeconfig` flag with the Helm install command. +. Install Cilium on your cluster. +- Replace `CILIUM_VERSION` with a Cilium version (for example `1.17.9-0` or `1.18.3-0`). It is recommended to use the latest patch version for the Cilium minor version. +- Ensure your nodes meet the kernel requirements for the version you choose. Cilium v1.18.3 requires Linux kernel >= 5.10. +- If you are using a specific kubeconfig file, use the `--kubeconfig` flag with the Helm install command. + [source,bash,subs="verbatim,attributes,quotes"] ---- -helm install cilium cilium/cilium \ +helm install cilium oci://public.ecr.aws/eks/cilium/cilium \ --version [.replaceable]`CILIUM_VERSION` \ --namespace kube-system \ --values cilium-values.yaml ---- -. You can confirm your Cilium installation was successful with the following commands. You should see the `cilium-operator` deployment and the `cilium-agent` running on each of your hybrid nodes. Additionally, your hybrid nodes should now have status `Ready`. For information on how to configure BGP for Cilium, proceed to the next step. +. Confirm your Cilium installation was successful with the following commands. You should see the `cilium-operator` deployment and the `cilium-agent` running on each of your hybrid nodes. Additionally, your hybrid nodes should now have status `Ready`. For information on how to configure Cilium BGP to advertise your pod CIDRs to your on-premises network, proceed to <>. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- kubectl get pods -n kube-system ---- + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- NAME READY STATUS RESTARTS AGE cilium-jjjn8 1/1 Running 0 11m cilium-operator-d4f4d7fcb-sc5xn 1/1 Running 0 11m ---- + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- kubectl get nodes ---- + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- NAME STATUS ROLES AGE VERSION mi-04a2cf999b7112233 Ready 19m v1.31.0-eks-a737599 ---- -. To use BGP with Cilium to advertise your pod addresses with your on-premises network, you must have installed Cilium with `bgpControlPlane.enabled: true`. To configure BGP in Cilium, first create a file called `cilium-bgp-cluster.yaml` with a `CiliumBGPClusterConfig` with the peerAddress set to your on-premises router IP that you are peering with. Configure the `localASN` and `peerASN` based on your on-premises router configuration. -+ -[source,yaml,subs="verbatim,attributes,quotes"] ----- -apiVersion: cilium.io/v2alpha1 -kind: CiliumBGPClusterConfig -metadata: - name: cilium-bgp -spec: - nodeSelector: - matchExpressions: - - key: eks.amazonaws.com/compute-type - operator: In - values: - - hybrid - bgpInstances: - - name: "rack0" - localASN: ONPREM_ROUTER_ASN - peers: - - name: "onprem-router" - peerASN: PEER_ASN - peerAddress: ONPREM_ROUTER_IP - peerConfigRef: - name: "cilium-peer" ----- - -. Apply the Cilium BGP Cluster configuration to your cluster. -+ -[source,bash,subs="verbatim,attributes,quotes"] ----- -kubectl apply -f cilium-bgp-cluster.yaml ----- - -. The `CiliumBGPPeerConfig` resource is used to define a BGP peer configuration. Multiple peers can share the same configuration and provide reference to the common `CiliumBGPPeerConfig` resource. Create a file named `cilium-bgp-peer.yaml` to configure the peer configuration for your on-premises network. See the https://docs.cilium.io/en/latest/network/bgp-control-plane/bgp-control-plane-v2/#bgp-peer-configuration[BGP Peer Configuration] in the Cilium documentation for a full list of configuration options. -+ -[source,yaml,subs="verbatim,attributes,quotes"] ----- -apiVersion: cilium.io/v2alpha1 -kind: CiliumBGPPeerConfig -metadata: - name: cilium-peer -spec: - timers: - holdTimeSeconds: 30 - keepAliveTimeSeconds: 10 - gracefulRestart: - enabled: true - restartTimeSeconds: 120 - families: - - afi: ipv4 - safi: unicast - advertisements: - matchLabels: - advertise: "bgp" ----- - -. Apply the Cilium BGP Peer configuration to your cluster. -+ -[source,bash,subs="verbatim,attributes,quotes"] ----- -kubectl apply -f cilium-bgp-peer.yaml ----- - -. The `CiliumBGPAdvertisement` resource is used to define various advertisement types and attributes associated with them. Create a file named `cilium-bgp-advertisement.yaml` and configure the `CiliumBGPAdvertisement` resource with your desired settings. -+ -[source,yaml,subs="verbatim,attributes,quotes"] ----- -apiVersion: cilium.io/v2alpha1 -kind: CiliumBGPAdvertisement -metadata: - name: bgp-advertisements - labels: - advertise: bgp -spec: - advertisements: - - advertisementType: "PodCIDR" - - advertisementType: "Service" - service: - addresses: - - ClusterIP - - ExternalIP - - LoadBalancerIP ----- - -. Apply the Cilium BGP Advertisement configuration to your cluster. -+ -[source,bash,subs="verbatim,attributes,quotes"] ----- -kubectl apply -f cilium-bgp-advertisement.yaml ----- -+ -You can confirm the BGP peering worked with the https://docs.cilium.io/en/stable/gettingstarted/k8s-install-default/#install-the-cilium-cli[Cilium CLI] by using the `cilium bgp peers` command. You should see the correct values in the output for your environment and the Session State as `established`. See the https://docs.cilium.io/en/latest/network/bgp-control-plane/bgp-control-plane/#troubleshooting-and-operation-guide[Troubleshooting and Operations Guide] in the Cilium documentation for more information on troubleshooting. - +[#hybrid-nodes-cilium-upgrade] == Upgrade Cilium on hybrid nodes -Before upgrading your Cilium deployment, carefully review the https://docs.cilium.io/en/v1.16/operations/upgrade/[Cilium upgrade documentation] and the upgrade notes to understand the changes in the target Cilium version. +Before upgrading your Cilium deployment, carefully review the https://docs.cilium.io/en/v1.17/operations/upgrade/[Cilium upgrade documentation] and the upgrade notes to understand the changes in the target Cilium version. . Ensure that you have installed the `helm` CLI on your command-line environment. See the https://helm.sh/docs/intro/quickstart/[Helm documentation] for installation instructions. -. Install the Cilium Helm repo. -+ -[source,bash,subs="verbatim,attributes,quotes"] ----- -helm repo add cilium https://helm.cilium.io/ ----- - -. Run the Cilium upgrade pre-flight check. Replace `CILIUM_VERSION` with your target Cilium version. It is recommended to run the latest patch version for your Cilium minor version. You can find the latest patch release for a given minor Cilium release in the https://github.com/cilium/cilium#stable-releases[Stable Releases section] of the Cilium documentation. +. Run the Cilium upgrade pre-flight check. Replace `CILIUM_VERSION` with your target Cilium version. We recommend that you run the latest patch version for your Cilium minor version. You can find the latest patch release for a given minor Cilium release in the https://github.com/cilium/cilium#stable-releases[Stable Releases section] of the Cilium documentation. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- -helm install cilium-preflight cilium/cilium --version CILIUM_VERSION \ +helm install cilium-preflight oci://public.ecr.aws/eks/cilium/cilium --version CILIUM_VERSION \ --namespace=kube-system \ --set preflight.enabled=true \ --set agent=false \ --set operator.enabled=false ---- -. After applying the `cilium-preflight.yaml`, ensure that the number of READY pods is the same number of Cilium pods running. +. After applying the `cilium-preflight.yaml`, ensure that the number of `READY` pods is the same number of Cilium pods running. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- kubectl get ds -n kube-system | sed -n '1p;/cilium/p' ---- + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE cilium 2 2 2 2 2 1h20m cilium-pre-flight-check 2 2 2 2 2 7m15s ---- -. Once the number of READY pods are equal, make sure the Cilium pre-flight deployment is also marked as READY 1/1. If it shows READY 0/1, consult the https://docs.cilium.io/en/v1.16/operations/upgrade/#cnp-validation[CNP Validation] section and resolve issues with the deployment before continuing with the upgrade. +. Once the number of READY pods are equal, make sure the Cilium pre-flight deployment is also marked as READY 1/1. If it shows READY 0/1, consult the https://docs.cilium.io/en/v1.17/operations/upgrade/#cnp-validation[CNP Validation] section and resolve issues with the deployment before continuing with the upgrade. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- kubectl get deployment -n kube-system cilium-pre-flight-check -w ---- + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- NAME READY UP-TO-DATE AVAILABLE AGE cilium-pre-flight-check 1/1 1 0 12s @@ -319,36 +233,41 @@ cilium-pre-flight-check 1/1 1 0 12s . Delete the preflight + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- helm uninstall cilium-preflight --namespace kube-system ---- - -. During normal cluster operations, all Cilium components should run the same version. The following steps describe how to upgrade all of the components from one stable release to a later stable release. When upgrading from one minor release to another minor release, it is recommended to upgrade to the latest patch release for the existing Cilium minor version first. To minimize disruption, the upgradeCompatibility option should be set to the initial Cilium version which was installed in this cluster. +. Before running the `helm upgrade` command, preserve the values for your deployment in a `existing-cilium-values.yaml` or use `--set` command line options for your settings when you run the upgrade command. The upgrade operation overwrites the Cilium ConfigMap, so it is critical that your configuration values are passed when you upgrade. ++ +[source,bash,subs="verbatim,attributes,quotes"] +---- +helm get values cilium --namespace kube-system -o yaml > existing-cilium-values.yaml +---- + -Before running the helm upgrade command, preserve the values for your deployment in a `cilium-values.yaml` or use `--set` command line options for your settings. The upgrade operation overwrites the Cilium ConfigMap, so it is critical that your configuration values are passed when you upgrade. If you are using BGP, it is recommended to use the `--set bgpControlPlane=true` command line option instead of supplying this information in your values file. +. During normal cluster operations, all Cilium components should run the same version. The following steps describe how to upgrade all of the components from one stable release to a later stable release. When upgrading from one minor release to another minor release, it is recommended to upgrade to the latest patch release for the existing Cilium minor version first. To minimize disruption, set the `upgradeCompatibility` option to the initial Cilium version that you installed in this cluster. + [source,bash,subs="verbatim,attributes,quotes"] ---- -helm upgrade cilium cilium/cilium --version CILIUM_VERSION \ +helm upgrade cilium oci://public.ecr.aws/eks/cilium/cilium --version [.replaceable]`CILIUM_VERSION` \ --namespace kube-system \ - --set upgradeCompatibility=1.X \ - -f cilium-values.yaml + --set upgradeCompatibility=[.replaceable]`1.X` \ + -f existing-cilium-values.yaml ---- - ++ . (Optional) If you need to rollback your upgrade due to issues, run the following commands. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- helm history cilium --namespace kube-system helm rollback cilium [REVISION] --namespace kube-system ---- +[#hybrid-nodes-cilium-delete] == Delete Cilium from hybrid nodes -. Run the following command to uninstall all Cilium components from your cluster. Note, uninstalling the CNI may impact the health of nodes and pods and shouldn't be performed on production clusters. +. Run the following command to uninstall all Cilium components from your cluster. Note, uninstalling the CNI might impact the health of nodes and pods and shouldn't be performed on production clusters. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- helm uninstall cilium --namespace kube-system ---- @@ -359,205 +278,7 @@ The interfaces and routes configured by Cilium are not removed by default when t . To remove the Cilium Custom Resource Definitions (CRDs) from your cluster, you can run the following commands. + -[source,bash,subs="verbatim,attributes,quotes"] ----- -kubectl get crds -oname | grep "cilium" | xargs kubectl delete ----- - -== Install Calico on hybrid nodes - -. Ensure that you have installed the helm CLI on your command-line environment. See the https://helm.sh/docs/intro/quickstart/[Helm documentation] for installation instructions. -. Install the Cilium Helm repo. -+ -[source,bash,subs="verbatim,attributes,quotes"] ----- -helm repo add projectcalico https://docs.tigera.io/calico/charts ----- - -. Create a yaml file called `calico-values.yaml` that configures Calico with affinity to run on hybrid nodes. For more information on the different Calico networking modes, see https://docs.tigera.io/calico/latest/networking/determine-best-networking[Determining the best networking option] in the Calico documentation. -.. Replace `POD_CIDR` with the CIDR ranges for your pods. If you configured your Amazon EKS cluster with remote pod networks, the `POD_CIDR` that you specify for Calico should be the same as the remote pod networks. For example, `10.100.0.0/24`. -.. Replace `CIDR_SIZE` with the size of the CIDR segment you wish to allocate to each node. For example, `25` for a /25 segment size. For more information on CIDR `blockSize` and changing the `blockSize`, see https://docs.tigera.io/calico/latest/networking/ipam/change-block-size[Change IP pool block size] in the Calico documentation. -.. In the example below, `natOutgoing` is enabled and `bgp` is disabled. In this configuration, Calico can run on Amazon EKS clusters that have Remote Pod Network configured and can run on clusters that do not have Remote Pod Network configured. If you have `natOutgoing` set to disabled, you must configure your cluster with your remote pod networks and your on-premises network must be able to properly route traffic destined for your pod CIDRs. A common way to advertise pod addresses with your on-premises network is by using BGP. To use BGP with Calico, you must enable `bgp`. The example below configures all of the Calico components to run on only the hybrid nodes, since they have the `eks.amazonaws.com/compute-type: hybrid` label. If you are running webhooks on your hybrid nodes, you must configure your cluster with your Remote Pod Networks and you must advertise your pod addresses with your on-premises network. The example below configures `controlPlaneReplicas: 1`, increase the value if you have multiple hybrid nodes and want to run the Calico control plane components in a highly available fashion. -+ -[source,yaml,subs="verbatim,attributes,quotes"] ----- -installation: - enabled: true - cni: - type: Calico - ipam: - type: Calico - calicoNetwork: - bgp: Disabled - ipPools: - - cidr: POD_CIDR - blockSize: CIDR_SIZE - encapsulation: VXLAN - natOutgoing: Enabled - nodeSelector: eks.amazonaws.com/compute-type == "hybrid" - controlPlaneReplicas: 1 - controlPlaneNodeSelector: - eks.amazonaws.com/compute-type: hybrid - calicoNodeDaemonSet: - spec: - template: - spec: - nodeSelector: - eks.amazonaws.com/compute-type: hybrid - csiNodeDriverDaemonSet: - spec: - template: - spec: - nodeSelector: - eks.amazonaws.com/compute-type: hybrid - calicoKubeControllersDeployment: - spec: - template: - spec: - nodeSelector: - eks.amazonaws.com/compute-type: hybrid - typhaDeployment: - spec: - template: - spec: - nodeSelector: - eks.amazonaws.com/compute-type: hybrid ----- - -. Install Calico on your cluster. Replace `CALICO_VERSION` with your desired Calico version (for example 3.29.0), see the https://github.com/projectcalico/calico/releases[Calico releases] to find the latest patch release for your Calico minor version. It is recommended to run the latest patch version for the Calico minor version. If you are using a specific `kubeconfig` file, use the `--kubeconfig` flag. -+ -[source,bash,subs="verbatim,attributes,quotes"] ----- -helm install calico projectcalico/tigera-operator \ - --version [.replaceable]`CALICO_VERSION` \ - --namespace kube-system \ - -f calico-values.yaml ----- - -. You can confirm your Calico installation was successful with the following commands. You should see the `tigera-operator` deployment, the `calico-node` agent running on each of your hybrid nodes, as well as the `calico-apiserver`, `csi-node-driver`, and `calico-kube-controllers` deployed. Additionally, your hybrid nodes should now have status `Ready`. If you are using `natOutgoing: Disabled`, then all of the Calico components will not be able to start successfully until you advertise your pod addresses with your on-premises network. For information on how to configure BGP for Calico, proceed to the next step. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl get pods -A ----- -+ [source,bash,subs="verbatim,attributes"] ---- -NAMESPACE NAME READY STATUS RESTARTS AGE -calico-apiserver calico-apiserver-6c77bb6d46-2n8mq 1/1 Running 0 69s -calico-system calico-kube-controllers-7c5f8556b5-7h267 1/1 Running 0 68s -calico-system calico-node-s5nnk 1/1 Running 0 68s -calico-system calico-typha-6487cc9d8c-wc5jm 1/1 Running 0 69s -calico-system csi-node-driver-cv42d 2/2 Running 0 68s -kube-system coredns-7bb495d866-2lc9v 1/1 Running 0 6m27s -kube-system coredns-7bb495d866-2t8ln 1/1 Running 0 157m -kube-system kube-proxy-lxzxh 1/1 Running 0 18m -kube-system tigera-operator-f8bc97d4c-28b4d 1/1 Running 0 90s ----- -+ -[source,bash,subs="verbatim,attributes,quotes"] ----- -kubectl get nodes ----- -+ -[source,bash,subs="verbatim,attributes,quotes"] ----- -NAME STATUS ROLES AGE VERSION -mi-0c6ec2f6f79176565 Ready 5h13m v1.31.0-eks-a737599 ----- -. If you installed Calico without BGP, skip this step. To configure BGP, create a file called `calico-bgp.yaml` with a `BGPPeer` configuration and a `BGPConfiguration`. It is important to distinguish `BGPPeer` and `BGPConfiguration`. The `BGPPeer` is the BGP-enabled router or remote resource with which the nodes in a Calico cluster will peer. The `asNumber` in the `BGPPeer` configuration is similar to the Cilium setting `peerASN` . The `BGPConfiguration` is applied to each Calico node and the `asNumber` for the `BGPConfiguration` is equivalent to the Cilium setting `localASN`. Replace `ONPREM_ROUTER_IP`, `ONPREM_ROUTER_ASN`, and `LOCAL_ASN` in the example below with the values for your on-premises environment. The `keepOriginalNextHop: true` setting is used to ensure each node advertises only the pod network CIDR that it owns. -+ -[source,bash,subs="verbatim,attributes,quotes"] ----- -apiVersion: projectcalico.org/v3 -kind: BGPPeer -metadata: - name: calico-hybrid-nodes -spec: - peerIP: [.replaceable]`ONPREM_ROUTER_IP` - asNumber: [.replaceable]`ONPREM_ROUTER_ASN` - keepOriginalNextHop: true ---- -apiVersion: projectcalico.org/v3 -kind: BGPConfiguration -metadata: - name: default -spec: - nodeToNodeMeshEnabled: false - asNumber: [.replaceable]`LOCAL_ASN` ----- - -. Apply the file to your cluster. -+ -[source,bash,subs="verbatim,attributes,quotes"] ----- -kubectl apply -f calico-bgp.yaml ----- - -. Confirm the Calico pods are running with the following command. -+ -[source,bash,subs="verbatim,attributes,quotes"] ----- -kubectl get pods -n calico-system -w ----- -+ -[source,bash,subs="verbatim,attributes,quotes"] ----- -NAMESPACE NAME READY STATUS RESTARTS AGE -calico-apiserver calico-apiserver-598bf99b6c-2vltk 1/1 Running 0 3h24m -calico-system calico-kube-controllers-75f84bbfd6-zwmnx 1/1 Running 31 (59m ago) 3h20m -calico-system calico-node-9b2pg 1/1 Running 0 5h17m -calico-system calico-typha-7d55c76584-kxtnq 1/1 Running 0 5h18m -calico-system csi-node-driver-dmnmm 2/2 Running 0 5h18m -kube-system coredns-7bb495d866-dtn4z 1/1 Running 0 6h23m -kube-system coredns-7bb495d866-mk7j4 1/1 Running 0 6h19m -kube-system kube-proxy-vms28 1/1 Running 0 6h12m -kube-system tigera-operator-55f9d9d565-jj9bg 1/1 Running 0 73m ----- - -If you encountered issues during these steps, see the https://docs.tigera.io/calico/latest/operations/troubleshoot/vpp[troubleshooting guidance] in the Calico documentation. - -== Upgrade Calico on hybrid nodes - -Before upgrading your Calico deployment, carefully review the https://docs.tigera.io/calico/latest/operations/upgrading/kubernetes-upgrade[Calico upgrade documentation] and the https://docs.tigera.io/calico/latest/release-notes/[release notes] to understand the changes in the target Calico version. The upgrade steps vary based on whether you are using Helm, the Calico operator, and the type of datastore. The steps below assume use of Helm. - -. Download the operator manifest for the version of Calico you are upgrading to. Replace `CALICO_VERSION` with the version you are upgrading to, for example `v3.29.0`. Make sure to prepend the `v` to the major.minor.patch. -+ -[source,bash,subs="verbatim,attributes,quotes"] ----- -kubectl apply --server-side --force-conflicts \ - -f https://raw.githubusercontent.com/projectcalico/calico/[.replaceable]`CALICO_VERSION`/manifests/operator-crds.yaml ----- - -. Run `helm upgrade` to upgrade your Calico deployment. Replace CALICO_VERSION with the version you are upgrading to, for example `v3.29.0`. Create the `calico-values.yaml` file from the configuration values that you used to install Calico. -+ -[source,bash,subs="verbatim,attributes,quotes"] ----- -helm upgrade calico projectcalico/tigera-operator \ - --version [.replaceable]`CALICO_VERSION` \ - --namespace kube-system \ - -f calico-values.yaml ----- - -== Delete Calico from hybrid nodes - -. Run the following command to uninstall Calico components from your cluster. Note that uninstalling the CNI may impact the health of nodes and pods and should not be performed on production clusters. If you installed Calico in a namespace other than `kube-system` change the namespace in the command below. -+ -[source,bash,subs="verbatim,attributes,quotes"] ----- -helm uninstall calico --namespace kube-system ----- -+ -Note that the interfaces and routes configured by Calico are not removed by default when the CNI is removed from the cluster. -. To clean up the on-disk configuration files and resources, remove the Calico files from the `/opt/cni` and `/etc/cni` directories. -. To remove the Calico CRDs from your cluster, run the following commands. -+ -[source,bash,subs="verbatim,attributes,quotes"] ----- -kubectl get crds -oname | grep "calico" | xargs kubectl delete ----- -+ -[source,bash,subs="verbatim,attributes,quotes"] ----- -kubectl get crds -oname | grep "tigera" | xargs kubectl delete +kubectl get crds -oname | grep "cilium" | xargs kubectl delete ---- diff --git a/latest/ug/nodes/hybrid-nodes-concepts-kubernetes.adoc b/latest/ug/nodes/hybrid-nodes-concepts-kubernetes.adoc new file mode 100644 index 000000000..b8fa38d67 --- /dev/null +++ b/latest/ug/nodes/hybrid-nodes-concepts-kubernetes.adoc @@ -0,0 +1,262 @@ +include::../attributes.txt[] + +[.topic] +[#hybrid-nodes-concepts-kubernetes] += Kubernetes concepts for hybrid nodes +:info_titleabbrev: Kubernetes concepts + +[abstract] +-- +Learn about the key Kubernetes concepts for EKS Hybrid Nodes. +-- + +This page details the key Kubernetes concepts that underpin the EKS Hybrid Nodes system architecture. + +[#hybrid-nodes-concepts-k8s-api] +== EKS control plane in the VPC + +The IPs of the EKS control plane ENIs are stored in the `kubernetes` `Endpoints` object in the `default` namespace. When EKS creates new ENIs or removes older ones, EKS updates this object so the list of IPs is always up-to-date. + +You can use these endpoints through the `kubernetes` Service, also in the `default` namespace. This service, of `ClusterIP` type, always gets assigned the first IP of the cluster's service CIDR. For example, for the service CIDR `172.16.0.0/16`, the service IP will be `172.16.0.1`. + +Generally, this is how pods (regardless if running in the cloud or hybrid nodes) access the EKS Kubernetes API server. Pods use the service IP as the destination IP, which gets translated to the actual IPs of one of the EKS control plane ENIs. The primary exception is `kube-proxy`, because it sets up the translation. + +[#hybrid-nodes-concepts-k8s-eks-api] +== EKS API server endpoint + +The `kubernetes` service IP isn't the only way to access the EKS API server. EKS also creates a Route53 DNS name when you create your cluster. This is the `endpoint` field of your EKS cluster when calling the EKS `DescribeCluster` API action. + +[source,json] +---- +{ + "cluster": { + "endpoint": "https://xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.gr7.us-west-2.eks.amazonaws.com", + "name": "my-cluster", + "status": "ACTIVE" + } +} +---- + +In a public endpoint access or public and private endpoint access cluster, your hybrid nodes will resolve this DNS name to a public IP by default, routable through the internet. In a private endpoint access cluster, the DNS name resolves to the private IPs of the EKS control plane ENIs. + +This is how the `kubelet` and `kube-proxy` access the Kubernetes API server. If you want all your Kubernetes cluster traffic to flow through the VPC, you either need to configure your cluster in private access mode or modify your on-premises DNS server to resolve the EKS cluster endpoint to the private IPs of the EKS control plane ENIs. + +[#hybrid-nodes-concepts-k8s-kubelet-api] +== `kubelet` endpoint + +The `kubelet` exposes several REST endpoints, allowing other parts of the system to interact with and gather information from each node. In most clusters, the majority of traffic to the `kubelet` server comes from the control plane, but certain monitoring agents might also interact with it. + +Through this interface, the `kubelet` handles various requests: fetching logs (`kubectl logs`), executing commands inside containers (`kubectl exec`), and port-forwarding traffic (`kubectl port-forward`). Each of these requests interacts with the underlying container runtime through the `kubelet`, appearing seamless to cluster administrators and developers. + +The most common consumer of this API is the Kubernetes API server. When you use any of the `kubectl` commands mentioned previously, `kubectl` makes an API request to the API server, which then calls the `kubelet` API of the node where the pod is running. This is the main reason why the node IP needs to be reachable from the EKS control plane and why, even if your pods are running, you won't be able to access their logs or `exec` if the node route is misconfigured. + +*Node IPs* + +When the EKS control plane communicates with a node, it uses one of the addresses reported in the `Node` object status (`status.addresses`). + +With EKS cloud nodes, it's common for the kubelet to report the private IP of the EC2 instance as an `InternalIP` during the node registration. This IP is then validated by the Cloud Controller Manager (CCM) making sure it belongs to the EC2 instance. In addition, the CCM typically adds the public IPs (as `ExternalIP`) and DNS names (`InternalDNS` and `ExternalDNS`) of the instance to the node status. + +However, there is no CCM for hybrid nodes. When you register a hybrid node with the EKS Hybrid Nodes CLI (`nodeadm`), it configures the kubelet to report your machine's IP directly in the node's status, without the CCM. + +[source,yaml] +---- +apiVersion: v1 +kind: Node +metadata: + name: my-node-1 +spec: + providerID: eks-hybrid:///us-west-2/my-cluster/my-node-1 +status: + addresses: + - address: 10.1.1.236 + type: InternalIP + - address: my-node-1 + type: Hostname +---- + +If your machine has multiple IPs, the kubelet will select one of them following its own logic. You can control the selected IP with the `--node-ip` flag, which you can pass in `nodeadm` config in `spec.kubelet.flags`. Only the IP reported in the `Node` object needs a route from the VPC. Your machines can have other IPs that aren't reachable from the cloud. + +[#hybrid-nodes-concepts-k8s-kube-proxy] +== `kube-proxy` + +`kube-proxy` is responsible for implementing the Service abstraction at the networking layer of each node. It acts as a network proxy and load balancer for traffic destined to Kubernetes Services. By continuously watching the Kubernetes API server for changes related to Services and Endpoints, `kube-proxy` dynamically updates the underlying host's networking rules to ensure traffic is properly directed. + +In `iptables` mode, `kube-proxy` programs several `netfilter` chains to handle service traffic. The rules form the following hierarchy: + +. *KUBE-SERVICES chain*: The entry point for all service traffic. It has rules matching each service's `ClusterIP` and port. +. *KUBE-SVC-XXX chains*: Service-specific chains has load balancing rules for each service. +. *KUBE-SEP-XXX chains*: Endpoint-specific chains has the actual `DNAT` rules. + +Let's examine what happens for a service `test-server` in the `default` namespace: ++*++ Service ClusterIP: `172.16.31.14` ++*++ Service port: `80` ++*++ Backing pods: `10.2.0.110`, `10.2.1.39`, and `10.2.2.254` + +When we inspect the `iptables` rules (using `iptables-save ++|++ grep -A10 KUBE-SERVICES`): + +. In the *KUBE-SERVICES* chain, we find a rule matching the service: ++ +[source,text] +---- +-A KUBE-SERVICES -d 172.16.31.14/32 -p tcp -m comment --comment "default/test-server cluster IP" -m tcp --dport 80 -j KUBE-SVC-XYZABC123456 +---- + +* This rule matches packets destined for 172.16.31.14:80 +* The comment indicates what this rule is for: `default/test-server cluster IP` +* Matching packets jump to the `KUBE-SVC-XYZABC123456` chain +. The *KUBE-SVC-XYZABC123456* chain has probability-based load balancing rules: ++ +[source,text] +---- +-A KUBE-SVC-XYZABC123456 -m statistic --mode random --probability 0.33333333349 -j KUBE-SEP-POD1XYZABC +-A KUBE-SVC-XYZABC123456 -m statistic --mode random --probability 0.50000000000 -j KUBE-SEP-POD2XYZABC +-A KUBE-SVC-XYZABC123456 -j KUBE-SEP-POD3XYZABC +---- + +* First rule: 33.3% chance to jump to `KUBE-SEP-POD1XYZABC` +* Second rule: 50% chance of the remaining traffic (33.3% of total) to jump to `KUBE-SEP-POD2XYZABC` +* Last rule: All remaining traffic (33.3% of total) jumps to `KUBE-SEP-POD3XYZABC` +. The individual *KUBE-SEP-XXX* chains perform the DNAT (Destination NAT): ++ +[source,text] +---- +-A KUBE-SEP-POD1XYZABC -p tcp -m tcp -j DNAT --to-destination 10.2.0.110:80 +-A KUBE-SEP-POD2XYZABC -p tcp -m tcp -j DNAT --to-destination 10.2.1.39:80 +-A KUBE-SEP-POD3XYZABC -p tcp -m tcp -j DNAT --to-destination 10.2.2.254:80 +---- +* These DNAT rules rewrite the destination IP and port to direct traffic to specific pods. +* Each rule handles about 33.3% of the traffic, providing even load balancing between `10.2.0.110`, `10.2.1.39` and `10.2.2.254`. + +This multi-level chain structure enables `kube-proxy` to efficiently implement service load balancing and redirection through kernel-level packet manipulation, without requiring a proxy process in the data path. + +[#hybrid-nodes-concepts-k8s-operations] +=== Impact on Kubernetes operations + +A broken `kube-proxy` on a node prevents that node from routing Service traffic properly, causing timeouts or failed connections for pods that rely on cluster Services. This can be especially disruptive when a node is first registered. The CNI needs to talk to the Kubernetes API server to get information, such as the node's pod CIDR, before it can configure any pod networking. To do that, it uses the `kubernetes` Service IP. However, if `kube-proxy` hasn't been able to start or has failed to set the right `iptables` rules, the requests going to the `kubernetes` service IP aren't translated to the actual IPs of the EKS control plane ENIs. As a consequence, the CNI will enter a crash loop and none of the pods will be able to run properly. + +We know pods use the `kubernetes` service IP to communicate with the Kubernetes API server, but `kube-proxy` needs to first set `iptables` rules to make that work. + +How does `kube-proxy` communicate with the API server? + +The `kube-proxy` must be configured to use the actual IP/s of the Kubernetes API server or a DNS name that resolves to them. In the case of EKS, EKS configures the default `kube-proxy` to point to the Route53 DNS name that EKS creates when you create the cluster. You can see this value in the `kube-proxy` ConfigMap in the `kube-system` namespace. The content of this ConfigMap is a `kubeconfig` that gets injected into the `kube-proxy` pod, so look for the `clusters++[]++.cluster.server` field. This value will match the `endpoint` field of your EKS cluster (when calling EKS `DescribeCluster` API). + +[source,yaml] +---- +apiVersion: v1 +data: + kubeconfig: |- + kind: Config + apiVersion: v1 + clusters: + - cluster: + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server: https://xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.gr7.us-west-2.eks.amazonaws.com + name: default + contexts: + - context: + cluster: default + namespace: default + user: default + name: default + current-context: default + users: + - name: default + user: + tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +kind: ConfigMap +metadata: + name: kube-proxy + namespace: kube-system +---- + +[#hybrid-nodes-concepts-k8s-pod-cidrs] +== Routable remote Pod CIDRs + +The <> page details the requirements to run webhooks on hybrid nodes or to have pods running on cloud nodes communicate with pods running on hybrid nodes. The key requirement is that the on-premises router needs to know which node is responsible for a particular pod IP. There are several ways to achieve this, including Border Gateway Protocol (BGP), static routes, and Address Resolution Protocol (ARP) proxying. These are covered in the following sections. + +*Border Gateway Protocol (BGP)* + +If your CNI supports it (such as Cilium and Calico), you can use the BGP mode of your CNI to propagate routes to your per node pod CIDRs from your nodes to your local router. When using the CNI's BGP mode, your CNI acts as a virtual router, so your local router thinks the pod CIDR belongs to a different subnet and your node is the gateway to that subnet. + +image::images/hybrid-nodes-bgp.png[Hybrid nodes BGP routing] + +*Static routes* + +Or, you can configure static routes in your local router. This is the simplest way to route the on-premises pod CIDR to your VPC, but it is also the most error prone and difficult to maintain. You need to make sure that the routes are always up-to-date with the existing nodes and their assigned pod CIDRs. If your number of nodes is small and infrastructure is static, this is a viable option and removes the need for BGP support in your router. If you opt for this, we recommend to configure your CNI with the pod CIDR slice that you want to assign to each node instead of letting its IPAM decide. + +image::images/hybrid-nodes-static-routes.png[Hybrid nodes static routing] + +*Address Resolution Protocol (ARP) proxying* + +ARP proxying is another approach to make on-premises pod IPs routable, particularly useful when your hybrid nodes are on the same Layer 2 network as your local router. With ARP proxying enabled, a node responds to ARP requests for pod IPs it hosts, even though those IPs belong to a different subnet. + +When a device on your local network tries to reach a pod IP, it first sends an ARP request asking "`Who has this IP?`". The hybrid node hosting that pod will respond with its own MAC address, saying "`I can handle traffic for that IP.`" This creates a direct path between devices on your local network and the pods without requiring router configuration. + +For this to work, your CNI must support proxy ARP functionality. Cilium has built-in support for proxy ARP that you can enable through configuration. The key consideration is that the pod CIDR must not overlap with any other network in your environment, as this could cause routing conflicts. + +This approach has several advantages: +* No need to configure your router with BGP or maintain static routes +* Works well in environments where you don't have control over your router configuration + +image::images/hybrid-nodes-arp-proxy.png[Hybrid nodes ARP proxying] + +[#hybrid-nodes-concepts-k8s-pod-encapsulation] +== Pod-to-Pod encapsulation + +In on-premises environments, CNIs typically use encapsulation protocols to create overlay networks that can operate on top of the physical network without the need to re-configure it. This section explains how this encapsulation works. Note that some of the details might vary depending on the CNI you are using. + +Encapsulation wraps original pod network packets inside another network packet that can be routed through the underlying physical network. This allows pods to communicate across nodes running the same CNI without requiring the physical network to know how to route those pod CIDRs. + +The most common encapsulation protocol used with Kubernetes is Virtual Extensible LAN (VXLAN), though others (such as `Geneve`) are also available depending on your CNI. + +=== VXLAN encapsulation + +VXLAN encapsulates Layer 2 Ethernet frames within UDP packets. When a pod sends traffic to another pod on a different node, the CNI performs the following: + +. The CNI intercepts packets from Pod A +. The CNI wraps the original packet in a VXLAN header +. This wrapped packet is then sent through the node's regular networking stack to the destination node +. The CNI on the destination node unwraps the packet and delivers it to Pod B + +Here's what happens to the packet structure during VXLAN encapsulation: + +Original Pod-to-Pod Packet: + +.... ++-----------------+---------------+-------------+-----------------+ +| Ethernet Header | IP Header | TCP/UDP | Payload | +| Src: Pod A MAC | Src: Pod A IP | Src Port | | +| Dst: Pod B MAC | Dst: Pod B IP | Dst Port | | ++-----------------+---------------+-------------+-----------------+ +.... + +After VXLAN Encapsulation: + +.... ++-----------------+-------------+--------------+------------+---------------------------+ +| Outer Ethernet | Outer IP | Outer UDP | VXLAN | Original Pod-to-Pod | +| Src: Node A MAC | Src: Node A | Src: Random | VNI: xx | Packet (unchanged | +| Dst: Node B MAC | Dst: Node B | Dst: 4789 | | from above) | ++-----------------+-------------+--------------+------------+---------------------------+ +.... + +The VXLAN Network Identifier (VNI) distinguishes between different overlay networks. + +=== Pod communication scenarios + +*Pods on the same hybrid node* + +When pods on the same hybrid node communicate, no encapsulation is typically needed. The CNI sets up local routes that direct traffic between pods through the node's internal virtual interfaces: + +.... +Pod A -> veth0 -> node's bridge/routing table -> veth1 -> Pod B +.... + +The packet never leaves the node and doesn't require encapsulation. + +*Pods on different hybrid nodes* + +Communication between pods on different hybrid nodes requires encapsulation: + +.... +Pod A -> CNI -> [VXLAN encapsulation] -> Node A network -> router or gateway -> Node B network -> [VXLAN decapsulation] -> CNI -> Pod B +.... + +This allows the pod traffic to traverse the physical network infrastructure without requiring the physical network to understand pod IP routing. \ No newline at end of file diff --git a/latest/ug/nodes/hybrid-nodes-concepts-networking.adoc b/latest/ug/nodes/hybrid-nodes-concepts-networking.adoc new file mode 100644 index 000000000..c95c72917 --- /dev/null +++ b/latest/ug/nodes/hybrid-nodes-concepts-networking.adoc @@ -0,0 +1,79 @@ +include::../attributes.txt[] + +[.topic] +[#hybrid-nodes-concepts-networking] += Networking concepts for hybrid nodes +:info_titleabbrev: Networking concepts + +[abstract] +-- +Learn about the networking concepts for EKS Hybrid Nodes. +-- + +This section details the core networking concepts and the constraints you must consider when designing your network topology for EKS Hybrid Nodes. + +== Networking concepts for EKS Hybrid Nodes + +image::images/hybrid-nodes-highlevel-network.png[High level hybrid nodes network diagram] + +*VPC as the network hub* + +All traffic that crosses the cloud boundary routes through your VPC. This includes traffic between the EKS control plane or pods running in {aws} to hybrid nodes or pods running on them. You can think of your cluster's VPC as the network hub between your hybrid nodes and the rest of the cluster. This architecture gives you full control of the traffic and its routing but also makes it your responsibility to correctly configure routes, security groups, and firewalls for the VPC. + +*EKS control plane to the VPC* + +The EKS control plane attaches *Elastic Network Interfaces (ENIs)* to your VPC. These ENIs handle traffic to and from the EKS API server. You control the placement of the EKS control plane ENIs when you configure your cluster, as EKS attaches ENIs to the subnets you pass during cluster creation. + +EKS associates Security Groups to the ENIs that EKS attaches to your subnets. These security groups allow traffic to and from the EKS control plane through the ENIs. This is important for EKS Hybrid Nodes because you must allow traffic from the hybrid nodes and the pods running on them to the EKS control plane ENIs. + +*Remote Node Networks* + +The remote node networks, specifically the remote node CIDRs, are the ranges of IPs assigned to the machines you use as hybrid nodes. When you provision hybrid nodes, they reside in your on-premises data center or edge location, which is a different network domain than the EKS control plane and VPC. Each hybrid node has an IP address, or addresses, from a remote node CIDR that is distinct from the subnets in your VPC. + +You configure the EKS cluster with these remote node CIDRs so EKS knows to route all traffic destined for the hybrid nodes IPs through your cluster VPC, such as requests to the kubelet API. The connections to the `kubelet` API are used in the `kubectl attach`, `kubectl cp`, `kubectl exec`, `kubectl logs`, and `kubectl port-forward` commands. + +*Remote Pod Networks* + +The remote pod networks are the ranges of IPs assigned to the pods running on the hybrid nodes. Generally, you configure your CNI with these ranges and the IP Address Management (IPAM) functionality of the CNI takes care of assigning a slice of these ranges to each hybrid node. When you create a pod, the CNI assigns an IP to the pod from the slice allocated to the node where the pod has been scheduled. + +You configure the EKS cluster with these remote pod CIDRs so the EKS control plane knows to route all traffic destined for the pods running on the hybrid nodes through your cluster's VPC, such as communication with webhooks. + +image::images/hybrid-nodes-remote-pod-cidrs.png[Remote Pod Networks] + +*On-premises to the VPC* + +The on-premises network you use for hybrid nodes must route to the VPC you use for your EKS cluster. There are several link:whitepapers/latest/aws-vpc-connectivity-options/network-to-amazon-vpc-connectivity-options.html[Network-to-Amazon VPC connectivity option,type="documentation"] available to connect your on-premises network to a VPC. You can also use your own VPN solution. + +It is important that you configure the routing correctly on the {aws} Cloud side in the VPC and in your on-premises network, so that both networks route the right traffic through the connection for the two networks. + +In the VPC, all traffic going to the remote node and remote pod networks must route through the connection to your on-premises network (referred to as the "gateway"). If some of your subnets have different route tables, you must configure each route table with the routes for your hybrid nodes and the pods running on them. This is true for the subnets where the EKS control plane ENIs are attached to, and subnets that contain EC2 nodes or pods that must communicate with hybrid nodes. + +In your on-premises network, you must configure your network to allow traffic to and from your EKS cluster's VPC and the other {aws} services required for hybrid nodes. The traffic for the EKS cluster traverses the gateway in both directions. + +== Networking constraints + +*Fully routed network* + +The main constraint is that the EKS control plane and all nodes, cloud or hybrid nodes, need to form a *fully routed* network. This means that all nodes must be able to reach each other at layer three, by IP address. + +The EKS control plane and cloud nodes are already reachable from each other because they are in a flat network (the VPC). The hybrid nodes, however, are in a different network domain. This is why you need to configure additional routing in the VPC and on your on-premises network to route traffic between the hybrid nodes and the rest of the cluster. If the hybrid nodes are reachable from each other and from the VPC, your hybrid nodes can be in one single flat network or in multiple segmented networks. + +*Routable remote pod CIDRs* + +For the EKS control plane to communicate with pods running on hybrid nodes (for example, webhooks or the Metrics Server) or for pods running on cloud nodes to communicate with pods running on hybrid nodes (workload east-west communication), your remote pod CIDR must be routable from the VPC. This means that the VPC must be able to route traffic to the pod CIDRs through the gateway to your on-premises network and that your on-premises network must be able to route the traffic for a pod to the right node. + +It's important to note the distinction between the pod routing requirements in the VPC and on-premises. The VPC only needs to know that any traffic going to a remote pod should go through the gateway. If you only have one remote pod CIDR, you only need one route. + +This requirement is true for all hops in your on-premises network up to the local router in the same subnet as your hybrid nodes. This is the only router that needs to be aware of the pod CIDR slice assigned to each node, making sure that traffic for a particular pod gets delivered to the node where the pod has been scheduled. + +You can choose to propagate these routes for the on-premises pod CIDRs from your local on-premises router to the VPC route tables, but it isn't necessary. If your on-premises pod CIDRs change frequently and your VPC route tables need to be updated to reflect the changing pod CIDRs, we recommend that you propagate the on-premises pod CIDRs to the VPC route tables, but this is uncommon. + +Note, the constraint for making your on-premises pod CIDRs routable is optional. If you don't need to run webhooks on your hybrid nodes or have pods on cloud nodes talk to pods on hybrid nodes, you don't need to configure routing for the pod CIDRs on your on-premises network. + +_Why do the on-premises pod CIDRs need to be routable with hybrid nodes?_ + +When using EKS with the VPC CNI for your cloud nodes, the VPC CNI assigns IPs directly from the VPC to the pods. This means there is no need for any special routing, as both cloud pods and the EKS control plane can reach the Pod IPs directly. + +When running on-premises (and with other CNIs in the cloud), the pods typically run in an isolated overlay network and the CNI takes care of delivering traffic between pods. This is commonly done through encapsulation: the CNI converts pod-to-pod traffic into node-to-node traffic, taking care of encapsulating and de-encapsulating on both ends. This way, there is no need for extra configuration on the nodes and on the routers. + +The networking with hybrid nodes is unique because it presents a combination of both topologies - the EKS control plane and cloud nodes (with the VPC CNI) expect a flat network including nodes and pods, while the pods running on hybrid nodes are in an overlay network by using VXLAN for encapsulation (by default in Cilium). Pods running on hybrid nodes can reach the EKS control plane and pods running on cloud nodes assuming the on-premises network can route to the VPC. However, without routing for the pod CIDRs on the on-premises network, any traffic coming back to an on-premises pod IP will be dropped eventually if the network doesn't know how to reach the overlay network and route to the correct nodes. diff --git a/latest/ug/nodes/hybrid-nodes-concepts-traffic-flows.adoc b/latest/ug/nodes/hybrid-nodes-concepts-traffic-flows.adoc new file mode 100644 index 000000000..c97d1308d --- /dev/null +++ b/latest/ug/nodes/hybrid-nodes-concepts-traffic-flows.adoc @@ -0,0 +1,697 @@ +include::../attributes.txt[] + +[.topic] +[#hybrid-nodes-concepts-traffic-flows] += Network traffic flows for hybrid nodes +:info_titleabbrev: Traffic flows + +[abstract] +-- +Learn about the network traffic flows for EKS Hybrid Nodes. +-- + +This page details the network traffic flows for EKS Hybrid Nodes with diagrams showing the end-to-end network paths for the different traffic types. + +The following traffic flows are covered: + +* <> +* <> +* <> +* <> +* <> +* <> + +[#hybrid-nodes-concepts-traffic-flows-kubelet-to-cp] +== Hybrid node `kubelet` to EKS control plane + +image::images/hybrid-nodes-kubelet-to-cp-public.png[Hybrid node kubelet to EKS control plane] + +=== Request + +*1. `kubelet` Initiates Request* + +When the `kubelet` on a hybrid node needs to communicate with the EKS control plane (for example, to report node status or get pod specs), it uses the `kubeconfig` file provided during node registration. This `kubeconfig` has the API server endpoint URL (the Route53 DNS name) rather than direct IP addresses. + +The `kubelet` performs a DNS lookup for the endpoint (for example, `https://xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.gr7.us-west-2.eks.amazonaws.com`). In a public access cluster, this resolves to a public IP address (say `54.239.118.52`) that belongs to the EKS service running in {aws}. The `kubelet` then creates a secure HTTPS request to this endpoint. The initial packet looks like this: + +.... ++--------------------+---------------------+-----------------+ +| IP Header | TCP Header | Payload | +| Src: 10.80.0.2 | Src: 52390 (random) | | +| Dst: 54.239.118.52 | Dst: 443 | | ++--------------------+---------------------+-----------------+ +.... + +*2. Local Router Routing* + +Since the destination IP is a public IP address and not part of the local network, the `kubelet` sends this packet to its default gateway (the local on-premises router). The router examines the destination IP and determines it's a public IP address. + +For public traffic, the router typically forwards the packet to an internet gateway or border router that handles outbound traffic to the internet. This is omitted in the diagram and will depend on how your on-premises network is setup. The packet traverses your on-premises network infrastructure and eventually reaches your internet service provider's network. + +*3. Delivery to the EKS control plane* + +The packet travels across the public internet and transit networks until it reaches {aws}'s network. {aws}'s network routes the packet to the EKS service endpoint in the appropriate region. When the packet reaches the EKS service, it's forwarded to the actual EKS control plane for your cluster. + +This routing through the public internet is different from the private VPC-routed path that we'll see in other traffic flows. The key difference is that when using public access mode, traffic from on-premises `kubelet` (although not from pods) to the EKS control plane does not go through your VPC - it uses the global internet infrastructure instead. + +=== Response + +After the EKS control plane processes the `kubelet` request, it sends a response back: + +*3. EKS control plane sends response* + +The EKS control plane creates a response packet. This packet has the public IP as the source and the hybrid node's IP as the destination: + +.... ++--------------------+---------------------+-----------------+ +| IP Header | TCP Header | Payload | +| Src: 54.239.118.52 | Src: 443 | | +| Dst: 10.80.0.2 | Dst: 52390 | | ++--------------------+---------------------+-----------------+ +.... + +*2. Internet Routing* + +The response packet travels back through the internet, following the routing path determined by internet service providers, until it reaches your on-premises network edge router. + +*1. Local Delivery* + +Your on-premises router receives the packet and recognizes the destination IP (`10.80.0.2`) as belonging to your local network. It forwards the packet through your local network infrastructure until it reaches the target hybrid node, where the `kubelet` receives and processes the response. + +== Hybrid node `kube-proxy` to EKS control plane + +If you enable public endpoint access for the cluster, the return traffic uses the public internet. This traffiic originates from the `kube-proxy` on the hybrid node to the EKS control plane and follows the same path as the traffic from the `kubelet` to the EKS control plane. + +[#hybrid-nodes-concepts-traffic-flows-cp-to-kubelet] +== EKS control plane to hybrid node (`kubelet` server) + +image::images/hybrid-nodes-cp-to-kubelet.png[EKS control plane to hybrid node] + +=== Request + +*1. EKS Kubernetes API server initiates request* + +The EKS Kubernetes API server retrieves the node's IP address (`10.80.0.2`) from the node object's status. It then routes this request through its ENI in the VPC, as the destination IP belongs to the configured remote node CIDR (`10.80.0.0/16`). The initial packet looks like this: + +.... ++-----------------+---------------------+-----------------+ +| IP Header | TCP Header | Payload | +| Src: 10.0.0.132 | Src: 67493 (random) | | +| Dst: 10.80.0.2 | Dst: 10250 | | ++-----------------+---------------------+-----------------+ +.... + +*2. VPC network processing* + +The packet leaves the ENI and enters the VPC networking layer, where it's directed to the subnet's gateway for further routing. + +*3. VPC route table lookup* + +The VPC route table for the subnet containing the EKS control plane ENI has a specific route (the second one in the diagram) for the remote node CIDR. Based on this routing rule, the packet is directed to the VPC-to-onprem gateway. + +*4. Cross-boundary transit* + +The gateway transfers the packet across the cloud boundary through your established connection (such as Direct Connect or VPN) to your on-premises network. + +*5. On-premises network reception* + +The packet arrives at your local on-premises router that handles traffic for the subnet where your hybrid nodes are located. + +*6. Final delivery* + +The local router identifies that the destination IP (`10.80.0.2`) address belongs to its directly connected network and forwards the packet directly to the target hybrid node, where the `kubelet` receives and processes the request. + +=== Response + +After the hybrid node's `kubelet` processes the request, it sends back a response following the same path in reverse: + +.... ++-----------------+---------------------+-----------------+ +| IP Header | TCP Header | Payload | +| Src: 10.80.0.2 | Src: 10250 | | +| Dst: 10.0.0.132 | Dst: 67493 | | ++-----------------+---------------------+-----------------+ +.... + +*6. `kubelet` Sends Response* + +The `kubelet` on the hybrid node (`10.80.0.2`) creates a response packet with the original source IP as the destination. The destination doesn't belong to the local network so its sent to the host's default gateway, which is the local router. + +*5. Local Router Routing* + +The router determines that the destination IP (`10.0.0.132`) belongs to `10.0.0.0/16`, which has a route pointing to the gateway connecting to {aws}. + +*4. Cross-Boundary Return* + +The packet travels back through the same on-premises to VPC connection (such as Direct Connect or VPN), crossing the cloud boundary in the reverse direction. + +*3. VPC Routing* + +When the packet arrives in the VPC, the route tables identify that the destination IP belongs to a VPC CIDR. The packet routes within the VPC. + +*2. VPC Network Delivery* + +The VPC networking layer forwards the packet to the subnet with the EKS control plane ENI (`10.0.0.132`). + +*1. ENI Reception* + +The packet reaches the EKS control plane ENI attached to the Kubernetes API server, completing the round trip. + +[#hybrid-nodes-concepts-traffic-flows-pods-to-cp] +== Pods running on hybrid nodes to EKS control plane + +image::images/hybrid-nodes-pod-to-cp.png[Pods running on hybrid nodes to EKS control plane] + +=== Without CNI NAT + +=== Request + +Pods generally talk to the Kubernetes API server through the `kubernetes` service. The service IP is the first IP of the cluster's service CIDR. This convention allows pods that need to run before CoreDNS is available to reach the API server, for example, the CNI. Requests leave the pod with the service IP as the destination. For example, if the service CIDR is `172.16.0.0/16`, the service IP will be `172.16.0.1`. + +*1. Pod Initiates Request* + +The pod sends a request to the `kubernetes` service IP (`172.16.0.1`) on the API server port (443) from a random source port. The packet looks like this: + +.... ++-----------------+---------------------+-----------------+ +| IP Header | TCP Header | Payload | +| Src: 10.85.1.56 | Src: 67493 (random) | | +| Dst: 172.16.0.1 | Dst: 443 | | ++-----------------+---------------------+-----------------+ +.... + +*2. CNI Processing* + +The CNI detects that the destination IP doesn't belong to any pod CIDR it manages. Since *outgoing NAT is disabled*, the CNI passes the packet to the host network stack without modifying it. + +*3. Node Network Processing* + +The packet enters the node's network stack where `netfilter` hooks trigger the `iptables` rules set by kube-proxy. Several rules apply in the following order: + +. The packet first hits the `KUBE-SERVICES` chain, which contains rules matching each service's ClusterIP and port. +. The matching rule jumps to the `KUBE-SVC-XXX` chain for the `kubernetes` service (packets destined for `172.16.0.1:443`), which contains load balancing rules. +. The load balancing rule randomly selects one of the `KUBE-SEP-XXX` chains for the control plane ENI IPs (`10.0.0.132` or `10.0.1.23`). +. The selected `KUBE-SEP-XXX` chain has the actual rule that changes the destination IP from the service IP to the selected IP. This is called Destination Network Address Translation (DNAT). + +After these rules are applied, assuming that the selected EKS control plane ENI's IP is `10.0.0.132`, the packet looks like this: + +.... ++-----------------+---------------------+-----------------+ +| IP Header | TCP Header | Payload | +| Src: 10.85.1.56 | Src: 67493 (random) | | +| Dst: 10.0.0.132 | Dst: 443 | | ++-----------------+---------------------+-----------------+ +.... + +The node forwards the packet to its default gateway because the destination IP is not in the local network. + +*4. Local Router Routing* + +The local router determines that the destination IP (`10.0.0.132`) belongs to the VPC CIDR (`10.0.0.0/16`) and forwards it to the gateway connecting to {aws}. + +*5. Cross-Boundary Transit* + +The packet travels through your established connection (such as Direct Connect or VPN) across the cloud boundary to the VPC. + +*6. VPC Network Delivery* + +The VPC networking layer routes the packet to the correct subnet where the EKS control plane ENI (`10.0.0.132`) is located. + +*7. ENI Reception* + +The packet reaches the EKS control plane ENI attached to the Kubernetes API server. + +=== Response + +After the EKS control plane processes the request, it sends a response back to the pod: + +*7. API Server Sends Response* + +The EKS Kubernetes API server creates a response packet with the original source IP as the destination. The packet looks like this: + +.... ++-----------------+---------------------+-----------------+ +| IP Header | TCP Header | Payload | +| Src: 10.0.0.132 | Src: 443 | | +| Dst: 10.85.1.56 | Dst: 67493 | | ++-----------------+---------------------+-----------------+ +.... + +Because the destination IP belongs to the configured remote pod CIDR (`10.85.0.0/16`), it sends it through its ENI in the VPC with the subnet's router as the next hop. + +*6. VPC Routing* + +The VPC route table contains an entry for the remote pod CIDR (`10.85.0.0/16`), directing this traffic to the VPC-to-onprem gateway. + +*5. Cross-Boundary Transit* + +The gateway transfers the packet across the cloud boundary through your established connection (such as Direct Connect or VPN) to your on-premises network. + +*4. On-Premises Network Reception* + +The packet arrives at your local on-premises router. + +*3. Delivery to node* + +The router's table has an entry for `10.85.1.0/24` with `10.80.0.2` as the next hop, delivering the packet to our node. + +*2. Node Network Processing* + +As the packet is processed by the node's network stack, `conntrack` (a part of `netfilter`) matches the packet with the connection the pod initially establish. Since DNAT was originally applied, `conntrack` reverses the DNAT by rewriting the source IP from the EKS control plane ENI's IP to the `kubernetes` service IP: + +.... ++-----------------+---------------------+-----------------+ +| IP Header | TCP Header | Payload | +| Src: 172.16.0.1 | Src: 443 | | +| Dst: 10.85.1.56 | Dst: 67493 | | ++-----------------+---------------------+-----------------+ +.... + +*1. CNI Processing* + +The CNI identifies that the destination IP belongs to a pod in its network and delivers the packet to the correct pod network namespace. + +This flow showcases why Remote Pod CIDRs must be properly routable from the VPC all the way to the specific node hosting each pod - the entire return path depends on proper routing of pod IPs across both cloud and on-premises networks. + +=== With CNI NAT + +This flow is very similar to the one _without CNI NAT_, but with one key difference: the CNI applies source NAT (SNAT) to the packet before sending it to the node's network stack. This changes the source IP of the packet to the node's IP, allowing the packet to be routed back to the node without requiring additional routing configuration. + +=== Request + +*1. Pod Initiates Request* + +The pod sends a request to the `kubernetes` service IP (`172.16.0.1`) on the EKS Kubernetes API server port (443) from a random source port. The packet looks like this: + +.... ++-----------------+---------------------+-----------------+ +| IP Header | TCP Header | Payload | +| Src: 10.85.1.56 | Src: 67493 (random) | | +| Dst: 172.16.0.1 | Dst: 443 | | ++-----------------+---------------------+-----------------+ +.... + +*2. CNI Processing* + +The CNI detects that the destination IP doesn't belong to any pod CIDR it manages. Since *outgoing NAT is enabled*, the CNI applies SNAT to the packet, changing the source IP to the node's IP before passing it to the node's network stack: + +.... ++-----------------+---------------------+-----------------+ +| IP Header | TCP Header | Payload | +| Src: 10.80.0.2 | Src: 67493 (random) | | +| Dst: 172.16.0.1 | Dst: 443 | | ++-----------------+---------------------+-----------------+ +.... + +Note: CNI and `iptables` are shown in the example as separate blocks for clarity, but in practice, it's possible that some CNIs use `iptables` to apply NAT. + +*3. Node Network Processing* + +Here the `iptables` rules set by `kube-proxy` behave the same as in the previous example, load balancing the packet to one of the EKS control plane ENIs. The packet now looks like this: + +.... ++-----------------+---------------------+-----------------+ +| IP Header | TCP Header | Payload | +| Src: 10.80.0.2 | Src: 67493 (random) | | +| Dst: 10.0.0.132 | Dst: 443 | | ++-----------------+---------------------+-----------------+ +.... + +The node forwards the packet to its default gateway because the destination IP is not in the local network. + +*4. Local Router Routing* + +The local router determines that the destination IP (`10.0.0.132`) belongs to the VPC CIDR (`10.0.0.0/16`) and forwards it to the gateway connecting to {aws}. + +*5. Cross-Boundary Transit* + +The packet travels through your established connection (such as Direct Connect or VPN) across the cloud boundary to the VPC. + +*6. VPC Network Delivery* + +The VPC networking layer routes the packet to the correct subnet where the EKS control plane ENI (`10.0.0.132`) is located. + +*7. ENI Reception* + +The packet reaches the EKS control plane ENI attached to the Kubernetes API server. + +=== Response + +After the EKS control plane processes the request, it sends a response back to the pod: + +*7. API Server Sends Response* + +The EKS Kubernetes API server creates a response packet with the original source IP as the destination. The packet looks like this: + +.... ++-----------------+---------------------+-----------------+ +| IP Header | TCP Header | Payload | +| Src: 10.0.0.132 | Src: 443 | | +| Dst: 10.80.0.2 | Dst: 67493 | | ++-----------------+---------------------+-----------------+ +.... + +Because the destination IP belongs to the configured remote node CIDR (`10.80.0.0/16`), it sends it through its ENI in the VPC with the subnet's router as the next hop. + +*6. VPC Routing* + +The VPC route table contains an entry for the remote node CIDR (`10.80.0.0/16`), directing this traffic to the VPC-to-onprem gateway. + +*5. Cross-Boundary Transit* + +The gateway transfers the packet across the cloud boundary through your established connection (such as Direct Connect or VPN) to your on-premises network. + +*4. On-Premises Network Reception* + +The packet arrives at your local on-premises router. + +*3. Delivery to node* + +The local router identifies that the destination IP (`10.80.0.2`) address belongs to its directly connected network and forwards the packet directly to the target hybrid node. + +*2. Node Network Processing* + +As the packet is processed by the node's network stack, `conntrack` (a part of `netfilter`) matches the packet with the connection the pod initially establish and since DNAT was originally applied, it reverses this by rewriting the source IP from the EKS control plane ENI's IP to the `kubernetes` service IP: + +.... ++-----------------+---------------------+-----------------+ +| IP Header | TCP Header | Payload | +| Src: 172.16.0.1 | Src: 443 | | +| Dst: 10.80.0.2 | Dst: 67493 | | ++-----------------+---------------------+-----------------+ +.... + +*1. CNI Processing* + +The CNI identifies this packet belongs to a connection where it has previously applied SNAT. It reverses the SNAT, changing the destination IP back to the pod's IP: + +.... ++-----------------+---------------------+-----------------+ +| IP Header | TCP Header | Payload | +| Src: 172.16.0.1 | Src: 443 | | +| Dst: 10.85.1.56 | Dst: 67493 | | ++-----------------+---------------------+-----------------+ +.... + +The CNI detects the destination IP belongs to a pod in its network and delivers the packet to the correct pod network namespace. + +This flow showcases how CNI NAT-ing can simplify configuration by allowing packets to be routed back to the node without requiring additional routing for the pod CIDRs. + +[#hybrid-nodes-concepts-traffic-flows-cp-to-pod] +== EKS control plane to pods running on a hybrid node (webhooks) + +image::images/hybrid-nodes-cp-to-pod.png[EKS control plane to pods running on a hybrid node] + +This traffic pattern is most commonly seen with webhooks, where the EKS control plane needs to directly initiate connections to webhook servers running in pods on hybrid nodes. Examples include validating and mutating admission webhooks, which are called by the API server during resource validation or mutation processes. + +=== Request + +*1. EKS Kubernetes API server initiates request* + +When a webhook is configured in the cluster and a relevant API operation triggers it, the EKS Kubernetes API server needs to make a direct connection to the webhook server pod. The API server first looks up the pod's IP address from the Service or Endpoint resource associated with the webhook. + +Assuming the webhook pod is running on a hybrid node with IP `10.85.1.23`, the EKS Kubernetes API server creates an HTTPS request to the webhook endpoint. The initial packet is sent through the EKS control plane ENI in your VPC because the destination IP `10.85.1.23` belongs to the configured remote pod CIDR (`10.85.0.0/16`). The packet looks like this: + +.... ++-----------------+---------------------+-----------------+ +| IP Header | TCP Header | Payload | +| Src: 10.0.0.132 | Src: 41892 (random) | | +| Dst: 10.85.1.23 | Dst: 8443 | | ++-----------------+---------------------+-----------------+ +.... + +*2. VPC Network Processing* + +The packet leaves the EKS control plane ENI and enters the VPC networking layer with the subnet's router as the next hop. + +*3. VPC Route Table Lookup* + +The VPC route table for the subnet containing the EKS control plane ENI contains a specific route for the remote pod CIDR (`10.85.0.0/16`). This routing rule directs the packet to the VPC-to-onprem gateway (for example, a Virtual Private Gateway for Direct Connect or VPN connections): + +.... +Destination Target +10.0.0.0/16 local +10.85.0.0/16 vgw-id (VPC-to-onprem gateway) +.... + +*4. Cross-Boundary Transit* + +The gateway transfers the packet across the cloud boundary through your established connection (such as Direct Connect or VPN) to your on-premises network. The packet maintains its original source and destination IP addresses as it traverses this connection. + +*5. On-Premises Network Reception* + +The packet arrives at your local on-premises router. The router consults its routing table to determine how to reach the 10.85.1.23 address. For this to work, your on-premises network must have routes for the pod CIDRs that direct traffic to the appropriate hybrid node. + +In this case, the router's route table contains an entry indicating that the `10.85.1.0/24` subnet is reachable through the hybrid node with IP `10.80.0.2`: + +.... +Destination Next Hop +10.85.1.0/24 10.80.0.2 +.... + +*6. Delivery to node* + +Based on the routing table entry, the router forwards the packet to the hybrid node (`10.80.0.2`). When the packet arrives at the node, it looks the same as when the EKS Kubernetes API server sent it, with the destination IP still being the pod's IP. + +*7. CNI Processing* + +The node's network stack receives the packet and, seeing that the destination IP is not the node's own IP, passes it to the CNI for processing. The CNI identifies that the destination IP belongs to a pod running locally on this node and forwards the packet to the correct pod through the appropriate virtual interfaces: + +.... +Original packet -> node routing -> CNI -> Pod's network namespace +.... + +The webhook server in the pod receives the request and processes it. + +=== Response + +After the webhook pod processes the request, it sends back a response following the same path in reverse: + +*7. Pod Sends Response* + +The webhook pod creates a response packet with its own IP as the source and the original requester (the EKS control plane ENI) as the destination: + +.... ++-----------------+---------------------+-----------------+ +| IP Header | TCP Header | Payload | +| Src: 10.85.1.23 | Src: 8443 | | +| Dst: 10.0.0.132 | Dst: 41892 | | ++-----------------+---------------------+-----------------+ +.... + +The CNI identifies that this packet goes to an external network (not a local pod) and passes the packet to the node's network stack with the original source IP preserved. + +*6. Node Network Processing* + +The node determines that the destination IP (`10.0.0.132`) is not in the local network and forwards the packet to its default gateway (the local router). + +*5. Local Router Routing* + +The local router consults its routing table and determines that the destination IP (`10.0.0.132`) belongs to the VPC CIDR (`10.0.0.0/16`). It forwards the packet to the gateway connecting to {aws}. + +*4. Cross-Boundary Transit* + +The packet travels back through the same on-premises to VPC connection, crossing the cloud boundary in the reverse direction. + +*3. VPC Routing* + +When the packet arrives in the VPC, the route tables identify that the destination IP belongs to a subnet within the VPC. The packet is routed accordingly within the VPC. + +*2. and 1. EKS control plane ENI Reception* + +The packet reaches the ENI attached to the EKS Kubernetes API server, completing the round trip. The API server receives the webhook response and continues processing the original API request based on this response. + +This traffic flow demonstrates why remote pod CIDRs must be properly configured and routed: + +* The VPC must have routes for the remote pod CIDRs pointing to the on-premises gateway +* Your on-premises network must have routes for pod CIDRs that direct traffic to the specific nodes hosting those pods +* Without this routing configuration, webhooks and other similar services running in pods on hybrid nodes would not be reachable from the EKS control plane. + +[#hybrid-nodes-concepts-traffic-flows-pod-to-pod] +== Pod-to-Pod running on hybrid nodes + +image::images/hybrid-nodes-pod-to-pod.png[Pod-to Pod running on hybrid nodes] + +This section explains how pods running on different hybrid nodes communicate with each other. This example assumes your CNI uses VXLAN for encapsulation, which is common for CNIs such as Cilium or Calico. The overall process is similar for other encapsulation protocols such as Geneve or IP-in-IP. + +=== Request + +*1. Pod A Initiates Communication* + +Pod A (`10.85.1.56`) on Node 1 wants to send traffic to Pod B (`10.85.2.67`) on Node 2. The initial packet looks like this: + +.... ++------------------+-----------------+-------------+-----------------+ +| Ethernet Header | IP Header | TCP/UDP | Payload | +| Src: Pod A MAC | Src: 10.85.1.56 | Src: 43721 | | +| Dst: Gateway MAC | Dst: 10.85.2.67 | Dst: 8080 | | ++------------------+-----------------+-------------+-----------------+ +.... + +*2. CNI Intercepts and Processes the Packet* + +When Pod A's packet leaves its network namespace, the CNI intercepts it. The CNI consults its routing table and determines: - The destination IP (`10.85.2.67`) belongs to the pod CIDR - This IP is not on the local node but belongs to Node 2 (`10.80.0.3`) - The packet needs to be encapsulated with VXLAN. + +The decision to encapsulate is critical because the underlying physical network doesn't know how to route pod CIDRs directly - it only knows how to route traffic between node IPs. + +The CNI encapsulates the entire original packet inside a VXLAN frame. This effectively creates a "`packet within a packet`" with new headers: + +.... ++-----------------+----------------+--------------+------------+---------------------------+ +| Outer Ethernet | Outer IP | Outer UDP | VXLAN | Original Pod-to-Pod | +| Src: Node1 MAC | Src: 10.80.0.2 | Src: Random | VNI: 42 | Packet (unchanged | +| Dst: Router MAC | Dst: 10.80.0.3 | Dst: 8472 | | from above) | ++-----------------+----------------+--------------+------------+---------------------------+ +.... + +Key points about this encapsulation: - The outer packet is addressed from Node 1 (`10.80.0.2`) to Node 2 (`10.80.0.3`) - UDP port `8472` is the VXLAN port Cilium uses by default - The VXLAN Network Identifier (VNI) identifies which overlay network this packet belongs to - The entire original packet (with Pod A's IP as source and Pod B's IP as destination) is preserved intact inside + +The encapsulated packet now enters the regular networking stack of Node 1 and is processed in the same way as any other packet: + +. *Node Network Processing*: Node 1's network stack routes the packet based on its destination (`10.80.0.3`) +. *Local Network Delivery*: +* If both nodes are on the same Layer 2 network, the packet is sent directly to Node 2 +* If they're on different subnets, the packet is forwarded to the local router first +. *Router Handling*: The router forwards the packet based on its routing table, delivering it to Node 2 + +*3. Receiving Node Processing* + +When the encapsulated packet arrives at Node 2 (`10.80.0.3`): + +. The node's network stack receives it and identifies it as a VXLAN packet (UDP port `4789`) +. The packet is passed to the CNI's VXLAN interface for processing + +*4. VXLAN Decapsulation* + +The CNI on Node 2 processes the VXLAN packet: + +. It strips away the outer headers (Ethernet, IP, UDP, and VXLAN) +. It extracts the original inner packet +. The packet is now back to its original form: + +.... ++------------------+-----------------+-------------+-----------------+ +| Ethernet Header | IP Header | TCP/UDP | Payload | +| Src: Pod A MAC | Src: 10.85.1.56 | Src: 43721 | | +| Dst: Gateway MAC | Dst: 10.85.2.67 | Dst: 8080 | | ++------------------+-----------------+-------------+-----------------+ +.... + +The CNI on Node 2 examines the destination IP (`10.85.2.67`) and: + +. Identifies that this IP belongs to a local pod +. Routes the packet through the appropriate virtual interfaces +. Delivers the packet to Pod B's network namespace + +=== Response + +When Pod B responds to Pod A, the entire process happens in reverse: + +[start=4] +. Pod B sends a packet to Pod A (`10.85.1.56`) +. Node 2's CNI encapsulates it with VXLAN, setting the destination to Node 1 (`10.80.0.2`) +. The encapsulated packet is delivered to Node 1 +. Node 1's CNI decapsulates it and delivers the original response to Pod A + +[#hybrid-nodes-concepts-traffic-flows-east-west] +== Pods on cloud nodes to pods on hybrid nodes (east-west traffic) + +image::images/hybrid-nodes-east-west.png[Pods on cloud nodes to pods on hybrid nodes] + +=== Request + +*1. Pod A Initiates Communication* + +Pod A (`10.0.0.56`) on the EC2 Node wants to send traffic to Pod B (`10.85.1.56`) on the Hybrid Node. The initial packet looks like this: + +.... ++-----------------+---------------------+-----------------+ +| IP Header | TCP Header | Payload | +| Src: 10.0.0.56 | Src: 52390 (random) | | +| Dst: 10.85.1.56 | Dst: 8080 | | ++-----------------+---------------------+-----------------+ +.... + +With the VPC CNI, Pod A has an IP from the VPC CIDR and is directly attached to an ENI on the EC2 instance. The pod's network namespace is connected to the VPC network, so the packet enters the VPC routing infrastructure directly. + +*2. VPC Routing* + +The VPC route table contains a specific route for the Remote Pod CIDR (`10.85.0.0/16`), directing this traffic to the VPC-to-onprem gateway: + +.... +Destination Target +10.0.0.0/16 local +10.85.0.0/16 vgw-id (VPC-to-onprem gateway) +.... + +Based on this routing rule, the packet is directed toward the gateway connecting to your on-premises network. + +*3. Cross-Boundary Transit* + +The gateway transfers the packet across the cloud boundary through your established connection (such as Direct Connect or VPN) to your on-premises network. The packet maintains its original source and destination IP addresses throughout this transit. + +*4. On-Premises Network Reception* + +The packet arrives at your local on-premises router. The router consults its routing table to determine the next hop for reaching the 10.85.1.56 address. Your on-premises router must have routes for the pod CIDRs that direct traffic to the appropriate hybrid node. + +The router's table has an entry indicating that the `10.85.1.0/24` subnet is reachable through the hybrid node with IP `10.80.0.2`: + +.... +Destination Next Hop +10.85.1.0/24 10.80.0.2 +.... + +*5. Node Network Processing* + +The router forwards the packet to the hybrid node (`10.80.0.2`). When the packet arrives at the node, it still has Pod A's IP as the source and Pod B's IP as the destination. + +*6. CNI Processing* + +The node's network stack receives the packet and, seeing that the destination IP is not its own, passes it to the CNI for processing. The CNI identifies that the destination IP belongs to a pod running locally on this node and forwards the packet to the correct pod through the appropriate virtual interfaces: + +.... +Original packet -> node routing -> CNI -> Pod B's network namespace +.... + +Pod B receives the packet and processes it as needed. + +=== Response + +*6. Pod B Sends Response* + +Pod B creates a response packet with its own IP as the source and Pod A's IP as the destination: + +.... ++-----------------+---------------------+-----------------+ +| IP Header | TCP Header | Payload | +| Src: 10.85.1.56 | Src: 8080 | | +| Dst: 10.0.0.56 | Dst: 52390 | | ++-----------------+---------------------+-----------------+ +.... + +The CNI identifies that this packet is destined for an external network and passes it to the node's network stack. + +*5. Node Network Processing* + +The node determines that the destination IP (`10.0.0.56`) does not belong to the local network and forwards the packet to its default gateway (the local router). + +*4. Local Router Routing* + +The local router consults its routing table and determines that the destination IP (`10.0.0.56`) belongs to the VPC CIDR (`10.0.0.0/16`). It forwards the packet to the gateway connecting to {aws}. + +*3. Cross-Boundary Transit* + +The packet travels back through the same on-premises to VPC connection, crossing the cloud boundary in the reverse direction. + +*2. VPC Routing* + +When the packet arrives in the VPC, the routing system identifies that the destination IP belongs to a subnet within the VPC. The packet is routed through the VPC network toward the EC2 instance hosting Pod A. + +*1. Pod A Receives Response* + +The packet arrives at the EC2 instance and is delivered directly to Pod A through its attached ENI. Since the VPC CNI doesn't use overlay networking for pods in the VPC, no additional decapsulation is needed - the packet arrives with its original headers intact. + +This east-west traffic flow demonstrates why remote pod CIDRs must be properly configured and routable from both directions: + +* The VPC must have routes for the remote pod CIDRs pointing to the on-premises gateway +* Your on-premises network must have routes for pod CIDRs that direct traffic to the specific nodes hosting those pods. \ No newline at end of file diff --git a/latest/ug/nodes/hybrid-nodes-concepts.adoc b/latest/ug/nodes/hybrid-nodes-concepts.adoc new file mode 100644 index 000000000..822fa01e9 --- /dev/null +++ b/latest/ug/nodes/hybrid-nodes-concepts.adoc @@ -0,0 +1,26 @@ +include::../attributes.txt[] + +[.topic] +[#hybrid-nodes-concepts] += Concepts for hybrid nodes +:info_titleabbrev: How it works + +[abstract] +-- +Learn about EKS Hybrid Nodes technical concepts and how it works +-- + +With _Amazon EKS Hybrid Nodes_, you join physical or virtual machines running in on-premises or edge environments to Amazon EKS clusters running in the {aws} Cloud. This approach brings many benefits, but also introduces new networking concepts and architectures for those familiar with running Kubernetes clusters in a single network environment. + +The following sections dive deep into the Kubernetes and networking concepts for EKS Hybrid Nodes and details how traffic flows through the hybrid architecture. These sections require that you are familiar with basic Kubernetes networking knowledge, such as the concepts of pods, nodes, services, Kubernetes control plane, kubelet and kube-proxy. + +We recommend reading these pages in order, starting with the <>, then the <>, and finally the <>. + +[.topiclist] +[[Topic List]] + +include::hybrid-nodes-concepts-networking.adoc[leveloffset=+1] + +include::hybrid-nodes-concepts-kubernetes.adoc[leveloffset=+1] + +include::hybrid-nodes-concepts-traffic-flows.adoc[leveloffset=+1] \ No newline at end of file diff --git a/latest/ug/nodes/hybrid-nodes-configure.adoc b/latest/ug/nodes/hybrid-nodes-configure.adoc new file mode 100644 index 000000000..b965e3208 --- /dev/null +++ b/latest/ug/nodes/hybrid-nodes-configure.adoc @@ -0,0 +1,46 @@ +include::../attributes.txt[] + +[.topic] +[#hybrid-nodes-configure] += Configure application networking, add-ons, and webhooks for hybrid nodes +:info_titleabbrev: Configure + +[abstract] +-- +Learn how to configure CNI, BGP, application networking, add-ons, webhooks, and proxy settings for hybrid nodes. +-- + +After you create an EKS cluster for hybrid nodes, configure additional capabilities for application networking (CNI, BGP, Ingress, Load Balancing, Network Policies), add-ons, webhooks, and proxy settings. For the complete list of the EKS and community add-ons that are compatible with hybrid nodes, see <>. + +*EKS cluster insights* +EKS includes insight checks for misconfigurations in your hybrid nodes setup that could impair functionality of your cluster or workloads. For more information on cluster insights, see <>. + +The following lists the common capabilities and add-ons that you can use with hybrid nodes: + +* *Container Networking Interface (CNI)*: {aws} supports link:https://docs.cilium.io/en/stable/index.html[Cilium] as the CNI for hybrid nodes. For more information, see <>. Note that the {aws} VPC CNI can't be used with hybrid nodes. +* *CoreDNS and `kube-proxy`*: CoreDNS and `kube-proxy` are installed automatically when hybrid nodes join the EKS cluster. These add-ons can be managed as EKS add-ons after cluster creation. +* *Ingress and Load Balancing*: You can use the {aws} Load Balancer Controller and Application Load Balancer (ALB) or Network Load Balancer (NLB) with the target type `ip` for workloads running on hybrid nodes. {aws} supports Cilium's built-in Ingress, Gateway, and Kubernetes Service load balancing features for workloads running on hybrid nodes. For more information, see <> and <>. +* *Metrics*: You can use Amazon Managed Service for Prometheus (AMP) agent-less scrapers, {aws} Distro for Open Telemetry (ADOT), and the Amazon CloudWatch Observability Agent with hybrid nodes. To use AMP agent-less scrapers for pod metrics on hybrid nodes, your pods must be accessible from the VPC that you use for the EKS cluster. +* *Logs*: You can enable EKS control plane logging for hybrid nodes-enabled clusters. You can use the ADOT EKS add-on and the Amazon CloudWatch Observability Agent EKS add-on for hybrid node and pod logging. +* *Pod Identities and IRSA*: You can use EKS Pod Identities and IAM Roles for Service Accounts (IRSA) with applications running on hybrid nodes to enable granular access for your pods running on hybrid nodes with other {aws} services. +* *Webhooks*: If you are running webhooks, see <> for considerations and steps to optionally run webhooks on cloud nodes if you cannot make your on-premises pod networks routable. +* *Proxy*: If you are using a proxy server in your on-premises environment for traffic leaving your data center or edge environment, you can configure your hybrid nodes and cluster to use your proxy server. For more information, see <>. + +[.topiclist] +[[Topic List]] + +include::hybrid-nodes-cni.adoc[leveloffset=+1] + +include::hybrid-nodes-add-ons.adoc[leveloffset=+1] + +include::hybrid-nodes-webhooks.adoc[leveloffset=+1] + +include::hybrid-nodes-proxy.adoc[leveloffset=+1] + +include::hybrid-nodes-cilium-bgp.adoc[leveloffset=+1] + +include::hybrid-nodes-ingress.adoc[leveloffset=+1] + +include::hybrid-nodes-load-balancing.adoc[leveloffset=+1] + +include::hybrid-nodes-network-policy.adoc[leveloffset=+1] diff --git a/latest/ug/nodes/hybrid-nodes-creds.adoc b/latest/ug/nodes/hybrid-nodes-creds.adoc index f64d1199f..a1a663778 100644 --- a/latest/ug/nodes/hybrid-nodes-creds.adoc +++ b/latest/ug/nodes/hybrid-nodes-creds.adoc @@ -1,6 +1,7 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[hybrid-nodes-creds,hybrid-nodes-creds.title]] +[#hybrid-nodes-creds] = Prepare credentials for hybrid nodes :info_titleabbrev: Prepare credentials @@ -8,15 +9,14 @@ -- Prepare credentials to authenticate hybrid nodes with Amazon EKS clusters -- -include::../attributes.txt[] - [abstract] -- Prepare credentials to authenticate hybrid nodes with Amazon EKS clusters -- -Amazon EKS Hybrid Nodes use temporary IAM credentials provisioned by {aws} SSM hybrid activations or {aws} IAM Roles Anywhere to authenticate with the Amazon EKS cluster. You must use either {aws} SSM hybrid activations or {aws} IAM Roles Anywhere with the Amazon EKS Hybrid Nodes CLI (`nodeadm`). You should not use both {aws} SSM hybrid activations and {aws} IAM Roles Anywhere. It is recommended to use {aws} SSM hybrid activations if you do not have existing Public Key Infrastructure (PKI) with a Certificate Authority (CA) and certificates for your on-premises environments. If you do have existing PKI and certificates on-premises, use {aws} IAM Roles Anywhere. +Amazon EKS Hybrid Nodes use temporary IAM credentials provisioned by {aws} SSM hybrid activations or {aws} IAM Roles Anywhere to authenticate with the Amazon EKS cluster. You must use either {aws} SSM hybrid activations or {aws} IAM Roles Anywhere with the Amazon EKS Hybrid Nodes CLI (`nodeadm`). You should not use both {aws} SSM hybrid activations and {aws} IAM Roles Anywhere. We recommend that you use {aws} SSM hybrid activations if you do not have existing Public Key Infrastructure (PKI) with a Certificate Authority (CA) and certificates for your on-premises environments. If you do have existing PKI and certificates on-premises, use {aws} IAM Roles Anywhere. +[#hybrid-nodes-role] == Hybrid Nodes IAM Role Before you can connect hybrid nodes to your Amazon EKS cluster, you must create an IAM role that will be used with {aws} SSM hybrid activations or {aws} IAM Roles Anywhere for your hybrid nodes credentials. After cluster creation, you will use this role with an Amazon EKS access entry or `aws-auth` ConfigMap entry to map the IAM role to Kubernetes Role-Based Access Control (RBAC). For more information on associating the Hybrid Nodes IAM role with Kubernetes RBAC, see <>. @@ -31,34 +31,36 @@ link:aws-managed-policy/latest/reference/AmazonSSMManagedInstanceCore.html[,Amaz * If using {aws} SSM, permissions to use the `ssm:DeregisterManagedInstance` action and `ssm:DescribeInstanceInformation` action for `nodeadm uninstall` to deregister instances. * (Optional) Permissions for the Amazon EKS Pod Identity Agent to use the `eks-auth:AssumeRoleForPodIdentity` action to retrieve credentials for pods. +[#hybrid-nodes-ssm] == Setup {aws} SSM hybrid activations -Before setting up {aws} SSM hybrid activations, you must have a Hybrid Nodes IAM role created and configured. For more information, see <>. Follow the instructions at link:systems-manager/latest/userguide/hybrid-activation-managed-nodes.html[Create a hybrid activation to register nodes with Systems Manager,type="documentation"] in the {aws} Systems Manager User Guide to create an {aws} SSM hybrid activation for your hybrid nodes. The Activation Code and ID you receive is used with `nodeadm` when you register your hosts as hybrid nodes with your Amazon EKS cluster. You can come back to this step at a later point after you have created and prepared your Amazon EKS clusters for hybrid nodes. +Before setting up {aws} SSM hybrid activations, you must have a Hybrid Nodes IAM role created and configured. For more information, see <>. Follow the instructions at link:systems-manager/latest/userguide/hybrid-activation-managed-nodes.html[Create a hybrid activation to register nodes with Systems Manager,type="documentation"] in the {aws} Systems Manager User Guide to create an {aws} SSM hybrid activation for your hybrid nodes. The Activation Code and ID you receive is used with `nodeadm` when you register your hosts as hybrid nodes with your Amazon EKS cluster. You can come back to this step at a later point after you have created and prepared your Amazon EKS clusters for hybrid nodes. [IMPORTANT] ==== Systems Manager immediately returns the Activation Code and ID to the console or the command window, depending on how you created the activation. Copy this information and store it in a safe place. If you navigate away from the console or close the command window, you might lose this information. If you lose it, you must create a new activation. ==== -By default, {aws} SSM hybrid activations are active for 24 hours. You can alternatively specify an `--expiration-date` when you create your hybrid activation in timestamp format, such as `2024-08-01T00:00:00`. When you use {aws} SSM as your credential provider, the node name for your hybrid nodes is not configurable, and is auto-generated by {aws} SSM. You can view and manage the {aws} SSM Managed Instances in the {aws} Systems Manager console under Fleet Manager. You can register up to 1,000 standard link:systems-manager/latest/userguide/activations.html[hybrid-activated nodes,type="documentation"] per account per {aws} Region at no additional cost. However, registering more than 1,000 hybrid nodes requires that you activate the advanced-instances tier. There is a charge to use the advanced-instances tier that is not included in the https://aws.amazon.com/eks/pricing/[Amazon EKS Hybrid Nodes pricing]. For more information, see https://aws.amazon.com/systems-manager/pricing/[{aws} Systems Manager Pricing]. +By default, {aws} SSM hybrid activations are active for 24 hours. You can alternatively specify an `--expiration-date` when you create your hybrid activation in timestamp format, such as `2024-08-01T00:00:00`. When you use {aws} SSM as your credential provider, the node name for your hybrid nodes is not configurable, and is auto-generated by {aws} SSM. You can view and manage the {aws} SSM Managed Instances in the {aws} Systems Manager console under Fleet Manager. You can register up to 1,000 standard link:systems-manager/latest/userguide/activations.html[hybrid-activated nodes,type="documentation"] per account per {aws} Region at no additional cost. However, registering more than 1,000 hybrid nodes requires that you activate the advanced-instances tier. There is a charge to use the advanced-instances tier that is not included in the link:eks/pricing/[Amazon EKS Hybrid Nodes pricing,type="marketing"]. For more information, see link:systems-manager/pricing/[{aws} Systems Manager Pricing,type="marketing"]. See the example below for how to create an {aws} SSM hybrid activation with your Hybrid Nodes IAM role. When you use {aws} SSM hybrid activations for your hybrid nodes credentials, the names of your hybrid nodes will have the format `mi-012345678abcdefgh` and the temporary credentials provisioned by {aws} SSM are valid for 1 hour. You cannot alter the node name or credential duration when using {aws} SSM as your credential provider. The temporary credentials are automatically rotated by {aws} SSM and the rotation does not impact the status of your nodes or applications. -It is recommended to use one {aws} SSM hybrid activation per EKS cluster to scope the {aws} SSM `ssm:DeregisterManagedInstance` permission of the Hybrid Nodes IAM role to only be able to deregister instances that are associated with your {aws} SSM hybrid activation. In the example on this page, a tag with the EKS cluster ARN is used, which can be used to map your {aws} SSM hybrid activation to the EKS cluster. You can alternatively use your preferred tag and method of scoping the {aws} SSM permissions based on your permission boundaries and requirements. The `REGISTRATION_LIMIT` option in the command below is an integer used to limit the number of machines that can use the {aws} SSM hybrid activation (for example `10`) +We recommend that you use one {aws} SSM hybrid activation per EKS cluster to scope the {aws} SSM `ssm:DeregisterManagedInstance` permission of the Hybrid Nodes IAM role to only be able to deregister instances that are associated with your {aws} SSM hybrid activation. In the example on this page, a tag with the EKS cluster ARN is used, which can be used to map your {aws} SSM hybrid activation to the EKS cluster. You can alternatively use your preferred tag and method of scoping the {aws} SSM permissions based on your permission boundaries and requirements. The `REGISTRATION_LIMIT` option in the command below is an integer used to limit the number of machines that can use the {aws} SSM hybrid activation (for example `10`) -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- aws ssm create-activation \ --region AWS_REGION \ --default-instance-name eks-hybrid-nodes \ --description "Activation for EKS hybrid nodes" \ --iam-role AmazonEKSHybridNodesRole \ - --tags Key=EKSClusterARN,Value=arn:aws:eks:AWS_REGION:AWS_ACCOUNT_ID:cluster/CLUSTER_NAME \ + --tags Key=EKSClusterARN,Value={arn-aws}eks:AWS_REGION:AWS_ACCOUNT_ID:cluster/CLUSTER_NAME \ --registration-limit REGISTRATION_LIMIT ---- Review the instructions on link:systems-manager/latest/userguide/hybrid-activation-managed-nodes.html[Create a hybrid activation to register nodes with Systems Manager,type="documentation"] for more information about the available configuration settings for {aws} SSM hybrid activations. +[#hybrid-nodes-iam-roles-anywhere] == Setup {aws} IAM Roles Anywhere Follow the instructions at link:rolesanywhere/latest/userguide/getting-started.html[Getting started with IAM Roles Anywhere,type="documentation"] in the IAM Roles Anywhere User Guide to set up the trust anchor and profile you will use for temporary IAM credentials for your Hybrid Nodes IAM role. When you create your profile, you can create it without adding any roles. You can create this profile, return to these steps to create your Hybrid Nodes IAM role, and then add your role to your profile after it is created. You can alternatively use the {aws} CloudFormation steps later on this page to complete your IAM Roles Anywhere setup for hybrid nodes. @@ -69,6 +71,7 @@ You can configure the credential validity duration with {aws} IAM Roles Anywhere The per-machine certificates and keys you generate from your certificate authority (CA) must be placed in the `/etc/iam/pki` directory on each hybrid node with the file names `server.pem` for the certificate and `server.key` for the key. +[#hybrid-nodes-create-role] == Create the Hybrid Nodes IAM role To run the steps in this section, the IAM principal using the {aws} console or {aws} CLI must have the following permissions. @@ -81,6 +84,7 @@ To run the steps in this section, the IAM principal using the {aws} console or { ** `rolesanywhere:CreateProfile` ** `iam:PassRole` +[#hybrid-nodes-creds-cloudformation] === {aws} CloudFormation Install and configure the {aws} CLI, if you haven't already. See link:cli/latest/userguide/getting-started-install.html[Installing or updating to the last version of the {aws} CLI,type="documentation"]. @@ -91,7 +95,7 @@ The CloudFormation stack creates the Hybrid Nodes IAM Role with the permissions . Download the {aws} SSM CloudFormation template for hybrid nodes: + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- curl -OL 'https://raw.githubusercontent.com/aws/eks-hybrid/refs/heads/main/example/hybrid-ssm-cfn.yaml' ---- @@ -99,9 +103,9 @@ curl -OL 'https://raw.githubusercontent.com/aws/eks-hybrid/refs/heads/main/examp . Create a `cfn-ssm-parameters.json` with the following options: .. Replace `ROLE_NAME` with the name for your Hybrid Nodes IAM role. By default, the CloudFormation template uses `AmazonEKSHybridNodesRole` as the name of the role it creates if you do not specify a name. .. Replace `TAG_KEY` with the {aws} SSM resource tag key you used when creating your {aws} SSM hybrid activation. The combination of the tag key and tag value is used in the condition for the `ssm:DeregisterManagedInstance` to only allow the Hybrid Nodes IAM role to deregister the {aws} SSM managed instances that are associated with your {aws} SSM hybrid activation. In the CloudFormation template, `TAG_KEY` defaults to `EKSClusterARN`. -.. Replace `TAG_VALUE` with the {aws} SSM resource tag value you used when creating your {aws} SSM hybrid activation. The combination of the tag key and tag value is used in the condition for the `ssm:DeregisterManagedInstance` to only allow the Hybrid Nodes IAM role to deregister the {aws} SSM managed instances that are associated with your {aws} SSM hybrid activation. If you are using the default `TAG_KEY` of `EKSClusterARN`, then pass your EKS cluster ARN as the `TAG_VALUE`. EKS cluster ARNs have the format `arn:aws:eks:AWS_REGION:AWS_ACCOUNT_ID:cluster/CLUSTER_NAME`. +.. Replace `TAG_VALUE` with the {aws} SSM resource tag value you used when creating your {aws} SSM hybrid activation. The combination of the tag key and tag value is used in the condition for the `ssm:DeregisterManagedInstance` to only allow the Hybrid Nodes IAM role to deregister the {aws} SSM managed instances that are associated with your {aws} SSM hybrid activation. If you are using the default `TAG_KEY` of `EKSClusterARN`, then pass your EKS cluster ARN as the `TAG_VALUE`. EKS cluster ARNs have the format `{arn-aws}eks:AWS_REGION:AWS_ACCOUNT_ID:cluster/CLUSTER_NAME`. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,json,subs="verbatim,attributes"] ---- { "Parameters": { @@ -113,7 +117,7 @@ curl -OL 'https://raw.githubusercontent.com/aws/eks-hybrid/refs/heads/main/examp ---- . Deploy the CloudFormation stack. Replace `STACK_NAME` with your name for the CloudFormation stack. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- aws cloudformation deploy \ --stack-name STACK_NAME \ @@ -132,7 +136,7 @@ The CloudFormation stack creates the {aws} IAM Roles Anywhere trust anchor with .. Certificates issued from public CAs cannot be used as trust anchors. . Download the {aws} IAM Roles Anywhere CloudFormation template for hybrid nodes + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- curl -OL 'https://raw.githubusercontent.com/aws/eks-hybrid/refs/heads/main/example/hybrid-ira-cfn.yaml' ---- @@ -141,7 +145,7 @@ curl -OL 'https://raw.githubusercontent.com/aws/eks-hybrid/refs/heads/main/examp .. Replace `CERT_ATTRIBUTE` with the per-machine certificate attribute that uniquely identifies your host. The certificate attribute you use must match the nodeName you use for the `nodeadm` configuration when you connect hybrid nodes to your cluster. For more information, see the <>. By default, the CloudFormation template uses `${aws:PrincipalTag/x509Subject/CN}` as the `CERT_ATTRIBUTE`, which corresponds to the CN field of your per-machine certificates. You can alternatively pass `$(aws:PrincipalTag/x509SAN/Name/CN}` as your `CERT_ATTRIBUTE`. .. Replace `CA_CERT_BODY` with the certificate body of your CA without line breaks. The `CA_CERT_BODY` must be in Privacy Enhanced Mail (PEM) format. If you have a CA certificate in PEM format, remove the line breaks and BEGIN CERTIFICATE and END CERTIFICATE lines before placing the CA certificate body in your `cfn-iamra-parameters.json` file. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,json,subs="verbatim,attributes"] ---- { "Parameters": { @@ -153,7 +157,7 @@ curl -OL 'https://raw.githubusercontent.com/aws/eks-hybrid/refs/heads/main/examp ---- . Deploy the CloudFormation template. Replace `STACK_NAME` with your name for the CloudFormation stack. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- aws cloudformation deploy \ --stack-name STACK_NAME \ @@ -162,7 +166,8 @@ aws cloudformation deploy \ --capabilities CAPABILITY_NAMED_IAM ---- -*{aws} CLI* +[#hybrid-nodes-creds-awscli] +=== {aws} CLI Install and configure the {aws} CLI, if you haven't already. See link:cli/latest/userguide/getting-started-install.html[Installing or updating to the last version of the {aws} CLI,type="documentation"]. @@ -170,25 +175,14 @@ Install and configure the {aws} CLI, if you haven't already. See link:cli/latest . Create a file named `eks-describe-cluster-policy.json` with the following contents: + -[source,bash,subs="verbatim,attributes,quotes"] +[source,json,subs="verbatim,attributes"] ---- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "eks:DescribeCluster" - ], - "Resource": "*" - } - ] -} +include::iam_snippet:48a31bf0-c614-41f5-bdbb-3f0423e5e5ba[] ---- . Create the policy with the following command: + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- aws iam create-policy \ --policy-name EKSDescribeClusterPolicy \ @@ -201,34 +195,15 @@ aws iam create-policy \ .. Replace `AWS_REGION` with the {aws} Region for your {aws} SSM hybrid activation. .. Replace `AWS_ACCOUNT_ID` with your {aws} account ID. .. Replace `TAG_KEY` with the {aws} SSM resource tag key you used when creating your {aws} SSM hybrid activation. The combination of the tag key and tag value is used in the condition for the `ssm:DeregisterManagedInstance` to only allow the Hybrid Nodes IAM role to deregister the {aws} SSM managed instances that are associated with your {aws} SSM hybrid activation. In the CloudFormation template, `TAG_KEY` defaults to `EKSClusterARN`. -.. Replace `TAG_VALUE` with the {aws} SSM resource tag value you used when creating your {aws} SSM hybrid activation. The combination of the tag key and tag value is used in the condition for the `ssm:DeregisterManagedInstance` to only allow the Hybrid Nodes IAM role to deregister the {aws} SSM managed instances that are associated with your {aws} SSM hybrid activation. If you are using the default `TAG_KEY` of `EKSClusterARN`, then pass your EKS cluster ARN as the `TAG_VALUE`. EKS cluster ARNs have the format `arn:aws:eks:AWS_REGION:AWS_ACCOUNT_ID:cluster/CLUSTER_NAME`. +.. Replace `TAG_VALUE` with the {aws} SSM resource tag value you used when creating your {aws} SSM hybrid activation. The combination of the tag key and tag value is used in the condition for the `ssm:DeregisterManagedInstance` to only allow the Hybrid Nodes IAM role to deregister the {aws} SSM managed instances that are associated with your {aws} SSM hybrid activation. If you are using the default `TAG_KEY` of `EKSClusterARN`, then pass your EKS cluster ARN as the `TAG_VALUE`. EKS cluster ARNs have the format `{arn-aws}eks:AWS_REGION:AWS_ACCOUNT_ID:cluster/CLUSTER_NAME`. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,json,subs="verbatim,attributes"] ---- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": "ssm:DescribeInstanceInformation", - "Resource": "*" - }, - { - "Effect": "Allow", - "Action": "ssm:DeregisterManagedInstance", - "Resource": "arn:aws:ssm:AWS_REGION:AWS_ACCOUNT_ID:managed-instance/*", - "Condition": { - "StringEquals": { - "ssm:resourceTag/TAG_KEY": "TAG_VALUE" - } - } - } - ] -} +include::iam_snippet:827fee9e-ea0e-495c-bf24-6a74bc22256d[] ---- . Create the policy with the following command + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- aws iam create-policy \ --policy-name EKSHybridSSMPolicy \ @@ -236,33 +211,13 @@ aws iam create-policy \ ---- . Create a file named `eks-hybrid-ssm-trust.json`. Replace `AWS_REGION` with the {aws} Region of your {aws} SSM hybrid activation and `AWS_ACCOUNT_ID` with your {aws} account ID. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,json,subs="verbatim,attributes"] ---- -{ - "Version":"2012-10-17", - "Statement":[ - { - "Sid":"", - "Effect":"Allow", - "Principal":{ - "Service":"ssm.amazonaws.com" - }, - "Action":"sts:AssumeRole", - "Condition":{ - "StringEquals":{ - "aws:SourceAccount":"AWS_ACCOUNT_ID" - }, - "ArnEquals":{ - "aws:SourceArn":"arn:aws:ssm:AWS_REGION:AWS_ACCOUNT_ID:*" - } - } - } - ] -} +include::iam_snippet:40d662dd-812b-448e-976c-ade3581a5e5e[] ---- . Create the role with the following command. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- aws iam create-role \ --role-name AmazonEKSHybridNodesRole \ @@ -272,87 +227,53 @@ aws iam create-role \ . Attach the `EKSDescribeClusterPolicy` and the `EKSHybridSSMPolicy` you created in the previous steps. Replace `AWS_ACCOUNT_ID` with your {aws} account ID. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- aws iam attach-role-policy \ --role-name AmazonEKSHybridNodesRole \ - --policy-arn arn:aws:iam::AWS_ACCOUNT_ID:policy/EKSDescribeClusterPolicy + --policy-arn {arn-aws}iam::AWS_ACCOUNT_ID:policy/EKSDescribeClusterPolicy ---- + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- aws iam attach-role-policy \ --role-name AmazonEKSHybridNodesRole \ - --policy-arn arn:aws:iam::AWS_ACCOUNT_ID:policy/EKSHybridSSMPolicy + --policy-arn {arn-aws}iam::AWS_ACCOUNT_ID:policy/EKSHybridSSMPolicy ---- . Attach the `AmazonEC2ContainerRegistryPullOnly` and `AmazonSSMManagedInstanceCore` {aws} managed policies. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- aws iam attach-role-policy \ --role-name AmazonEKSHybridNodesRole \ - --policy-arn arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPullOnly + --policy-arn {arn-aws}iam::aws:policy/AmazonEC2ContainerRegistryPullOnly ---- + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- aws iam attach-role-policy \ --role-name AmazonEKSHybridNodesRole \ - --policy-arn arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore + --policy-arn {arn-aws}iam::aws:policy/AmazonSSMManagedInstanceCore ---- *Steps for {aws} IAM Roles Anywhere* -To use {aws} IAM Roles Anywhere, you must set up your {aws} IAM Roles Anywhere trust anchor before creating the Hybrid Nodes IAM Role. See <> for instructions. +To use {aws} IAM Roles Anywhere, you must set up your {aws} IAM Roles Anywhere trust anchor before creating the Hybrid Nodes IAM Role. See <> for instructions. -. Create a file named `eks-hybrid-iamra-trust.json`. Replace `TRUST_ANCHOR ARN` with the ARN of the trust anchor you created in the <> steps. The condition in this trust policy restricts the ability of {aws} IAM Roles Anywhere to assume the Hybrid Nodes IAM role to exchange temporary IAM credentials only when the role session name matches the CN in the x509 certificate installed on your hybrid nodes. You can alternatively use other certificate attributes to uniquely identify your node. The certificate attribute that you use in the trust policy must correspond to the `nodeName` you set in your `nodeadm` configuration. For more information, see the <>. +. Create a file named `eks-hybrid-iamra-trust.json`. Replace `TRUST_ANCHOR ARN` with the ARN of the trust anchor you created in the <> steps. The condition in this trust policy restricts the ability of {aws} IAM Roles Anywhere to assume the Hybrid Nodes IAM role to exchange temporary IAM credentials only when the role session name matches the CN in the x509 certificate installed on your hybrid nodes. You can alternatively use other certificate attributes to uniquely identify your node. The certificate attribute that you use in the trust policy must correspond to the `nodeName` you set in your `nodeadm` configuration. For more information, see the <>. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,json,subs="verbatim,attributes"] ---- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Service": "rolesanywhere.amazonaws.com" - }, - "Action": [ - "sts:TagSession", - "sts:SetSourceIdentity" - ], - "Condition": { - "ArnEquals": { - "aws:SourceArn": "TRUST_ANCHOR_ARN" - } - } - }, - { - "Effect": "Allow", - "Principal": { - "Service": "rolesanywhere.amazonaws.com" - }, - "Action": "sts:AssumeRole", - "Condition": { - "StringEquals": { - "sts:RoleSessionName": "${aws:PrincipalTag/x509Subject/CN}" - }, - "ArnEquals": { - "aws:SourceArn": "TRUST_ANCHOR_ARN" - } - } - } - ] -} +include::iam_snippet:ee530bf3-9b59-49d3-9d8f-595d81a3c888[] ---- . Create the role with the following command. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- aws iam create-role \ --role-name AmazonEKSHybridNodesRole \ @@ -362,28 +283,29 @@ aws iam create-role \ . Attach the `EKSDescribeClusterPolicy` you created in the previous steps. Replace `AWS_ACCOUNT_ID` with your {aws} account ID. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- aws iam attach-role-policy \ --role-name AmazonEKSHybridNodesRole \ - --policy-arn arn:aws:iam::AWS_ACCOUNT_ID:policy/EKSDescribeClusterPolicy + --policy-arn {arn-aws}iam::AWS_ACCOUNT_ID:policy/EKSDescribeClusterPolicy ---- . Attach the `AmazonEC2ContainerRegistryPullOnly` {aws} managed policy + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- aws iam attach-role-policy \ --role-name AmazonEKSHybridNodesRole \ - --policy-arn arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPullOnly + --policy-arn {arn-aws}iam::aws:policy/AmazonEC2ContainerRegistryPullOnly ---- -=== {aws} Management Console +[#hybrid-nodes-creds-console] +=== {aws-management-console} *Create EKS Describe Cluster Policy* -. Open the link:iam/home#/clusters[Amazon IAM console,type="console"] +. Open the link:iam/home[Amazon IAM console,type="console"] . In the left navigation pane, choose *Policies*. . On the *Policies* page, choose *Create policy*. . On the Specify permissions page, in the Select a service panel, choose EKS. @@ -395,33 +317,14 @@ aws iam attach-role-policy \ *Steps for {aws} SSM hybrid activations* -. Open the link:iam/home#/clusters[Amazon IAM console,type="console"] +. Open the link:iam/home[Amazon IAM console,type="console"] . In the left navigation pane, choose *Policies*. . On the *Policies page*, choose *Create policy*. . On the *Specify permissions* page, in the *Policy editor* top right navigation, choose *JSON*. Paste the following snippet. Replace `AWS_REGION` with the {aws} Region of your {aws} SSM hybrid activation and replace `AWS_ACCOUNT_ID` with your {aws} account ID. Replace `TAG_KEY` and `TAG_VALUE` with the {aws} SSM resource tag key you used when creating your {aws} SSM hybrid activation. + -[source,yaml,subs="verbatim,attributes,quotes"] +[source,json,subs="verbatim,attributes"] ---- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": "ssm:DescribeInstanceInformation", - "Resource": "*" - }, - { - "Effect": "Allow", - "Action": "ssm:DeregisterManagedInstance", - "Resource": "arn:aws:ssm:AWS_REGION:AWS_ACCOUNT_ID:managed-instance/*", - "Condition": { - "StringEquals": { - "ssm:resourceTag/TAG_KEY": "TAG_VALUE" - } - } - } - ] -} +include::iam_snippet:2a60ddcc-a7b9-42a0-8dcb-22cbca8f9e9e[] ---- .. Choose *Next*. @@ -433,29 +336,9 @@ aws iam attach-role-policy \ . On the *Select trusted entity* page, do the following: .. In the *Trusted entity* type section, choose *Custom trust policy*. Paste the following into the Custom trust policy editor. Replace `AWS_REGION` with the {aws} Region of your {aws} SSM hybrid activation and `AWS_ACCOUNT_ID` with your {aws} account ID. + -[source,yaml,subs="verbatim,attributes,quotes"] +[source,yaml,subs="verbatim,attributes"] ---- -{ - "Version":"2012-10-17", - "Statement":[ - { - "Sid":"", - "Effect":"Allow", - "Principal":{ - "Service":"ssm.amazonaws.com" - }, - "Action":"sts:AssumeRole", - "Condition":{ - "StringEquals":{ - "aws:SourceAccount":"AWS_ACCOUNT_ID" - }, - "ArnEquals":{ - "aws:SourceArn":"arn:aws:ssm:AWS_REGION:AWS_ACCOUNT_ID:*" - } - } - } - ] -} +include::iam_snippet:4184ce45-ea47-4357-aaea-23c00afd29bb[] ---- .. Choose Next. . On the *Add permissions* page, attach a custom policy or do the following: @@ -471,51 +354,17 @@ aws iam attach-role-policy \ *Steps for {aws} IAM Roles Anywhere* -To use {aws} IAM Roles Anywhere, you must set up your {aws} IAM Roles Anywhere trust anchor before creating the Hybrid Nodes IAM Role. See <> for instructions. +To use {aws} IAM Roles Anywhere, you must set up your {aws} IAM Roles Anywhere trust anchor before creating the Hybrid Nodes IAM Role. See <> for instructions. -. Open the link:iam/home#/clusters[Amazon IAM console,type="console"] +. Open the link:iam/home[Amazon IAM console,type="console"] . In the left navigation pane, choose *Roles*. . On the *Roles* page, choose *Create role*. . On the *Select trusted entity* page, do the following: -.. In the *Trusted entity type section*, choose *Custom trust policy*. Paste the following into the Custom trust policy editor. Replace `TRUST_ANCHOR ARN` with the ARN of the trust anchor you created in the <> steps. The condition in this trust policy restricts the ability of {aws} IAM Roles Anywhere to assume the Hybrid Nodes IAM role to exchange temporary IAM credentials only when the role session name matches the CN in the x509 certificate installed on your hybrid nodes. You can alternatively use other certificate attributes to uniquely identify your node. The certificate attribute that you use in the trust policy must correspond to the nodeName you set in your nodeadm configuration. For more information, see the <>. +.. In the *Trusted entity type section*, choose *Custom trust policy*. Paste the following into the Custom trust policy editor. Replace `TRUST_ANCHOR ARN` with the ARN of the trust anchor you created in the <> steps. The condition in this trust policy restricts the ability of {aws} IAM Roles Anywhere to assume the Hybrid Nodes IAM role to exchange temporary IAM credentials only when the role session name matches the CN in the x509 certificate installed on your hybrid nodes. You can alternatively use other certificate attributes to uniquely identify your node. The certificate attribute that you use in the trust policy must correspond to the nodeName you set in your nodeadm configuration. For more information, see the <>. + -[source,bash,subs="verbatim,attributes,quotes"] +[source,json,subs="verbatim,attributes"] ---- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Service": "rolesanywhere.amazonaws.com" - }, - "Action": [ - "sts:TagSession", - "sts:SetSourceIdentity" - ], - "Condition": { - "ArnEquals": { - "aws:SourceArn": "TRUST_ANCHOR_ARN" - } - } - }, - { - "Effect": "Allow", - "Principal": { - "Service": "rolesanywhere.amazonaws.com" - }, - "Action": "sts:AssumeRole", - "Condition": { - "StringEquals": { - "sts:RoleSessionName": "${aws:PrincipalTag/x509Subject/CN}" - }, - "ArnEquals": { - "aws:SourceArn": "TRUST_ANCHOR_ARN" - } - } - } - ] -} +include::iam_snippet:44c203c0-92dd-497c-91fd-f18d4cbeaa7b[] ---- .. Choose Next. . On the *Add permissions* page, attach a custom policy or do the following: diff --git a/latest/ug/nodes/hybrid-nodes-ingress.adoc b/latest/ug/nodes/hybrid-nodes-ingress.adoc new file mode 100644 index 000000000..509885d3d --- /dev/null +++ b/latest/ug/nodes/hybrid-nodes-ingress.adoc @@ -0,0 +1,1203 @@ +include::../attributes.txt[] + +[.topic] +[#hybrid-nodes-ingress] += Configure Kubernetes Ingress for hybrid nodes +:info_titleabbrev: Configure Ingress + +[abstract] +-- +Configure Kubernetes Ingress for hybrid nodes +-- + +This topic describes how to configure Kubernetes Ingress for workloads running on Amazon EKS Hybrid Nodes. link:https://kubernetes.io/docs/concepts/services-networking/ingress/[Kubernetes Ingress] exposes HTTP and HTTPS routes from outside the cluster to services within the cluster. To make use of Ingress resources, a Kubernetes Ingress controller is required to set up the networking infrastructure and components that serve the network traffic. + +{aws} supports {aws} Application Load Balancer (ALB) and Cilium for Kubernetes Ingress for workloads running on EKS Hybrid Nodes. The decision to use ALB or Cilium for Ingress is based on the source of application traffic. If application traffic originates from an {aws} Region, {aws} recommends using {aws} ALB and the {aws} Load Balancer Controller. If application traffic originates from the local on-premises or edge environment, {aws} recommends using Cilium's built-in Ingress capabilities, which can be used with or without load balancer infrastructure in your environment. + +image::images/hybrid-nodes-ingress.png[EKS Hybrid Nodes Ingress,scaledwidth=50%,align="center"] + +[#hybrid-nodes-ingress-alb] +== {aws} Application Load Balancer + +You can use the <> and Application Load Balancer (ALB) with the target type `ip` for workloads running on hybrid nodes. When using target type `ip`, ALB forwards traffic directly to the pods, bypassing the Service layer network path. For ALB to reach the pod IP targets on hybrid nodes, your on-premises pod CIDR must be routable on your on-premises network. Additionally, the {aws} Load Balancer Controller uses webhooks and requires direct communication from the EKS control plane. For more information, see <>. + +=== Considerations + +- See <> and <> for more information on {aws} Application Load Balancer and {aws} Load Balancer Controller. +- See link:eks/latest/best-practices/load-balacing.html[Best Practices for Load Balancing,type="documentation"] for information on how to choose between {aws} Application Load Balancer and {aws} Network Load Balancer. +- See link:https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/guide/ingress/annotations/[{aws} Load Balancer Controller Ingress annotations] for the list of annotations that can be configured for Ingress resources with {aws} Application Load Balancer. + +=== Prerequisites + +- Cilium installed following the instructions in <>. +- Cilium BGP Control Plane enabled following the instructions in <>. If you do not want to use BGP, you must use an alternative method to make your on-premises pod CIDRs routable on your on-premises network. If you do not make your on-premises pod CIDRs routable, ALB will not be able to register or contact your pod IP targets. +- Helm installed in your command-line environment, see the <> for more information. +- eksctl installed in your command-line environment, see the <> for more information. + +=== Procedure + +. Download an IAM policy for the {aws} Load Balancer Controller that allows it to make calls to {aws} APIs on your behalf. ++ +[source,bash] +---- +curl -O https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/refs/heads/main/docs/install/iam_policy.json +---- ++ +. Create an IAM policy using the policy downloaded in the previous step. ++ +[source,bash,subs="verbatim,attributes,quotes"] +---- +aws iam create-policy \ + --policy-name AWSLoadBalancerControllerIAMPolicy \ + --policy-document file://iam_policy.json +---- ++ +. Replace the value for cluster name (`CLUSTER_NAME`), {aws} Region (`AWS_REGION`), and {aws} account ID (`AWS_ACCOUNT_ID`) with your settings and run the following command. ++ +[source,bash,subs="verbatim,attributes"] +---- +eksctl create iamserviceaccount \ + --cluster=CLUSTER_NAME \ + --namespace=kube-system \ + --name=aws-load-balancer-controller \ + --attach-policy-arn=arn:aws:iam::AWS_ACCOUNT_ID:policy/AWSLoadBalancerControllerIAMPolicy \ + --override-existing-serviceaccounts \ + --region AWS_REGION \ + --approve +---- ++ +. Add the eks-charts Helm chart repository and update your local Helm repository to make sure that you have the most recent charts. ++ +[source,bash] +---- +helm repo add eks https://aws.github.io/eks-charts +---- ++ +[source,bash] +---- +helm repo update eks +---- ++ +. Install the {aws} Load Balancer Controller. Replace the value for cluster name (`CLUSTER_NAME`), {aws} Region (`AWS_REGION`), VPC ID (`VPC_ID`), and {aws} Load Balancer Controller Helm chart version (`AWS_LBC_HELM_VERSION`) with your settings and run the following command. If you are running a mixed mode cluster with both hybrid nodes and nodes in {aws} Cloud, you can run the {aws} Load Balancer Controller on cloud nodes following the instructions at <>. ++ +- You can find the latest version of the Helm chart by running `helm search repo eks/aws-load-balancer-controller --versions`. ++ +[source,bash,subs="verbatim,attributes,quotes"] +---- +helm install aws-load-balancer-controller eks/aws-load-balancer-controller \ + -n kube-system \ + --version [.replaceable]`AWS_LBC_HELM_VERSION` \ + --set clusterName=[.replaceable]`CLUSTER_NAME` \ + --set region=[.replaceable]`AWS_REGION` \ + --set vpcId=[.replaceable]`VPC_ID` \ + --set serviceAccount.create=false \ + --set serviceAccount.name=aws-load-balancer-controller +---- ++ +. Verify the {aws} Load Balancer Controller was installed successfully. ++ +[source,bash] +---- +kubectl get -n kube-system deployment aws-load-balancer-controller +---- ++ +[source,bash] +---- +NAME READY UP-TO-DATE AVAILABLE AGE +aws-load-balancer-controller 2/2 2 2 84s +---- +. Create a sample application. The example below uses the link:https://istio.io/latest/docs/examples/bookinfo/[Istio Bookinfo] sample microservices application. ++ +[source,bash] +---- +kubectl apply -f https://raw.githubusercontent.com/istio/istio/refs/heads/master/samples/bookinfo/platform/kube/bookinfo.yaml +---- ++ +. Create a file named `my-ingress-alb.yaml` with the following contents. ++ +[source,yaml] +---- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: my-ingress + namespace: default + annotations: + alb.ingress.kubernetes.io/load-balancer-name: "my-ingress-alb" + alb.ingress.kubernetes.io/target-type: "ip" + alb.ingress.kubernetes.io/scheme: "internet-facing" + alb.ingress.kubernetes.io/healthcheck-path: "/details/1" +spec: + ingressClassName: alb + rules: + - http: + paths: + - backend: + service: + name: details + port: + number: 9080 + path: /details + pathType: Prefix +---- ++ +. Apply the Ingress configuration to your cluster. ++ +[source,bash] +---- +kubectl apply -f my-ingress-alb.yaml +---- ++ +. Provisioning the ALB for your Ingress resource may take a few minutes. Once the ALB is provisioned, your Ingress resource will have an address assigned to it that corresponds to the DNS name of the ALB deployment. The address will have the format `-..elb.amazonaws.com`. ++ +[source,bash] +---- +kubectl get ingress my-ingress +---- ++ +[source,bash] +---- +NAME CLASS HOSTS ADDRESS PORTS AGE +my-ingress alb * my-ingress-alb-..elb.amazonaws.com 80 23m +---- ++ +. Access the Service using the address of the ALB. ++ +[source,bash] +---- +curl -s http//my-ingress-alb-..elb.amazonaws.com:80/details/1 | jq +---- ++ +[source,bash] +---- +{ + "id": 1, + "author": "William Shakespeare", + "year": 1595, + "type": "paperback", + "pages": 200, + "publisher": "PublisherA", + "language": "English", + "ISBN-10": "1234567890", + "ISBN-13": "123-1234567890" + "details": "This is the details page" +} +---- + +[#hybrid-nodes-ingress-cilium] +== Cilium Ingress and Cilium Gateway Overview + +Cilium's Ingress capabilities are built into Cilium's architecture and can be managed with the Kubernetes Ingress API or Gateway API. If you don't have existing Ingress resources, {aws} recommends to start with the Gateway API, as it is a more expressive and flexible way to define and manage Kubernetes networking resources. The link:https://gateway-api.sigs.k8s.io/[Kubernetes Gateway API] aims to standardize how networking resources for Ingress, Load Balancing, and Service Mesh are defined and managed in Kubernetes clusters. + +When you enable Cilium's Ingress or Gateway features, the Cilium operator reconciles Ingress / Gateway objects in the cluster and Envoy proxies on each node process the Layer 7 (L7) network traffic. Cilium does not directly provision Ingress / Gateway infrastructure such as load balancers. If you plan to use Cilium Ingress / Gateway with a load balancer, you must use the load balancer's tooling, commonly an Ingress or Gateway controller, to deploy and manage the load balancer's infrastructure. + +For Ingress / Gateway traffic, Cilium handles the core network traffic and L3/L4 policy enforcement, and integrated Envoy proxies process the L7 network traffic. With Cilium Ingress / Gateway, Envoy is responsible for applying L7 routing rules, policies, and request manipulation, advanced traffic management such as traffic splitting and mirroring, and TLS termination and origination. Cilium's Envoy proxies are deployed as a separate DaemonSet (`cilium-envoy`) by default, which enables Envoy and the Cilium agent to be separately updated, scaled, and managed. + +For more information on how Cilium Ingress and Cilium Gateway work, see the link:https://docs.cilium.io/en/stable/network/servicemesh/ingress/[Cilium Ingress] and link:https://docs.cilium.io/en/stable/network/servicemesh/gateway-api/gateway-api/[Cilium Gateway] pages in the Cilium documentation. + +[#hybrid-nodes-ingress-cilium-comparison] +== Cilium Ingress and Gateway Comparison + +The table below summarizes the Cilium Ingress and Cilium Gateway features as of *Cilium version 1.17.x*. + +[%header,cols="3,2,2"] +|=== +|Feature +^|Ingress +^|Gateway + +|Service type LoadBalancer +^|Yes +^|Yes + +|Service type NodePort +^|Yes +^|No^1^ + +|Host network +^|Yes +^|Yes + +|Shared load balancer +^|Yes +^|Yes + +|Dedicated load balancer +^|Yes +^|No^2^ + +|Network policies +^|Yes +^|Yes + +|Protocols +^|Layer 7 (HTTP(S), gRPC) +^|Layer 7 (HTTP(S), gRPC)^3^ + +|TLS Passthrough +^|Yes +^|Yes + +|Traffic Management +^|Path and Host routing +^|Path and Host routing, URL redirect and rewrite, +traffic splitting, header modification + +|=== + +^1^ Cilium Gateway support for NodePort services is planned for Cilium version 1.18.x (link:https://github.com/cilium/cilium/pull/27273[#27273]) + +^2^ Cilium Gateway support for dedicated load balancers (link:https://github.com/cilium/cilium/issues/25567[#25567]) + +^3^ Cilium Gateway support for TCP/UDP (link:https://github.com/cilium/cilium/issues/21929[#21929]) + +[#hybrid-nodes-ingress-cilium-gateway-install] +== Install Cilium Gateway + +=== Considerations + +- Cilium must be configured with `nodePort.enabled` set to `true` as shown in the examples below. If you are using Cilium's kube-proxy replacement feature, you do not need to set `nodePort.enabled` to `true`. +- Cilium must be configured with `envoy.enabled` set to `true` as shown in the examples below. +- Cilium Gateway can be deployed in load balancer (default) or host network mode. +- When using Cilium Gateway in load balancer mode, the `service.beta.kubernetes.io/aws-load-balancer-type: "external"` annotation must be set on the Gateway resource to prevent the legacy {aws} cloud provider from creating a Classic Load Balancer for the Service of type LoadBalancer that Cilium creates for the Gateway resource. +- When using Cilium Gateway in host network mode, the Service of type LoadBalancer mode is disabled. Host network mode is useful for environments that do not have load balancer infrastructure, see <> for more information. + +=== Prerequisites + +. Helm installed in your command-line environment, see <>. +. Cilium installed following the instructions in <>. + +=== Procedure + +. Install the Kubernetes Gateway API Custom Resource Definitions (CRDs). ++ +[source,bash,subs="verbatim,attributes,quotes"] +---- +kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.2.1/config/crd/standard/gateway.networking.k8s.io_gatewayclasses.yaml +kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.2.1/config/crd/standard/gateway.networking.k8s.io_gateways.yaml +kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.2.1/config/crd/standard/gateway.networking.k8s.io_httproutes.yaml +kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.2.1/config/crd/standard/gateway.networking.k8s.io_referencegrants.yaml +kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.2.1/config/crd/standard/gateway.networking.k8s.io_grpcroutes.yaml +---- ++ +. Create a file called `cilium-gateway-values.yaml` with the following contents. The example below configures Cilium Gateway to use the default load balancer mode and to use a separate `cilium-envoy` DaemonSet for Envoy proxies configured to run only on hybrid nodes. ++ +[source,yaml] +---- +gatewayAPI: + enabled: true + # uncomment to use host network mode + # hostNetwork: + # enabled: true +nodePort: + enabled: true +envoy: + enabled: true + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: In + values: + - hybrid +---- ++ +. Apply the Helm values file to your cluster. ++ +[source,bash,subs="verbatim,attributes,quotes"] +---- +helm upgrade cilium oci://public.ecr.aws/eks/cilium/cilium \ + --namespace kube-system \ + --reuse-values \ + --set operator.rollOutPods=true \ + --values cilium-gateway-values.yaml +---- ++ +. Confirm the Cilium operator, agent, and Envoy pods are running. ++ +[source,bash] +---- +kubectl -n kube-system get pods --selector=app.kubernetes.io/part-of=cilium +---- ++ +[source,bash] +---- +NAME READY STATUS RESTARTS AGE +cilium-envoy-5pgnd 1/1 Running 0 6m31s +cilium-envoy-6fhg4 1/1 Running 0 6m30s +cilium-envoy-jskrk 1/1 Running 0 6m30s +cilium-envoy-k2xtb 1/1 Running 0 6m31s +cilium-envoy-w5s9j 1/1 Running 0 6m31s +cilium-grwlc 1/1 Running 0 4m12s +cilium-operator-68f7766967-5nnbl 1/1 Running 0 4m20s +cilium-operator-68f7766967-7spfz 1/1 Running 0 4m20s +cilium-pnxcv 1/1 Running 0 6m29s +cilium-r7qkj 1/1 Running 0 4m12s +cilium-wxhfn 1/1 Running 0 4m1s +cilium-z7hlb 1/1 Running 0 6m30s +---- + +[#hybrid-nodes-ingress-cilium-gateway-configure] +== Configure Cilium Gateway + +Cilium Gateway is enabled on Gateway objects by setting the `gatewayClassName` to `cilium`. The Service that Cilium creates for Gateway resources can be configured with fields on the Gateway object. Common annotations used by Gateway controllers to configure the load balancer infrastructure can be configured with the Gateway object's `infrastructure` field. When using Cilium's LoadBalancer IPAM (see example in <>), the IP address to use for the Service of type LoadBalancer can be configured on the Gateway object's `addresses` field. For more information on Gateway configuration, see the link:https://gateway-api.sigs.k8s.io/reference/spec/#gateway[Kubernetes Gateway API specification]. + +[source,yaml] +---- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: my-gateway +spec: + gatewayClassName: cilium + infrastructure: + annotations: + service.beta.kubernetes.io/... + service.kuberentes.io/... + addresses: + - type: IPAddress + value: + listeners: + ... +---- + +Cilium and the Kubernetes Gateway specification support the GatewayClass, Gateway, HTTPRoute, GRPCRoute, and ReferenceGrant resources. + +- See link:https://gateway-api.sigs.k8s.io/api-types/httproute/HTTPRoute[HTTPRoute] and link:https://gateway-api.sigs.k8s.io/api-types/grpcroute/GRPCRoute[GRPCRoute] specifications for the list of available fields. +- See the examples in the <> section below and the examples in the link:https://docs.cilium.io/en/stable/network/servicemesh/gateway-api/gateway-api/#examples[Cilium documentation] for how to use and configure these resources. + +[#hybrid-nodes-ingress-cilium-gateway-deploy] +== Deploy Cilium Gateway + +. Create a sample application. The example below uses the link:https://istio.io/latest/docs/examples/bookinfo/[Istio Bookinfo] sample microservices application. ++ +[source,bash] +---- +kubectl apply -f https://raw.githubusercontent.com/istio/istio/refs/heads/master/samples/bookinfo/platform/kube/bookinfo.yaml +---- ++ +. Confirm the application is running successfully. ++ +[source,bash] +---- +kubectl get pods +---- ++ +[source,bash] +---- +NAME READY STATUS RESTARTS AGE +details-v1-766844796b-9965p 1/1 Running 0 81s +productpage-v1-54bb874995-jmc8j 1/1 Running 0 80s +ratings-v1-5dc79b6bcd-smzxz 1/1 Running 0 80s +reviews-v1-598b896c9d-vj7gb 1/1 Running 0 80s +reviews-v2-556d6457d-xbt8v 1/1 Running 0 80s +reviews-v3-564544b4d6-cpmvq 1/1 Running 0 80s +---- ++ +. Create a file named `my-gateway.yaml` with the following contents. The example below uses the `service.beta.kubernetes.io/aws-load-balancer-type: "external"` annotation to prevent the legacy {aws} cloud provider from creating a Classic Load Balancer for the Service of type LoadBalancer that Cilium creates for the Gateway resource. ++ +[source,yaml] +---- +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: my-gateway +spec: + gatewayClassName: cilium + infrastructure: + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: "external" + listeners: + - protocol: HTTP + port: 80 + name: web-gw + allowedRoutes: + namespaces: + from: Same +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: http-app-1 +spec: + parentRefs: + - name: my-gateway + namespace: default + rules: + - matches: + - path: + type: PathPrefix + value: /details + backendRefs: + - name: details + port: 9080 +---- ++ +. Apply the Gateway resource to your cluster. ++ +[source,bash] +---- +kubectl apply -f my-gateway.yaml +---- ++ +. Confirm the Gateway resource and corresponding Service were created. At this stage, it is expected that the `ADDRESS` field of the Gateway resource is not populated with an IP address or hostname, and that the Service of type LoadBalancer for the Gateway resource similarly does not have an IP address or hostname assigned. ++ +[source,bash] +---- +kubectl get gateway my-gateway +---- ++ +[source,bash] +---- +NAME CLASS ADDRESS PROGRAMMED AGE +my-gateway cilium True 10s +---- ++ +[source,bash] +---- +kubectl get svc cilium-gateway-my-gateway +---- ++ +[source,bash] +---- +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +cilium-gateway-my-gateway LoadBalancer 172.16.227.247 80:30912/TCP 24s +---- ++ +. Proceed to <> to configure the Gateway resource to use an IP address allocated by Cilium Load Balancer IPAM, and <> or <> to configure the Gateway resource to use NodePort or host network addresses. + +[#hybrid-nodes-ingress-cilium-ingress-install] +== Install Cilium Ingress + +=== Considerations + +- Cilium must be configured with `nodePort.enabled` set to `true` as shown in the examples below. If you are using Cilium's kube-proxy replacement feature, you do not need to set `nodePort.enabled` to `true`. +- Cilium must be configured with `envoy.enabled` set to `true` as shown in the examples below. +- With `ingressController.loadbalancerMode` set to `dedicated`, Cilium creates dedicated Services for each Ingress resource. With `ingressController.loadbalancerMode` set to `shared`, Cilium creates a shared Service of type LoadBalancer for all Ingress resources in the cluster. When using the `shared` load balancer mode, the settings for the shared Service such as `labels`, `annotations`, `type`, and `loadBalancerIP` are configured in the `ingressController.service` section of the Helm values. See the link:https://github.com/cilium/cilium/blob/v1.17.6/install/kubernetes/cilium/values.yaml#L887[Cilium Helm values reference] for more information. +- With `ingressController.default` set to `true`, Cilium is configured as the default Ingress controller for the cluster and will create Ingress entries even when the `ingressClassName` is not specified on Ingress resources. +- Cilium Ingress can be deployed in load balancer (default), node port, or host network mode. When Cilium is installed in host network mode, the Service of type LoadBalancer and Service of type NodePort modes are disabled. See <> for more information. +- Always set `ingressController.service.annotations` to `service.beta.kubernetes.io/aws-load-balancer-type: "external"` in the Helm values to prevent the legacy {aws} cloud provider from creating a Classic Load Balancer for the default `cilium-ingress` Service created by the link:https://github.com/cilium/cilium/blob/main/install/kubernetes/cilium/templates/cilium-ingress-service.yaml[Cilium Helm chart]. + +=== Prerequisites + +. Helm installed in your command-line environment, see <>. +. Cilium installed following the instructions in <>. + +=== Procedure + +. Create a file called `cilium-ingress-values.yaml` with the following contents. The example below configures Cilium Ingress to use the default load balancer `dedicated` mode and to use a separate `cilium-envoy` DaemonSet for Envoy proxies configured to run only on hybrid nodes. + ++ +[source,yaml] +---- +ingressController: + enabled: true + loadbalancerMode: dedicated + service: + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: "external" +nodePort: + enabled: true +envoy: + enabled: true + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: In + values: + - hybrid +---- ++ +. Apply the Helm values file to your cluster. ++ +[source,bash,subs="verbatim,attributes,quotes"] +---- +helm upgrade cilium oci://public.ecr.aws/eks/cilium/cilium \ + --namespace kube-system \ + --reuse-values \ + --set operator.rollOutPods=true \ + --values cilium-ingress-values.yaml +---- ++ +. Confirm the Cilium operator, agent, and Envoy pods are running. ++ +[source,bash] +---- +kubectl -n kube-system get pods --selector=app.kubernetes.io/part-of=cilium +---- ++ +[source,bash] +---- +NAME READY STATUS RESTARTS AGE +cilium-envoy-5pgnd 1/1 Running 0 6m31s +cilium-envoy-6fhg4 1/1 Running 0 6m30s +cilium-envoy-jskrk 1/1 Running 0 6m30s +cilium-envoy-k2xtb 1/1 Running 0 6m31s +cilium-envoy-w5s9j 1/1 Running 0 6m31s +cilium-grwlc 1/1 Running 0 4m12s +cilium-operator-68f7766967-5nnbl 1/1 Running 0 4m20s +cilium-operator-68f7766967-7spfz 1/1 Running 0 4m20s +cilium-pnxcv 1/1 Running 0 6m29s +cilium-r7qkj 1/1 Running 0 4m12s +cilium-wxhfn 1/1 Running 0 4m1s +cilium-z7hlb 1/1 Running 0 6m30s +---- + +[#hybrid-nodes-ingress-cilium-ingress-configure] +== Configure Cilium Ingress + +Cilium Ingress is enabled on Ingress objects by setting the `ingressClassName` to `cilium`. The Service(s) that Cilium creates for Ingress resources can be configured with annotations on the Ingress objects when using the `dedicated` load balancer mode and in the Cilium / Helm configuration when using the `shared` load balancer mode. These annotations are commonly used by Ingress controllers to configure the load balancer infrastructure, or other attributes of the Service such as the service type, load balancer mode, ports, and TLS passthrough. Key annotations are described below. For a full list of supported annotations, see the link:https://docs.cilium.io/en/stable/network/servicemesh/ingress/#supported-ingress-annotations[Cilium Ingress annotations] in the Cilium documentation. + +[%header,cols="2,3"] +|=== +|Annotation +|Description + +|`ingress.cilium.io/loadbalancer-mode` +|`dedicated`: Dedicated Service of type LoadBalancer for each Ingress resource (default). + +`shared`: Single Service of type LoadBalancer for all Ingress resources. + +|`ingress.cilium.io/service-type` +| `LoadBalancer`: The Service will be of type LoadBalancer (default) + +`NodePort`: The Service will be of type NodePort. + +|`service.beta.kubernetes.io/aws-load-balancer-type` +| `"external"`: Prevent legacy {aws} cloud provider from provisioning Classic Load Balancer for Services of type LoadBalancer. + +|`lbipam.cilium.io/ips` +| List of IP addresses to allocate from Cilium LoadBalancer IPAM + +|=== + +Cilium and the Kubernetes Ingress specification support Exact, Prefix, and Implementation-specific matching rules for Ingress paths. Cilium supports regex as its implementation-specific matching rule. For more information, see link:https://docs.cilium.io/en/stable/network/servicemesh/ingress/#ingress-path-types-and-precedence[Ingress path types and precedence] and link:https://docs.cilium.io/en/stable/network/servicemesh/path-types/[Path types examples] in the Cilium documentation, and the examples in the <> section of this page. + +An example Cilium Ingress object is shown below. + +[source,yaml] +---- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: my-ingress + annotations: + service.beta.kuberentes.io/... + service.kuberentes.io/... +spec: + ingressClassName: cilium + rules: + ... +---- + +[#hybrid-nodes-ingress-cilium-ingress-deploy] +== Deploy Cilium Ingress + +. Create a sample application. The example below uses the link:https://istio.io/latest/docs/examples/bookinfo/[Istio Bookinfo] sample microservices application. ++ +[source,bash] +---- +kubectl apply -f https://raw.githubusercontent.com/istio/istio/refs/heads/master/samples/bookinfo/platform/kube/bookinfo.yaml +---- ++ +. Confirm the application is running successfully. ++ +[source,bash] +---- +kubectl get pods +---- ++ +[source,bash] +---- +NAME READY STATUS RESTARTS AGE +details-v1-766844796b-9965p 1/1 Running 0 81s +productpage-v1-54bb874995-jmc8j 1/1 Running 0 80s +ratings-v1-5dc79b6bcd-smzxz 1/1 Running 0 80s +reviews-v1-598b896c9d-vj7gb 1/1 Running 0 80s +reviews-v2-556d6457d-xbt8v 1/1 Running 0 80s +reviews-v3-564544b4d6-cpmvq 1/1 Running 0 80s +---- ++ +. Create a file named `my-ingress.yaml` with the following contents. The example below uses the `service.beta.kubernetes.io/aws-load-balancer-type: "external"` annotation to prevent the legacy {aws} cloud provider from creating a Classic Load Balancer for the Service of type LoadBalancer that Cilium creates for the Ingress resource. ++ +[source,yaml] +---- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: my-ingress + namespace: default + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: "external" +spec: + ingressClassName: cilium + rules: + - http: + paths: + - backend: + service: + name: details + port: + number: 9080 + path: /details + pathType: Prefix +---- ++ +. Apply the Ingress resource to your cluster. ++ +[source,bash] +---- +kubectl apply -f my-ingress.yaml +---- ++ +. Confirm the Ingress resource and corresponding Service were created. At this stage, it is expected that the `ADDRESS` field of the Ingress resource is not populated with an IP address or hostname, and that the shared or dedicated Service of type LoadBalancer for the Ingress resource similarly does not have an IP address or hostname assigned. ++ +[source,bash] +---- +kubectl get ingress my-ingress +---- ++ +[source,bash] +---- +NAME CLASS HOSTS ADDRESS PORTS AGE +my-ingress cilium * 80 8s +---- ++ +For load balancer mode `shared` ++ +[source,bash] +---- +kubectl -n kube-system get svc cilium-ingress +---- ++ +[source,bash] +---- +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +cilium-ingress LoadBalancer 172.16.217.48 80:32359/TCP,443:31090/TCP 10m +---- ++ +For load balancer mode `dedicated` ++ +[source,bash] +---- +kubectl -n default get svc cilium-ingress-my-ingress +---- ++ +[source,bash] +---- +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +cilium-ingress-my-ingress LoadBalancer 172.16.193.15 80:32088/TCP,443:30332/TCP 25s +---- ++ +. Proceed to <> to configure the Ingress resource to use an IP address allocated by Cilium Load Balancer IPAM, and <> or <> to configure the Ingress resource to use NodePort or host network addresses. + +[#hybrid-nodes-ingress-cilium-loadbalancer] +== Service type LoadBalancer + +=== Existing load balancer infrastructure + +By default, for both Cilium Ingress and Cilium Gateway, Cilium creates Kubernetes Service(s) of type LoadBalancer for the Ingress / Gateway resources. The attributes of the Service(s) that Cilium creates can be configured through the Ingress and Gateway resources. When you create Ingress or Gateway resources, the externally exposed IP address or hostnames for the Ingress or Gateway are allocated from the load balancer infrastructure, which is typically provisioned by an Ingress or Gateway controller. + +Many Ingress and Gateway controllers use annotations to detect and configure the load balancer infrastructure. The annotations for these Ingress and Gateway controllers are configured on the Ingress or Gateway resources as shown in the previous examples above. Reference your Ingress or Gateway controller's documentation for the annotations it supports and see the link:https://kubernetes.io/docs/concepts/services-networking/ingress-controllers/[Kubernetes Ingress documentation] and link:https://gateway-api.sigs.k8s.io/implementations/[Kubernetes Gateway documentation] for a list of popular controllers. + +[IMPORTANT] +==== +Cilium Ingress and Gateway cannot be used with the {aws} Load Balancer Controller and {aws} Network Load Balancers (NLBs) with EKS Hybrid Nodes. Attempting to use these together results in unregistered targets, as the NLB attempts to directly connect to the Pod IPs that back the Service of type LoadBalancer when the NLB's `target-type` is set to `ip` (requirement for using NLB with workloads running on EKS Hybrid Nodes). +==== + +=== No load balancer infrastructure + +If you do not have load balancer infrastructure and corresponding Ingress / Gateway controller in your environment, Ingress / Gateway resources and corresponding Services of type LoadBalancer can be configured to use IP addresses allocated by Cilium's link:https://docs.cilium.io/en/stable/network/lb-ipam/[Load Balancer IP address management] (LB IPAM). Cilium LB IPAM can be configured with known IP address ranges from your on-premises environment, and can use Cilium's built-in Border Gateway Protocol (BGP) support or L2 announcements to advertise the LoadBalancer IP addresses to your on-premises network. + +The example below shows how to configure Cilium's LB IPAM with an IP address to use for your Ingress / Gateway resources, and how to configure Cilium BGP Control Plane to advertise the LoadBalancer IP address with the on-premises network. Cilium's LB IPAM feature is enabled by default, but is not activated until a `CiliumLoadBalancerIPPool` resource is created. + +==== Prerequisites + +- Cilium Ingress or Gateway installed following the instructions in <> or <>. +- Cilium Ingress or Gateway resources with sample application deployed following the instructions in <> or <>. +- Cilium BGP Control Plane enabled following the instructions in <>. If you do not want to use BGP, you can skip this prerequisite, but you will not be able to access your Ingress or Gateway resource until the LoadBalancer IP address allocated by Cilium LB IPAM is routable on your on-premises network. + +==== Procedure + +. Optionally patch the Ingress or Gateway resource to request a specific IP address to use for the Service of type LoadBalancer. If you do not request a specific IP address, Cilium will allocate an IP address from the IP address range configured in the `CiliumLoadBalancerIPPool` resource in the subsequent step. In the commands below, replace `LB_IP_ADDRESS` with the IP address to request for the Service of type LoadBalancer. ++ +*Gateway* ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl patch gateway -n default my-gateway --type=merge -p '{ + "spec": { + "addresses": [{"type": "IPAddress", "value": "LB_IP_ADDRESS"}] + } +}' +---- ++ +*Ingress* ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl patch ingress my-ingress --type=merge -p '{ + "metadata": {"annotations": {"lbipam.cilium.io/ips": "LB_IP_ADDRESS"}} +}' +---- ++ +. Create a file named `cilium-lbip-pool-ingress.yaml` with a `CiliumLoadBalancerIPPool` resource to configure the Load Balancer IP address range for your Ingress / Gateway resources. +- If you are using Cilium Ingress, Cilium automatically applies the `cilium.io/ingress: "true"` label to the Services it creates for Ingress resources. You can use this label in the `serviceSelector` field of the `CiliumLoadBalancerIPPool` resource definition to select the Services eligible for LB IPAM. +- If you are using Cilium Gateway, you can use the `gateway.networking.k8s.io/gateway-name` label in the `serviceSelector` fields of the `CiliumLoadBalancerIPPool` resource definition to select the Gateway resources eligible for LB IPAM. +- Replace `LB_IP_CIDR` with the IP address range to use for the Load Balancer IP addresses. To select a single IP address, use a `/32` CIDR. For more information, see link:https://docs.cilium.io/en/stable/network/lb-ipam/[LoadBalancer IP Address Management] in the Cilium documentation. ++ +[source,yaml,subs="verbatim,attributes,quotes"] +---- +apiVersion: cilium.io/v2alpha1 +kind: CiliumLoadBalancerIPPool +metadata: + name: bookinfo-pool +spec: + blocks: + - cidr: "LB_IP_CIDR" + serviceSelector: + # if using Cilium Gateway + matchExpressions: + - { key: gateway.networking.k8s.io/gateway-name, operator: In, values: [ my-gateway ] } + # if using Cilium Ingress + matchLabels: + cilium.io/ingress: "true" +---- ++ +. Apply the `CiliumLoadBalancerIPPool` resource to your cluster. ++ +[source,bash] +---- +kubectl apply -f cilium-lbip-pool-ingress.yaml +---- ++ +. Confirm an IP address was allocated from Cilium LB IPAM for the Ingress / Gateway resource. ++ +*Gateway* ++ +[source,bash] +---- +kubectl get gateway my-gateway +---- ++ +[source,bash,subs="verbatim,attributes,quotes"] +---- +NAME CLASS ADDRESS PROGRAMMED AGE +my-gateway cilium [.replaceable]`LB_IP_ADDRESS` True 6m41s +---- ++ +*Ingress* ++ +[source,bash,subs="verbatim,attributes,quotes"] +---- +kubectl get ingress my-ingress +---- ++ +[source,bash,subs="verbatim,attributes,quotes"] +---- +NAME CLASS HOSTS ADDRESS PORTS AGE +my-ingress cilium * [.replaceable]`LB_IP_ADDRESS` 80 10m +---- ++ +. Create a file named `cilium-bgp-advertisement-ingress.yaml` with a `CiliumBGPAdvertisement` resource to advertise the LoadBalancer IP address for the Ingress / Gateway resources. If you are not using Cilium BGP, you can skip this step. The LoadBalancer IP address used for your Ingress / Gateway resource must be routable on your on-premises network for you to be able to query the service in the next step. ++ +[source,yaml] +---- +apiVersion: cilium.io/v2alpha1 +kind: CiliumBGPAdvertisement +metadata: + name: bgp-advertisement-lb-ip + labels: + advertise: bgp +spec: + advertisements: + - advertisementType: "Service" + service: + addresses: + - LoadBalancerIP + selector: + # if using Cilium Gateway + matchExpressions: + - { key: gateway.networking.k8s.io/gateway-name, operator: In, values: [ my-gateway ] } + # if using Cilium Ingress + matchLabels: + cilium.io/ingress: "true" +---- +. Apply the `CiliumBGPAdvertisement` resource to your cluster. ++ +[source,bash] +---- +kubectl apply -f cilium-bgp-advertisement-ingress.yaml +---- ++ +. Access the service using the IP address allocated from Cilium LB IPAM. ++ +[source,bash,subs="verbatim,attributes,quotes"] +---- +curl -s http://[.replaceable]`LB_IP_ADDRESS`:80/details/1 | jq +---- ++ +[source,bash] +---- +{ + "id": 1, + "author": "William Shakespeare", + "year": 1595, + "type": "paperback", + "pages": 200, + "publisher": "PublisherA", + "language": "English", + "ISBN-10": "1234567890", + "ISBN-13": "123-1234567890" +} +---- + +[#hybrid-nodes-ingress-cilium-nodeport] +== Service type NodePort + +If you do not have load balancer infrastructure and corresponding Ingress controller in your environment, or if you are self-managing your load balancer infrastructure or using DNS-based load balancing, you can configure Cilium Ingress to create Services of type NodePort for the Ingress resources. When using NodePort with Cilium Ingress, the Service of type NodePort is exposed on a port on each node in port range 30000-32767. In this mode, when traffic reaches any node in the cluster on the NodePort, it is then forwarded to a pod that backs the service, which may be on the same node or a different node. + +[NOTE] +==== +Cilium Gateway support for NodePort services is planned for Cilium version 1.18.x (link:https://github.com/cilium/cilium/pull/27273[#27273]) +==== + +=== Prerequisites + +- Cilium Ingress installed following the instructions in <>. +- Cilium Ingress resources with sample application deployed following the instructions in <>. + +=== Procedure + +. Patch the existing Ingress resource `my-ingress` to change it from Service type LoadBalancer to NodePort. ++ +[source,bash] +---- +kubectl patch ingress my-ingress --type=merge -p '{ + "metadata": {"annotations": {"ingress.cilium.io/service-type": "NodePort"}} +}' +---- ++ +If you have not created the Ingress resource, you can create it by applying the following Ingress definition to your cluster. Note, the Ingress definition below uses the Istio Bookinfo sample application described in <>. ++ +[source,yaml] +---- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: my-ingress + namespace: default + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: "external" + "ingress.cilium.io/service-type": "NodePort" +spec: + ingressClassName: cilium + rules: + - http: + paths: + - backend: + service: + name: details + port: + number: 9080 + path: /details + pathType: Prefix +---- ++ +. Confirm the Service for the Ingress resource was updated to use Service type NodePort. Note the Port for the HTTP protocol in the output. In the example below this HTTP port is `32353`, which will be used in a subsequent step to query the Service. The benefit of using Cilium Ingress with Service of type NodePort is that you can apply path and host-based routing, as well as network policies for the Ingress traffic, which you cannot do for a standard Service of type NodePort without Ingress. ++ +[source,bash] +---- +kubectl -n default get svc cilium-ingress-my-ingress +---- ++ +[source,bash] +---- +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +cilium-ingress-my-ingress NodePort 172.16.47.153 80:32353/TCP,443:30253/TCP 27m +---- ++ +. Get the IP addresses of your nodes in your cluster. ++ +[source,bash] +---- +kubectl get nodes -o wide +---- ++ +[source,bash] +---- +NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME +mi-026d6a261e355fba7 Ready 23h v1.32.3-eks-473151a 10.80.146.150 Ubuntu 22.04.5 LTS 5.15.0-142-generic containerd://1.7.27 +mi-082f73826a163626e Ready 23h v1.32.3-eks-473151a 10.80.146.32 Ubuntu 22.04.4 LTS 5.15.0-142-generic containerd://1.7.27 +mi-09183e8a3d755abf6 Ready 23h v1.32.3-eks-473151a 10.80.146.33 Ubuntu 22.04.4 LTS 5.15.0-142-generic containerd://1.7.27 +mi-0d78d815980ed202d Ready 23h v1.32.3-eks-473151a 10.80.146.97 Ubuntu 22.04.4 LTS 5.15.0-142-generic containerd://1.7.27 +mi-0daa253999fe92daa Ready 23h v1.32.3-eks-473151a 10.80.146.100 Ubuntu 22.04.4 LTS 5.15.0-142-generic containerd://1.7.27 +---- ++ +. Access the Service of type NodePort using the IP addresses of your nodes and the NodePort captured above. In the example below the node IP address used is `10.80.146.32` and the NodePort is `32353`. Replace these with the values for your environment. ++ +[source,bash] +---- +curl -s http://10.80.146.32:32353/details/1 | jq +---- ++ +[source,bash] +---- +{ + "id": 1, + "author": "William Shakespeare", + "year": 1595, + "type": "paperback", + "pages": 200, + "publisher": "PublisherA", + "language": "English", + "ISBN-10": "1234567890", + "ISBN-13": "123-1234567890" +} +---- + +[#hybrid-nodes-ingress-cilium-host-network] +== Host network + +Similar to Service of type NodePort, if you do not have load balancer infrastructure and an Ingress or Gateway controller, or if you are self-managing your load balancing with an external load balancer, you can configure Cilium Ingress and Cilium Gateway to expose Ingress and Gateway resources directly on the host network. When the host network mode is enabled for an Ingress or Gateway resource, the Service of type LoadBalancer and NodePort modes are automatically disabled, host network mode is mutually exclusive with these alternative modes for each Ingress or Gateway resource. Compared to the Service of type NodePort mode, host network mode offers additional flexibility for the range of ports that can be used (it's not restricted to the 30000-32767 NodePort range) and you can configure a subset of nodes where the Envoy proxies run on the host network. + +=== Prerequisites + +- Cilium Ingress or Gateway installed following the instructions in <> or <>. + +=== Procedure + +==== Gateway + +. Create a file named `cilium-gateway-host-network.yaml` with the following content. ++ +[source,yaml] +---- +gatewayAPI: + enabled: true + hostNetwork: + enabled: true + # uncomment to restrict nodes where Envoy proxies run on the host network + # nodes: + # matchLabels: + # role: gateway +---- ++ +. Apply the host network Cilium Gateway configuration to your cluster. ++ +[source,bash,subs="verbatim,attributes,quotes"] +---- +helm upgrade cilium oci://public.ecr.aws/eks/cilium/cilium \ + --namespace kube-system \ + --reuse-values \ + --set operator.rollOutPods=true \ + -f cilium-gateway-host-network.yaml +---- ++ +If you have not created the Gateway resource, you can create it by applying the following Gateway definition to your cluster. The Gateway definition below uses the Istio Bookinfo sample application described in <>. In the example below, the Gateway resource is configured to use the `8111` port for the HTTP listener, which is the shared listener port for the Envoy proxies running on the host network. If you are using a privileged port (lower than 1023) for the Gateway resource, reference the link:https://docs.cilium.io/en/stable/network/servicemesh/gateway-api/gateway-api/#bind-to-privileged-port[Cilium documentation] for instructions. ++ +[source,yaml] +---- +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: my-gateway +spec: + gatewayClassName: cilium + listeners: + - protocol: HTTP + port: 8111 + name: web-gw + allowedRoutes: + namespaces: + from: Same +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: http-app-1 +spec: + parentRefs: + - name: my-gateway + namespace: default + rules: + - matches: + - path: + type: PathPrefix + value: /details + backendRefs: + - name: details + port: 9080 +---- ++ +You can observe the applied Cilium Envoy Configuration with the following command. ++ +[source,bash] +---- +kubectl get cec cilium-gateway-my-gateway -o yaml +---- ++ +You can get the Envoy listener port for the `cilium-gateway-my-gateway` Service with the following command. In this example, the shared listener port is `8111`. ++ +[source,bash] +---- +kubectl get cec cilium-gateway-my-gateway -o jsonpath={.spec.services[0].ports[0]} +---- ++ +. Get the IP addresses of your nodes in your cluster. ++ +[source,bash] +---- +kubectl get nodes -o wide +---- ++ +[source,bash] +---- +NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME +mi-026d6a261e355fba7 Ready 23h v1.32.3-eks-473151a 10.80.146.150 Ubuntu 22.04.5 LTS 5.15.0-142-generic containerd://1.7.27 +mi-082f73826a163626e Ready 23h v1.32.3-eks-473151a 10.80.146.32 Ubuntu 22.04.4 LTS 5.15.0-142-generic containerd://1.7.27 +mi-09183e8a3d755abf6 Ready 23h v1.32.3-eks-473151a 10.80.146.33 Ubuntu 22.04.4 LTS 5.15.0-142-generic containerd://1.7.27 +mi-0d78d815980ed202d Ready 23h v1.32.3-eks-473151a 10.80.146.97 Ubuntu 22.04.4 LTS 5.15.0-142-generic containerd://1.7.27 +mi-0daa253999fe92daa Ready 23h v1.32.3-eks-473151a 10.80.146.100 Ubuntu 22.04.4 LTS 5.15.0-142-generic containerd://1.7.27 +---- ++ +. Access the Service using the IP addresses of your nodes and the listener port for the `cilium-gateway-my-gateway` resource. In the example below the node IP address used is `10.80.146.32` and the listener port is `8111`. Replace these with the values for your environment. ++ +[source,bash] +---- +curl -s http://10.80.146.32:8111/details/1 | jq +---- ++ +[source,bash] +---- +{ + "id": 1, + "author": "William Shakespeare", + "year": 1595, + "type": "paperback", + "pages": 200, + "publisher": "PublisherA", + "language": "English", + "ISBN-10": "1234567890", + "ISBN-13": "123-1234567890" +} +---- + +==== Ingress + +Due to an upstream Cilium issue (link:https://github.com/cilium/cilium/issues/34028[#34028]), Cilium Ingress in host network mode requires using `loadbalancerMode: shared`, which creates a single Service of type ClusterIP for all Ingress resources in the cluster. If you are using a privileged port (lower than 1023) for the Ingress resource, reference the link:https://docs.cilium.io/en/stable/network/servicemesh/ingress/#bind-to-privileged-port[Cilium documentation] for instructions. + +. Create a file named `cilium-ingress-host-network.yaml` with the following content. ++ +[source,yaml] +---- +ingressController: + enabled: true + loadbalancerMode: shared + # This is a workaround for the upstream Cilium issue + service: + externalTrafficPolicy: null + type: ClusterIP + hostNetwork: + enabled: true + # ensure the port does not conflict with other services on the node + sharedListenerPort: 8111 + # uncomment to restrict nodes where Envoy proxies run on the host network + # nodes: + # matchLabels: + # role: ingress +---- ++ +. Apply the host network Cilium Ingress configuration to your cluster. ++ +[source,bash,subs="verbatim,attributes,quotes"] +---- +helm upgrade cilium oci://public.ecr.aws/eks/cilium/cilium \ + --namespace kube-system \ + --reuse-values \ + --set operator.rollOutPods=true \ + -f cilium-ingress-host-network.yaml +---- ++ +If you have not created the Ingress resource, you can create it by applying the following Ingress definition to your cluster. The Ingress definition below uses the Istio Bookinfo sample application described in <>. ++ +[source,yaml] +---- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: my-ingress + namespace: default +spec: + ingressClassName: cilium + rules: + - http: + paths: + - backend: + service: + name: details + port: + number: 9080 + path: /details + pathType: Prefix +---- ++ +You can observe the applied Cilium Envoy Configuration with the following command. ++ +[source,bash] +---- +kubectl get cec -n kube-system cilium-ingress -o yaml +---- ++ +You can get the Envoy listener port for the `cilium-ingress` Service with the following command. In this example, the shared listener port is `8111`. ++ +[source,bash] +---- +kubectl get cec -n kube-system cilium-ingress -o jsonpath={.spec.services[0].ports[0]} +---- ++ +. Get the IP addresses of your nodes in your cluster. ++ +[source,bash] +---- +kubectl get nodes -o wide +---- ++ +[source,bash] +---- +NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME +mi-026d6a261e355fba7 Ready 23h v1.32.3-eks-473151a 10.80.146.150 Ubuntu 22.04.5 LTS 5.15.0-142-generic containerd://1.7.27 +mi-082f73826a163626e Ready 23h v1.32.3-eks-473151a 10.80.146.32 Ubuntu 22.04.4 LTS 5.15.0-142-generic containerd://1.7.27 +mi-09183e8a3d755abf6 Ready 23h v1.32.3-eks-473151a 10.80.146.33 Ubuntu 22.04.4 LTS 5.15.0-142-generic containerd://1.7.27 +mi-0d78d815980ed202d Ready 23h v1.32.3-eks-473151a 10.80.146.97 Ubuntu 22.04.4 LTS 5.15.0-142-generic containerd://1.7.27 +mi-0daa253999fe92daa Ready 23h v1.32.3-eks-473151a 10.80.146.100 Ubuntu 22.04.4 LTS 5.15.0-142-generic containerd://1.7.27 +---- ++ +. Access the Service using the IP addresses of your nodes and the `sharedListenerPort` for the `cilium-ingress` resource. In the example below the node IP address used is `10.80.146.32` and the listener port is `8111`. Replace these with the values for your environment. ++ +[source,bash] +---- +curl -s http://10.80.146.32:8111/details/1 | jq +---- ++ +[source,bash] +---- +{ + "id": 1, + "author": "William Shakespeare", + "year": 1595, + "type": "paperback", + "pages": 200, + "publisher": "PublisherA", + "language": "English", + "ISBN-10": "1234567890", + "ISBN-13": "123-1234567890" +} +---- diff --git a/latest/ug/nodes/hybrid-nodes-join.adoc b/latest/ug/nodes/hybrid-nodes-join.adoc index 37db4d870..7516682f2 100644 --- a/latest/ug/nodes/hybrid-nodes-join.adoc +++ b/latest/ug/nodes/hybrid-nodes-join.adoc @@ -1,20 +1,20 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[hybrid-nodes-join,hybrid-nodes-join.title]] +[#hybrid-nodes-join] = Connect hybrid nodes -:info_doctype: section -:info_title: Connect hybrid nodes to Amazon EKS cluster :info_titleabbrev: Connect hybrid nodes -:keywords: on-premises, hybrid -:info_abstract: Connect hybrid nodes to Amazon EKS cluster - -include::../attributes.txt[] [abstract] -- Connect hybrid nodes to Amazon EKS cluster. -- +[NOTE] +==== +The following steps apply to hybrid nodes running compatible operating systems except Bottlerocket. For steps to connect a hybrid node that runs Bottlerocket, see <>. +==== + This topic describes how to connect hybrid nodes to an Amazon EKS cluster. After your hybrid nodes join the cluster, they will appear with status Not Ready in the Amazon EKS console and in Kubernetes-compatible tooling such as kubectl. After completing the steps on this page, proceed to <> to make your hybrid nodes ready to run applications. == Prerequisites @@ -65,13 +65,12 @@ Run the command below to install the hybrid nodes dependencies on your on-premis The hybrid nodes CLI (`nodeadm`) must be run with a user that has sudo/root access on your host. ==== -* Replace `K8S_VERSION` with the Kubernetes minor version of your Amazon EKS cluster, for example `1.31`. See link:eks/latest/userguide/kubernetes-versions.html[Amazon EKS Kubernetes versions,type="documentation"] for a list of the supported Kubernetes versions. +* Replace `K8S_VERSION` with the Kubernetes minor version of your Amazon EKS cluster, for example `1.31`. See link:eks/latest/userguide/kubernetes-versions.html[Amazon EKS supported versions,type="documentation"] for a list of the supported Kubernetes versions. * Replace `CREDS_PROVIDER` with the on-premises credential provider you are using. Valid values are `ssm` for {aws} SSM and `iam-ra` for {aws} IAM Roles Anywhere. [source,bash,subs="verbatim,attributes,quotes"] ---- nodeadm install [.replaceable]`K8S_VERSION` --credential-provider [.replaceable]`CREDS_PROVIDER` - ---- == Step 3: Connect hybrid nodes to your cluster @@ -137,7 +136,7 @@ spec: nodeadm init -c file://nodeConfig.yaml ---- -If the above command completes successfully, your hybrid node has joined your Amazon EKS cluster. You can verify this in the Amazon EKS console by navigating to the Compute tab for your cluster (https://docs.aws.amazon.com/eks/latest/userguide/view-kubernetes-resources.html#view-kubernetes-resources-permissions[ensure IAM principal has permissions to view]) or with `kubectl get nodes`. +If the above command completes successfully, your hybrid node has joined your Amazon EKS cluster. You can verify this in the Amazon EKS console by navigating to the Compute tab for your cluster (<>) or with `kubectl get nodes`. [IMPORTANT] ==== @@ -146,4 +145,4 @@ Your nodes will have status `Not Ready`, which is expected and is due to the lac == Step 4: Configure a CNI for hybrid nodes -To make your hybrid nodes ready to run applications, continue with the steps on <>. +To make your hybrid nodes ready to run applications, continue with the steps on <>. \ No newline at end of file diff --git a/latest/ug/nodes/hybrid-nodes-load-balancing.adoc b/latest/ug/nodes/hybrid-nodes-load-balancing.adoc new file mode 100644 index 000000000..e21bd3442 --- /dev/null +++ b/latest/ug/nodes/hybrid-nodes-load-balancing.adoc @@ -0,0 +1,427 @@ +include::../attributes.txt[] + +[.topic] +[#hybrid-nodes-load-balancing] += Configure Services of type LoadBalancer for hybrid nodes +:info_titleabbrev: Configure LoadBalancer Services + +[abstract] +-- +Configure Services of type LoadBalancer for hybrid nodes +-- + +This topic describes how to configure Layer 4 (L4) load balancing for applications running on Amazon EKS Hybrid Nodes. Kubernetes Services of type LoadBalancer are used to expose Kubernetes applications external to the cluster. Services of type LoadBalancer are commonly used with physical load balancer infrastructure in the cloud or on-premises environment to serve the workload's traffic. This load balancer infrastructure is commonly provisioned with an environment-specific controller. + +{aws} supports {aws} Network Load Balancer (NLB) and Cilium for Services of type LoadBalancer running on EKS Hybrid Nodes. The decision to use NLB or Cilium is based on the source of application traffic. If application traffic originates from an {aws} Region, {aws} recommends using {aws} NLB and the {aws} Load Balancer Controller. If application traffic originates from the local on-premises or edge environment, {aws} recommends using Cilium's built-in load balancing capabilities, which can be used with or without load balancer infrastructure in your environment. + +For Layer 7 (L7) application traffic load balancing, see <>. For general information on Load Balancing with EKS, see link:eks/latest/best-practices/load-balancing.html[Best Practices for Load Balancing,type="documentation"]. + +[#hybrid-nodes-service-lb-nlb] +== {aws} Network Load Balancer + +You can use the <> and NLB with the target type `ip` for workloads running on hybrid nodes. When using target type `ip`, NLB forwards traffic directly to the pods, bypassing the Service layer network path. For NLB to reach the pod IP targets on hybrid nodes, your on-premises pod CIDRs must be routable on your on-premises network. Additionally, the {aws} Load Balancer Controller uses webhooks and requires direct communication from the EKS control plane. For more information, see <>. + +- See <> for subnet configuration requirements, and <> and link:eks/latest/best-practices/load-balancing.html[Best Practices for Load Balancing,type="documentation"] for additional information about {aws} Network Load Balancer and {aws} Load Balancer Controller. +- See link:https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/guide/service/nlb/[{aws} Load Balancer Controller NLB configurations] for configurations that can be applied to Services of type `LoadBalancer` with {aws} Network Load Balancer. + +=== Prerequisites + +- Cilium installed following the instructions in <>. +- Cilium BGP Control Plane enabled following the instructions in <>. If you do not want to use BGP, you must use an alternative method to make your on-premises pod CIDRs routable on your on-premises network, see <> for more information. +- Helm installed in your command-line environment, see <>. +- eksctl installed in your command-line environment, see <>. + +=== Procedure + +. Download an IAM policy for the {aws} Load Balancer Controller that allows it to make calls to {aws} APIs on your behalf. ++ +[source,bash] +---- +curl -O https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/refs/heads/main/docs/install/iam_policy.json +---- ++ +. Create an IAM policy using the policy downloaded in the previous step. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam create-policy \ + --policy-name AWSLoadBalancerControllerIAMPolicy \ + --policy-document file://iam_policy.json +---- ++ +. Replace the values for cluster name (`CLUSTER_NAME`), {aws} Region (`AWS_REGION`), and {aws} account ID (`AWS_ACCOUNT_ID`) with your settings and run the following command. ++ +[source,bash,subs="verbatim,attributes"] +---- +eksctl create iamserviceaccount \ + --cluster=CLUSTER_NAME \ + --namespace=kube-system \ + --name=aws-load-balancer-controller \ + --attach-policy-arn=arn:aws:iam::AWS_ACCOUNT_ID:policy/AWSLoadBalancerControllerIAMPolicy \ + --override-existing-serviceaccounts \ + --region AWS_REGION \ + --approve +---- ++ +. Add the eks-charts Helm chart repository. {aws} maintains this repository on GitHub. ++ +[source,bash] +---- +helm repo add eks https://aws.github.io/eks-charts +---- ++ +. Update your local Helm repository to make sure that you have the most recent charts. ++ +[source,bash] +---- +helm repo update eks +---- ++ +. Install the {aws} Load Balancer Controller. Replace the values for cluster name (`CLUSTER_NAME`), {aws} Region (`AWS_REGION`), VPC ID (`VPC_ID`), and {aws} Load Balancer Controller Helm chart version (`AWS_LBC_HELM_VERSION`) with your settings. You can find the latest version of the Helm chart by running `helm search repo eks/aws-load-balancer-controller --versions`. If you are running a mixed mode cluster with both hybrid nodes and nodes in {aws} Cloud, you can run the {aws} Load Balancer Controller on cloud nodes following the instructions at <>. ++ +[source,bash,subs="verbatim,attributes,quotes"] +---- +helm install aws-load-balancer-controller eks/aws-load-balancer-controller \ + -n kube-system \ + --version [.replaceable]`AWS_LBC_HELM_VERSION` \ + --set clusterName=[.replaceable]`CLUSTER_NAME` \ + --set region=[.replaceable]`AWS_REGION` \ + --set vpcId=[.replaceable]`VPC_ID` \ + --set serviceAccount.create=false \ + --set serviceAccount.name=aws-load-balancer-controller +---- ++ +. Verify the {aws} Load Balancer Controller was installed successfully. ++ +[source,bash] +---- +kubectl get -n kube-system deployment aws-load-balancer-controller +---- ++ +[source,bash] +---- +NAME READY UP-TO-DATE AVAILABLE AGE +aws-load-balancer-controller 2/2 2 2 84s +---- +. Define a sample application in a file named `tcp-sample-app.yaml`. The example below uses a simple NGINX deployment with a TCP port. ++ +[source,bash] +---- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tcp-sample-app + namespace: default +spec: + replicas: 3 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: public.ecr.aws/nginx/nginx:1.23 + ports: + - name: tcp + containerPort: 80 +---- ++ +. Apply the deployment to your cluster. ++ +[source,bash] +---- +kubectl apply -f tcp-sample-app.yaml +---- ++ +. Define a Service of type LoadBalancer for the deployment in a file named `tcp-sample-service.yaml`. ++ +[source,yaml] +---- +apiVersion: v1 +kind: Service +metadata: + name: tcp-sample-service + namespace: default + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: external + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip + service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing +spec: + ports: + - port: 80 + targetPort: 80 + protocol: TCP + type: LoadBalancer + selector: + app: nginx +---- ++ +. Apply the Service configuration to your cluster. ++ +[source,bash] +---- +kubectl apply -f tcp-sample-service.yaml +---- ++ +. Provisioning the NLB for the Service may take a few minutes. Once the NLB is provisioned, the Service will have an address assigned to it that corresponds to the DNS name of the NLB deployment. ++ +[source,bash] +---- +kubectl get svc tcp-sample-service +---- ++ +[source,bash] +---- +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +tcp-sample-service LoadBalancer 172.16.115.212 k8s-default-tcpsampl-xxxxxxxxxx-xxxxxxxxxxxxxxxx.elb..amazonaws.com 80:30396/TCP 8s +---- ++ +. Access the Service using the address of the NLB. ++ +[source,bash] +---- +curl k8s-default-tcpsampl-xxxxxxxxxx-xxxxxxxxxxxxxxxx.elb..amazonaws.com +---- +An example output is below. ++ +[source,bash] +---- + + + +Welcome to nginx! +[...] +---- ++ +. Clean up the resources you created. ++ +[source,bash] +---- +kubectl delete -f tcp-sample-service.yaml +kubectl delete -f tcp-sample-app.yaml +---- + +[#hybrid-nodes-service-lb-cilium] +== Cilium in-cluster load balancing + +Cilium can be used as an in-cluster load balancer for workloads running on EKS Hybrid Nodes, which can be useful for environments that do not have load balancer infrastructure. Cilium's load balancing capabilities are built on a combination of Cilium features including kube-proxy replacement, Load Balancer IP Address Management (IPAM), and BGP Control Plane. The responsibilities of these features are detailed below: + +- *Cilium kube-proxy replacement*: Handles routing Service traffic to backend pods. +- *Cilium Load Balancer IPAM*: Manages IP addresses that can be assigned to Services of type `LoadBalancer`. +- *Cilium BGP Control Plane*: Advertises IP addresses allocated by Load Balancer IPAM to the on-premises network. + +If you are not using Cilium's kube-proxy replacement, you can still use Cilium Load Balancer IPAM and BGP Control Plane to allocate and assign IP addresses for Services of type LoadBalancer. If you are not using Cilium's kube-proxy replacement, the load balancing for Services to backend pods is handled by kube-proxy and iptables rules by default in EKS. + +=== Prerequisites + +- Cilium installed following the instructions in <> with or without kube-proxy replacement enabled. Cilium's kube-proxy replacement requires running an operating system with a Linux kernel at least as recent as v4.19.57, v5.1.16, or v5.2.0. All recent versions of the operating systems supported for use with hybrid nodes meet this criteria, with the exception of Red Hat Enterprise Linux (RHEL) 8.x. +- Cilium BGP Control Plane enabled following the instructions in <>. If you do not want to use BGP, you must use an alternative method to make your on-premises pod CIDRs routable on your on-premises network, see <> for more information. +- Helm installed in your command-line environment, see <>. + +=== Procedure + +. Create a file named `cilium-lbip-pool-loadbalancer.yaml` with a `CiliumLoadBalancerIPPool` resource to configure the Load Balancer IP address range for your Services of type LoadBalancer. ++ +- Replace `LB_IP_CIDR` with the IP address range to use for the Load Balancer IP addresses. To select a single IP address, use a `/32` CIDR. For more information, see link:https://docs.cilium.io/en/stable/network/lb-ipam/[LoadBalancer IP Address Management] in the Cilium documentation. +- The `serviceSelector` field is configured to match against the name of the Service you will create in a subsequent step. With this configuration, IPs from this pool will only be allocated to Services with the name `tcp-sample-service`. ++ +[source,yaml,subs="verbatim,attributes"] +---- +apiVersion: cilium.io/v2alpha1 +kind: CiliumLoadBalancerIPPool +metadata: + name: tcp-service-pool +spec: + blocks: + - cidr: "LB_IP_CIDR" + serviceSelector: + matchLabels: + io.kubernetes.service.name: tcp-sample-service +---- ++ +. Apply the `CiliumLoadBalancerIPPool` resource to your cluster. ++ +[source,bash] +---- +kubectl apply -f cilium-lbip-pool-loadbalancer.yaml +---- ++ +. Confirm there is at least one IP address available in the pool. ++ +[source,bash] +---- +kubectl get ciliumloadbalancerippools.cilium.io +---- ++ +[source,bash] +---- +NAME DISABLED CONFLICTING IPS AVAILABLE AGE +tcp-service-pool false False 1 24m +---- ++ +. Create a file named `cilium-bgp-advertisement-loadbalancer.yaml` with a `CiliumBGPAdvertisement` resource to advertise the load balancer IP address for the Service you will create in the next step. If you are not using Cilium BGP, you can skip this step. The load balancer IP address used for your Service must be routable on your on-premises network for you to be able to query the service in the final step. +- The `advertisementType` field is set to `Service` and `service.addresses` is set to `LoadBalancerIP` to only advertise the `LoadBalancerIP` for Services of type `LoadBalancer`. +- The `selector` field is configured to match against the name of the Service you will create in a subsequent step. With this configuration, only `LoadBalancerIP` for Services with the name `tcp-sample-service` will be advertised. ++ +[source,yaml,subs="verbatim,attributes,quotes"] +---- +apiVersion: cilium.io/v2alpha1 +kind: CiliumBGPAdvertisement +metadata: + name: bgp-advertisement-tcp-service + labels: + advertise: bgp +spec: + advertisements: + - advertisementType: "Service" + service: + addresses: + - LoadBalancerIP + selector: + matchLabels: + io.kubernetes.service.name: tcp-sample-service +---- ++ +. Apply the `CiliumBGPAdvertisement` resource to your cluster. If you are not using Cilium BGP, you can skip this step. ++ +[source,bash] +---- +kubectl apply -f cilium-bgp-advertisement-loadbalancer.yaml +---- ++ +. Define a sample application in a file named `tcp-sample-app.yaml`. The example below uses a simple NGINX deployment with a TCP port. ++ +[source,bash] +---- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tcp-sample-app + namespace: default +spec: + replicas: 3 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: public.ecr.aws/nginx/nginx:1.23 + ports: + - name: tcp + containerPort: 80 +---- ++ +. Apply the deployment to your cluster. ++ +[source,bash] +---- +kubectl apply -f tcp-sample-app.yaml +---- ++ +. Define a Service of type LoadBalancer for the deployment in a file named `tcp-sample-service.yaml`. ++ +- You can request a specific IP address from the load balancer IP pool with the `lbipam.cilium.io/ips` annotation on the Service object. You can remove this annotation if you do not want to request a specific IP address for the Service. +- The `loadBalancerClass` spec field is required to prevent the legacy {aws} Cloud Provider from creating a Classic Load Balancer for the Service. In the example below this is configured to `io.cilium/bgp-control-plane` to use Cilium's BGP Control Plane as the load balancer class. This field can alternatively be configured to `io.cilium/l2-announcer` to use Cilium's link:https://docs.cilium.io/en/latest/network/l2-announcements/[L2 Announcements feature] (currently in beta and not officially supported by {aws}). ++ +[source,yaml,subs="verbatim,attributes,quotes"] +---- +apiVersion: v1 +kind: Service +metadata: + name: tcp-sample-service + namespace: default + annotations: + lbipam.cilium.io/ips: [.replaceable]`"LB_IP_ADDRESS"` +spec: + loadBalancerClass: io.cilium/bgp-control-plane + ports: + - port: 80 + targetPort: 80 + protocol: TCP + type: LoadBalancer + selector: + app: nginx +---- ++ +. Apply the Service to your cluster. The Service will be created with an external IP address that you can use to access the application. ++ +[source,bash] +---- +kubectl apply -f tcp-sample-service.yaml +---- +. Verify the Service was created successfully and has an IP assigned to it from the `CiliumLoadBalancerIPPool` created in the previous step. ++ +[source,bash] +---- +kubectl get svc tcp-sample-service +---- ++ +[source,bash,subs="verbatim,attributes,quotes"] +---- +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +tcp-sample-service LoadBalancer 172.16.117.76 [.replaceable]`LB_IP_ADDRESS` 80:31129/TCP 14m +---- ++ +. If you are using Cilium in kube-proxy replacement mode, you can confirm Cilium is handling the load balancing for the Service by running the following command. In the output below, the `10.86.2.x` addresses are the pod IP addresses of the backend pods for the Service. ++ +[source,bash] +---- +kubectl -n kube-system exec ds/cilium -- cilium-dbg service list +---- ++ +[source,bash,subs="verbatim,attributes,quotes"] +---- +ID Frontend Service Type Backend +... +41 [.replaceable]`LB_IP_ADDRESS`:80/TCP LoadBalancer 1 => 10.86.2.76:80/TCP (active) + 2 => 10.86.2.130:80/TCP (active) + 3 => 10.86.2.141:80/TCP (active) +---- ++ +. Confirm Cilium is advertising the IP address to the on-premises network via BGP. In the example below, there are five hybrid nodes, each advertising the `LB_IP_ADDRESS` for the `tcp-sample-service` Service to the on-premises network. ++ +[source,bash,subs="verbatim,attributes,quotes"] +---- +Node VRouter Prefix NextHop Age Attrs +mi-026d6a261e355fba7 [.replaceable]`NODES_ASN` [.replaceable]`LB_IP_ADDRESS`/32 0.0.0.0 12m3s [{Origin: i} {Nexthop: 0.0.0.0}] +mi-082f73826a163626e [.replaceable]`NODES_ASN` [.replaceable]`LB_IP_ADDRESS`/32 0.0.0.0 12m3s [{Origin: i} {Nexthop: 0.0.0.0}] +mi-09183e8a3d755abf6 [.replaceable]`NODES_ASN` [.replaceable]`LB_IP_ADDRESS`/32 0.0.0.0 12m3s [{Origin: i} {Nexthop: 0.0.0.0}] +mi-0d78d815980ed202d [.replaceable]`NODES_ASN` [.replaceable]`LB_IP_ADDRESS`/32 0.0.0.0 12m3s [{Origin: i} {Nexthop: 0.0.0.0}] +mi-0daa253999fe92daa [.replaceable]`NODES_ASN` [.replaceable]`LB_IP_ADDRESS`/32 0.0.0.0 12m3s [{Origin: i} {Nexthop: 0.0.0.0}] +---- ++ +. Access the Service using the assigned load balancerIP address. ++ +[source,bash,subs="verbatim,attributes,quotes"] +---- +curl [.replaceable]`LB_IP_ADDRESS` +---- +An example output is below. ++ +[source,bash] +---- + + + +Welcome to nginx! +[...] +---- ++ +. Clean up the resources you created. ++ +[source,bash] +---- +kubectl delete -f tcp-sample-service.yaml +kubectl delete -f tcp-sample-app.yaml +kubectl delete -f cilium-lb-ip-pool.yaml +kubectl delete -f cilium-bgp-advertisement.yaml +---- diff --git a/latest/ug/nodes/hybrid-nodes-network-policy.adoc b/latest/ug/nodes/hybrid-nodes-network-policy.adoc new file mode 100644 index 000000000..d8451cf89 --- /dev/null +++ b/latest/ug/nodes/hybrid-nodes-network-policy.adoc @@ -0,0 +1,372 @@ +include::../attributes.txt[] + +[.topic] +[#hybrid-nodes-network-policies] += Configure Kubernetes Network Policies for hybrid nodes +:info_titleabbrev: Configure Network Policies + +[abstract] +-- +Configure Kubernetes Network Policies for hybrid nodes +-- + +{aws} supports Kubernetes Network Policies (Layer 3 / Layer 4) for pod ingress and egress traffic when using Cilium as the CNI with EKS Hybrid Nodes. If you are running EKS clusters with nodes in {aws} Cloud, {aws} supports the <>. + +This topic covers how to configure Cilium and Kubernetes Network Policies with EKS Hybrid Nodes. For detailed information on Kubernetes Network Policies, see link:https://kubernetes.io/docs/concepts/services-networking/network-policies/[Kubernetes Network Policies] in the Kubernetes documentation. + +[#hybrid-nodes-configure-network-policies] +== Configure network policies + +=== Considerations + +- {aws} supports the upstream Kubernetes Network Policies and specfication for pod ingress and egress. {aws} currently does not support `CiliumNetworkPolicy` or `CiliumClusterwideNetworkPolicy`. +- The `policyEnforcementMode` Helm value can be used to control the default Cilium policy enforcement behavior. The default behavior allows all egress and ingress traffic. When an endpoint is selected by a network policy, it transitions to a default-deny state, where only explicitly allowed traffic is allowed. See the Cilium documentation for more information on the link:https://docs.cilium.io/en/stable/security/policy/intro/#policy-mode-default[default policy mode] and link:https://docs.cilium.io/en/stable/security/policy/intro/#policy-enforcement-modes[policy enforcement modes]. +- If you are changing `policyEnforcementMode` for an existing Cilium installation, you must restart the Cilium agent DaemonSet to apply the new policy enforcement mode. +- Use `namespaceSelector` and `podSelector` to allow or deny traffic to/from namespaces and pods with matching labels. The `namespaceSelector` and `podSelector` can be used with `matchLabels` or `matchExpressions` to select namespaces and pods based on their labels. +- Use `ingress.ports` and `egress.ports` to allow or deny traffic to/from ports and protocols. +- The `ipBlock` field cannot be used to selectively allow or deny traffic to/from pod IP addresses (link:https://github.com/cilium/cilium/issues/9209[#9209]). Using `ipBlock` selectors for node IPs is a beta feature in Cilium and is not supported by {aws}. +- See the link:https://kubernetes.io/docs/concepts/services-networking/network-policies/#networkpolicy-resource[NetworkPolicy resource] in the Kubernetes documentation for information on the available fields for Kubernetes Network Policies. + +=== Prerequisites + +- Cilium installed following the instructions in <>. +- Helm installed in your command-line environment, see <>. + +=== Procedure + +The following procedure sets up network policies for a sample microservices application so that components can only talk to other components that are required for the application to function. The procedure uses the link:https://istio.io/latest/docs/examples/bookinfo/[Istio Bookinfo] sample microservices application. + +The Bookinfo application consists of four separate microservices with the following relationships: + +- *productpage*. The productpage microservice calls the details and reviews microservices to populate the page. +- *details*. The details microservice contains book information. +- *reviews*. The reviews microservice contains book reviews. It also calls the ratings microservice. +- *ratings*. The ratings microservice contains book ranking information that accompanies a book review. + +. Create the sample application. ++ +[source,bash] +---- +kubectl apply -f https://raw.githubusercontent.com/istio/istio/refs/heads/master/samples/bookinfo/platform/kube/bookinfo.yaml +---- ++ +. Confirm the application is running successfully and note the pod IP address for the productpage microservice. You will use this pod IP address to query each microservice in the subsequent steps. ++ +[source,bash] +---- +kubectl get pods -o wide +---- ++ +[source,bash] +---- +NAME READY STATUS RESTARTS AGE IP NODE +details-v1-766844796b-9wff2 1/1 Running 0 7s 10.86.3.7 mi-0daa253999fe92daa +productpage-v1-54bb874995-lwfgg 1/1 Running 0 7s 10.86.2.193 mi-082f73826a163626e +ratings-v1-5dc79b6bcd-59njm 1/1 Running 0 7s 10.86.2.232 mi-082f73826a163626e +reviews-v1-598b896c9d-p2289 1/1 Running 0 7s 10.86.2.47 mi-026d6a261e355fba7 +reviews-v2-556d6457d-djktc 1/1 Running 0 7s 10.86.3.58 mi-0daa253999fe92daa +reviews-v3-564544b4d6-g8hh4 1/1 Running 0 7s 10.86.2.69 mi-09183e8a3d755abf6 +---- ++ +. Create a pod that will be used throughout to test the network policies. Note the pod is created in the `default` namespace with the label `access: true`. ++ +[source,bash] +---- +kubectl run curl-pod --image=curlimages/curl -i --tty --labels=access=true --namespace=default --overrides='{"spec": { "nodeSelector": {"eks.amazonaws.com/compute-type": "hybrid"}}}' -- /bin/sh +---- ++ +. Test access to the productpage microservice. In the example below, we use the pod IP address of the productpage pod (`10.86.2.193`) to query the microservice. Replace this with the pod IP address of the productpage pod in your environment. ++ +[source,bash] +---- +curl -s http://10.86.2.193:9080/productpage | grep -o ".*" +---- ++ +[source,bash] +---- +Simple Bookstore App +---- ++ +. You can exit the test curl pod by typing `exit` and can reattach to the pod by running the following command. ++ +[source,bash] +---- +kubectl attach curl-pod -c curl-pod -i -t +---- ++ +. To demonstrate the effects of the network policies in the following steps, we first create a network policy that denies all traffic for the BookInfo microservices. Create a file called `network-policy-deny-bookinfo.yaml` that defines the deny network policy. ++ +[source,yaml] +---- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: deny-bookinfo + namespace: default +spec: + podSelector: + matchExpressions: + - key: app + operator: In + values: ["productpage", "details", "reviews", "ratings"] + policyTypes: + - Ingress + - Egress +---- ++ +. Apply the deny network policy to your cluster. ++ +[source,bash] +---- +kubectl apply -f network-policy-default-deny-bookinfo.yaml +---- ++ +. Test access to the BookInfo application. In the example below, we use the pod IP address of the productpage pod (`10.86.2.193`) to query the microservice. Replace this with the pod IP address of the productpage pod in your environment. ++ +[source,bash] +---- +curl http://10.86.2.193:9080/productpage --max-time 10 +---- ++ +[source,bash] +---- +curl: (28) Connection timed out after 10001 milliseconds +---- ++ +. Create a file called `network-policy-productpage.yaml` that defines the productpage network policy. The policy has the following rules: ++ +* allows ingress traffic from pods with the label `access: true` (the curl pod created in the previous step) +* allows egress TCP traffic on port `9080` for the details, reviews, and ratings microservices +* allows egress TCP/UDP traffic on port `53` for CoreDNS which runs in the `kube-system` namespace ++ +[source,yaml] +---- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: productpage-policy + namespace: default +spec: + podSelector: + matchLabels: + app: productpage + policyTypes: + - Ingress + - Egress + ingress: + - from: + - podSelector: + matchLabels: + access: "true" + egress: + - to: + - podSelector: + matchExpressions: + - key: app + operator: In + values: ["details", "reviews", "ratings"] + ports: + - port: 9080 + protocol: TCP + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP +---- ++ +. Apply the productpage network policy to your cluster. ++ +[source,bash] +---- +kubectl apply -f network-policy-productpage.yaml +---- ++ +. Connect to the curl pod and test access to the Bookinfo application. Access to the productpage microservice is now allowed, but the other microservices are still denied because they are still subject to the deny network policy. In the examples below, we use the pod IP address of the productpage pod (`10.86.2.193`) to query the microservice. Replace this with the pod IP address of the productpage pod in your environment. ++ +[source,bash] +---- +kubectl attach curl-pod -c curl-pod -i -t +---- ++ +[source,bash] +---- +curl -s http://10.86.2.193:9080/productpage | grep -o ".*" +Simple Bookstore App +---- ++ +[source,bash] +---- +curl -s http://10.86.2.193:9080/api/v1/products/1 +{"error": "Sorry, product details are currently unavailable for this book."} +---- ++ +[source,bash] +---- +curl -s http://10.86.2.193:9080/api/v1/products/1/reviews +{"error": "Sorry, product reviews are currently unavailable for this book."} +---- ++ +[source,bash] +---- +curl -s http://10.86.2.193:9080/api/v1/products/1/ratings +{"error": "Sorry, product ratings are currently unavailable for this book."} +---- ++ +. Create a file called `network-policy-details.yaml` that defines the details network policy. The policy allows only ingress traffic from the productpage microservice. ++ +[source,yaml] +---- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: details-policy + namespace: default +spec: + podSelector: + matchLabels: + app: details + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + app: productpage +---- ++ +. Create a file called `network-policy-reviews.yaml` that defines the reviews network policy. The policy allows only ingress traffic from the productpage microservice and only egress traffic to the ratings microservice and CoreDNS. ++ +[source,yaml] +---- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: reviews-policy + namespace: default +spec: + podSelector: + matchLabels: + app: reviews + policyTypes: + - Ingress + - Egress + ingress: + - from: + - podSelector: + matchLabels: + app: productpage + egress: + - to: + - podSelector: + matchLabels: + app: ratings + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP +---- ++ +. Create a file called `network-policy-ratings.yaml` that defines the ratings network policy. The policy allows only ingress traffic from the productpage and reviews microservices. ++ +[source,yaml] +---- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: ratings-policy + namespace: default +spec: + podSelector: + matchLabels: + app: ratings + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchExpressions: + - key: app + operator: In + values: ["productpage", "reviews"] +---- ++ +. Apply the details, reviews, and ratings network policies to your cluster. ++ +[source,bash] +---- +kubectl apply -f network-policy-details.yaml +kubectl apply -f network-policy-reviews.yaml +kubectl apply -f network-policy-ratings.yaml +---- ++ +. Connect to the curl pod and test access to the Bookinfo application. In the examples below, we use the pod IP address of the productpage pod (`10.86.2.193`) to query the microservice. Replace this with the pod IP address of the productpage pod in your environment. ++ +[source,bash] +---- +kubectl attach curl-pod -c curl-pod -i -t +---- ++ +Test the details microservice. ++ +[source,bash] +---- +curl -s http://10.86.2.193:9080/api/v1/products/1 +---- ++ +[source,bash] +---- +{"id": 1, "author": "William Shakespeare", "year": 1595, "type": "paperback", "pages": 200, "publisher": "PublisherA", "language": "English", "ISBN-10": "1234567890", "ISBN-13": "123-1234567890"} +---- ++ +Test the reviews microservice. ++ +[source,bash] +---- +curl -s http://10.86.2.193:9080/api/v1/products/1/reviews +---- ++ +[source,bash] +---- +{"id": "1", "podname": "reviews-v1-598b896c9d-p2289", "clustername": "null", "reviews": [{"reviewer": "Reviewer1", "text": "An extremely entertaining play by Shakespeare. The slapstick humour is refreshing!"}, {"reviewer": "Reviewer2", "text": "Absolutely fun and entertaining. The play lacks thematic depth when compared to other plays by Shakespeare."}]} +---- ++ +Test the ratings microservice. ++ +[source,bash] +---- +curl -s http://10.86.2.193:9080/api/v1/products/1/ratings +---- ++ +[source,bash] +---- +{"id": 1, "ratings": {"Reviewer1": 5, "Reviewer2": 4}} +---- ++ +. Clean up the resources you created in this procedure. ++ +[source,bash] +---- +kubectl delete -f network-policy-deny-bookinfo.yaml +kubectl delete -f network-policy-productpage.yaml +kubectl delete -f network-policy-details.yaml +kubectl delete -f network-policy-reviews.yaml +kubectl delete -f network-policy-ratings.yaml +kubectl delete -f https://raw.githubusercontent.com/istio/istio/refs/heads/master/samples/bookinfo/platform/kube/bookinfo.yaml +kubectl delete pod curl-pod +---- \ No newline at end of file diff --git a/latest/ug/nodes/hybrid-nodes-networking.adoc b/latest/ug/nodes/hybrid-nodes-networking.adoc index 69cd37d74..00a1857e9 100644 --- a/latest/ug/nodes/hybrid-nodes-networking.adoc +++ b/latest/ug/nodes/hybrid-nodes-networking.adoc @@ -1,18 +1,13 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[hybrid-nodes-networking,hybrid-nodes-networking.title]] +[#hybrid-nodes-networking] = Prepare networking for hybrid nodes -:info_doctype: section -:info_title: Prepare networking for hybrid nodes :info_titleabbrev: Prepare networking -:keywords: on-premises, hybrid -:info_abstract: Prepare networking - -include::../attributes.txt[] [abstract] -- -Learn about and configure the VPC and on-premises networking for joining nodes from your data centers to Amazon EKS [.noloc]`Kubernetes` clusters with Amazon EKS Hybrid Nodes. +Learn about and configure the VPC and on-premises networking for joining nodes from your data centers to Amazon EKS Kubernetes clusters with Amazon EKS Hybrid Nodes. -- This topic provides an overview of the networking setup you must have configured before creating your Amazon EKS cluster and attaching hybrid nodes. This guide assumes you have met the prerequisite requirements for hybrid network connectivity using link:vpn/latest/s2svpn/SetUpVPNConnections.html[{aws} Site-to-Site VPN,type="documentation"], link:directconnect/latest/UserGuide/Welcome.html[{aws} Direct Connect,type="documentation"], or your own VPN solution. @@ -20,29 +15,57 @@ This topic provides an overview of the networking setup you must have configured image::images/hybrid-prereq-diagram.png[Hybrid node network connectivity.,scaledwidth=50%] -[[hybrid-nodes-networking-on-prem,hybrid-nodes-networking-on-prem.title]] +[#hybrid-nodes-networking-on-prem] == On-premises networking configuration -*Minimum network requirements* +[#hybrid-nodes-networking-min-reqs] +=== Minimum network requirements -For an optimal experience, {aws} recommends reliable network connectivity of at least 100 Mbps and a maximum of 200ms round trip latency for the hybrid nodes connection to the {aws-region}. The bandwidth and latency requirements can vary depending on the number of hybrid nodes and your workload characteristics such as application image size, application elasticity, monitoring and logging configurations, and application dependencies on accessing data stored in other {aws-services}. +For an optimal experience, we recommend that you have reliable network connectivity of at least 100 Mbps and a maximum of 200ms round trip latency for the hybrid nodes connection to the {aws} Region. This is general guidance that accommodates most use cases but is not a strict requirement. The bandwidth and latency requirements can vary depending on the number of hybrid nodes and your workload characteristics, such as application image size, application elasticity, monitoring and logging configurations, and application dependencies on accessing data stored in other {aws} services. We recommend that you test with your own applications and environments before deploying to production to validate that your networking setup meets the requirements for your workloads. -*On-premises node and pod CIDRs* +[#hybrid-nodes-networking-on-prem-cidrs] +=== On-premises node and pod CIDRs -Identify the node and pod CIDRs you will use for your hybrid nodes and the workloads running on them. The node CIDR is allocated from your on-premises network and the pod CIDR is allocated from your Container Network Interface (CNI) if you are using an overlay network for your CNI. You pass your on-premises node CIDRs and optionally pod CIDRs as inputs when you create your Amazon EKS cluster with the `RemoteNodeNetwork` and `RemotePodNetwork` fields. +Identify the node and pod CIDRs you will use for your hybrid nodes and the workloads running on them. The node CIDR is allocated from your on-premises network and the pod CIDR is allocated from your Container Network Interface (CNI) if you are using an overlay network for your CNI. You pass your on-premises node CIDRs and pod CIDRs as inputs when you create your EKS cluster with the `RemoteNodeNetwork` and `RemotePodNetwork` fields. Your on-premises node CIDRs must be routable on your on-premises network. See the following section for information on the on-premises pod CIDR routability. The on-premises node and pod CIDR blocks must meet the following requirements: 1. Be within one of the following `IPv4` RFC-1918 ranges: `10.0.0.0/8`, `172.16.0.0/12`, or `192.168.0.0/16`. -2. Not overlap with each other, the VPC CIDR for your Amazon EKS cluster, or your Kubernetes service `IPv4` CIDR. +2. Not overlap with each other, the VPC CIDR for your EKS cluster, or your Kubernetes service `IPv4` CIDR. + +[#hybrid-nodes-networking-on-prem-pod-routing] +=== On-premises pod network routing + +When using EKS Hybrid Nodes, we generally recommend that you make your on-premises pod CIDRs routable on your on-premises network to enable full cluster communication and functionality between cloud and on-premises environments. + +*Routable pod networks* + +If you are able to make your pod network routable on your on-premises network, follow the guidance below. -If your CNI performs Network Address Translation (NAT) for pod traffic as it leaves your on-premises hosts, you do not need to advertise your pod CIDR to your on-premises network or configure your Amazon EKS cluster with your _remote pod network_ for hybrid nodes to become ready to workloads. If your CNI does not use NAT for pod traffic as it leaves your on-premises hosts, you must advertise your pod CIDR with your on-premises network and you must configure your Amazon EKS cluster with your remote pod network for hybrid nodes to become ready to workloads. If you are running webhooks on your hybrid nodes, you must advertise your pod CIDR to your on-premises network and configure your Amazon EKS cluster with your remote pod network so the Amazon EKS control plane can directly connect to the webhooks running on hybrid nodes. +1. Configure the `RemotePodNetwork` field for your EKS cluster with your on-premises pod CIDR, your VPC route tables with your on-premises pod CIDR, and your EKS cluster security group with your on-premises pod CIDR. +2. There are several techniques you can use to make your on-premises pod CIDR routable on your on-premises network including Border Gateway Protocol (BGP), static routes, or other custom routing solutions. BGP is the recommended solution as it is more scalable and easier to manage than alternative solutions that require custom or manual route configuration. {aws} supports the BGP capabilities of Cilium and Calico for advertising pod CIDRs, see <> and <> for more information. +3. Webhooks can run on hybrid nodes as the EKS control plane is able to communicate with the Pod IP addresses assigned to the webhooks. +4. Workloads running on cloud nodes are able to communicate directly with workloads running on hybrid nodes in the same EKS cluster. +5. Other {aws} services, such as {aws} Application Load Balancers and Amazon Managed Service for Prometheus, are able to communicate with workloads running on hybrid nodes to balance network traffic and scrape pod metrics. -*Access required during hybrid node installation and upgrade* +*Unroutable pod networks* + +If you are _not_ able to make your pod networks routable on your on-premises network, follow the guidance below. + +1. Webhooks cannot run on hybrid nodes because webhooks require connectivity from the EKS control plane to the Pod IP addresses assigned to the webhooks. In this case, we recommend that you run webhooks on cloud nodes in the same EKS cluster as your hybrid nodes, see <> for more information. +2. Workloads running on cloud nodes are not able to communicate directly with workloads running on hybrid nodes when using the VPC CNI for cloud nodes and Cilium or Calico for hybrid nodes. +3. Use Service Traffic Distribution to keep traffic local to the zone it is originating from. For more information on Service Traffic Distribution, see <>. +4. Configure your CNI to use egress masquerade or network address translation (NAT) for pod traffic as it leaves your on-premises hosts. This is enabled by default in Cilium. Calico requires `natOutgoing` to be set to `true`. +5. Other {aws} services, such as {aws} Application Load Balancers and Amazon Managed Service for Prometheus, are not able to communicate with workloads running on hybrid nodes. + +[#hybrid-nodes-networking-access-reqs] +=== Access required during hybrid node installation and upgrade You must have access to the following domains during the installation process where you install the hybrid nodes dependencies on your hosts. This process can be done once when you are building your operating system images or it can be done on each host at runtime. This includes initial installation and when you upgrade the Kubernetes version of your hybrid nodes. -[cols="1,1,1,1", options="header"] +Some packages are installed using the OS's default package manager. For AL2023 and RHEL, the `yum` command is used to install `containerd`, `ca-certificates`, `iptables` and `amazon-ssm-agent`. For Ubuntu, `apt` is used to install `containerd`, `ca-certificates`, and `iptables`, and `snap` is used to install `amazon-ssm-agent`. + +[%header,cols="4"] |=== |Component |URL @@ -59,6 +82,11 @@ You must have access to the following domains during the installation process wh |HTTPS |443 +|link:general/latest/gr/ecr.html[ECR service endpoints,type="documentation"] +|\https://api.ecr.[.replaceable]`region`.amazonaws.com +|HTTPS +|443 + |EKS ECR endpoints |See <> for regional endpoints. |HTTPS @@ -83,16 +111,22 @@ You must have access to the following domains during the installation process wh |\https://rolesanywhere.[.replaceable]`region`.amazonaws.com |HTTPS |443 + +|Operating System package manager endpoints +|Package repository endpoints are OS-specific and might vary by geographic region. +|HTTPS +|443 |=== [NOTE] ==== ^1^ Access to the {aws} SSM endpoints are only required if you are using {aws} SSM hybrid activations for your on-premises IAM credential provider. -^2^ Access to the {aws} IAM endpoints are only required if you are using {aws} IAM Roles Anywhere for your on-premises IAM credential provider. +^2^ Access to the {aws} IAM endpoints are only required if you are using {aws} IAM Roles Anywhere for your on-premises IAM credential provider. ==== -*Access required for ongoing cluster operations* +[#hybrid-nodes-networking-access-reqs-ongoing] +=== Access required for ongoing cluster operations The following network access for your on-premises firewall is required for ongoing cluster operations. @@ -101,7 +135,7 @@ The following network access for your on-premises firewall is required for ongoi Depending on your choice of CNI, you need to configure additional network access rules for the CNI ports. See the link:https://docs.cilium.io/en/stable/operations/system_requirements/#firewall-rules[Cilium documentation] and the link:https://docs.tigera.io/calico/latest/getting-started/kubernetes/requirements#network-requirements[Calico documentation] for details. ==== -[cols="1,1,1,1,1,1,1", options="header"] +[%header,cols="7"] |=== |Type |Protocol @@ -117,15 +151,15 @@ Depending on your choice of CNI, you need to configure additional network access |443 |Remote Node CIDR(s) |EKS cluster IPs ^1^ -|[.noloc]`kubelet` to Kubernetes API server +|kubelet to Kubernetes API server |HTTPS |TCP |Outbound |443 -|Remote [.noloc]`Pod` CIDR(s) +|Remote Pod CIDR(s) |EKS cluster IPs ^1^ -|[.noloc]`Pod` to Kubernetes API server +|Pod to Kubernetes API server |HTTPS |TCP @@ -147,9 +181,9 @@ Depending on your choice of CNI, you need to configure additional network access |TCP |Outbound |443 -|Remote [.noloc]`Pod` CIDR(s) +|Remote Pod CIDR(s) |link:general/latest/gr/sts.html[STS Regional Endpoint,type="documentation"] -|[.noloc]`Pod` to STS endpoint, only required for IRSA +|Pod to STS endpoint, only required for IRSA |HTTPS |TCP @@ -157,7 +191,7 @@ Depending on your choice of CNI, you need to configure additional network access |443 |Remote Node CIDR(s) |link:general/latest/gr/eks.html[Amazon EKS Auth service endpoint,type="documentation"] -|Node to Amazon EKS Auth endpoint, only required for Amazon EKS [.noloc]`Pod` Identity +|Node to Amazon EKS Auth endpoint, only required for Amazon EKS Pod Identity |HTTPS |TCP @@ -165,43 +199,44 @@ Depending on your choice of CNI, you need to configure additional network access |10250 |EKS cluster IPs ^1^ |Remote Node CIDR(s) -|[.noloc]`kubelet` to Kubernetes API server +|Kubernetes API server to kubelet |HTTPS |TCP |Inbound |Webhook ports |EKS cluster IPs ^1^ -|Remote [.noloc]`Pod` CIDR(s) +|Remote Pod CIDR(s) |Kubernetes API server to webhooks |HTTPS |TCP,UDP |Inbound,Outbound |53 -|Remote [.noloc]`Pod` CIDR(s) -|Remote [.noloc]`Pod` CIDR(s) -|[.noloc]`Pod` to CoreDNS. If you run at least 1 replica of CoreDNS in the cloud, you must allow DNS traffic to the VPC where CoreDNS is running. +|Remote Pod CIDR(s) +|Remote Pod CIDR(s) +|Pod to CoreDNS. If you run at least 1 replica of CoreDNS in the cloud, you must allow DNS traffic to the VPC where CoreDNS is running. |User-defined |User-defined |Inbound,Outbound |App ports -|Remote [.noloc]`Pod` CIDR(s) -|Remote [.noloc]`Pod` CIDR(s) -|[.noloc]`Pod` to [.noloc]`Pod` +|Remote Pod CIDR(s) +|Remote Pod CIDR(s) +|Pod to Pod |=== [NOTE] ==== -^1^ The IPs of the Amazon EKS cluster. See the following section on Amazon EKS elastic network interfaces. +^1^ The IPs of the EKS cluster. See the following section on Amazon EKS elastic network interfaces. ==== -*Amazon EKS network interfaces* +[#hybrid-nodes-networking-eks-network-interfaces] +=== Amazon EKS network interfaces -Amazon EKS attaches network interfaces to the subnets in the VPC you pass during cluster creation to enable the communication between the Amazon EKS control plane and your VPC. The network interfaces that Amazon EKS creates can be found after cluster creation in the Amazon EC2 console or with the {cli}. The original network interfaces are deleted and new network interfaces are created when changes are applied on your Amazon EKS cluster, such as Kubernetes version upgrades. You can restrict the IP range for the Amazon EKS network interfaces by using constrained subnet sizes for the subnets you pass during cluster creation, which makes it easier to configure your on-premises firewall to allow inbound/outbound connectivity to this known, constrained set of IPs. To control which subnets network interfaces are created in, you can limit the number of subnets you specify when you create a cluster or you can update the subnets after creating the cluster. +Amazon EKS attaches network interfaces to the subnets in the VPC you pass during cluster creation to enable the communication between the EKS control plane and your VPC. The network interfaces that Amazon EKS creates can be found after cluster creation in the Amazon EC2 console or with the {aws} CLI. The original network interfaces are deleted and new network interfaces are created when changes are applied on your EKS cluster, such as Kubernetes version upgrades. You can restrict the IP range for the Amazon EKS network interfaces by using constrained subnet sizes for the subnets you pass during cluster creation, which makes it easier to configure your on-premises firewall to allow inbound/outbound connectivity to this known, constrained set of IPs. To control which subnets network interfaces are created in, you can limit the number of subnets you specify when you create a cluster or you can update the subnets after creating the cluster. -The network interfaces provisioned by Amazon EKS have a description of the format `Amazon EKS [.replaceable]``your-cluster-name```. See the example below for an {cli} command you can use to find the IP addresses of the network interfaces that Amazon EKS provisions. Replace `VPC_ID` with the ID of the VPC you pass during cluster creation. +The network interfaces provisioned by Amazon EKS have a description of the format `Amazon EKS [.replaceable]``your-cluster-name```. See the example below for an {aws} CLI command you can use to find the IP addresses of the network interfaces that Amazon EKS provisions. Replace `VPC_ID` with the ID of the VPC you pass during cluster creation. [source,cli,subs="verbatim,attributes,quotes"] ---- @@ -210,12 +245,12 @@ aws ec2 describe-network-interfaces \ ---- -[[hybrid-nodes-networking-vpc,hybrid-nodes-networking-vpc.title]] +[#hybrid-nodes-networking-vpc] == {aws} VPC and subnet setup The existing <> for Amazon EKS apply to clusters with hybrid nodes. Additionally, your VPC CIDR can't overlap with your on-premises node and pod CIDRs. You must configure routes in your VPC routing table for your on-premises node and optionally pod CIDRs. These routes must be setup to route traffic to the gateway you are using for your hybrid network connectivity, which is commonly a virtual private gateway (VGW) or transit gateway (TGW). If you are using TGW or VGW to connect your VPC with your on-premises environment, you must create a TGW or VGW attachment for your VPC. Your VPC must have DNS hostname and DNS resolution support. -The following steps use the {cli}. You can also create these resources in the {aws-management-console} or with other interfaces such as {aws} CloudFormation, {aws} CDK, or Terraform. +The following steps use the {aws} CLI. You can also create these resources in the {aws-management-console} or with other interfaces such as {aws} CloudFormation, {aws} CDK, or Terraform. === Step 1: Create VPC @@ -237,7 +272,7 @@ aws ec2 modify-vpc-attribute --vpc-id [.replaceable]`VPC_ID` --enable-dns-hostna Create at least 2 subnets. Amazon EKS uses these subnets for the cluster network interfaces. For more information, see the <>. -. You can find the availability zones for an {aws-region} with the following command. Replace `us-west-2` with your region. +. You can find the availability zones for an {aws} Region with the following command. Replace `us-west-2` with your region. + [source,cli,subs="verbatim,attributes,quotes"] ---- @@ -254,9 +289,9 @@ aws ec2 create-subnet \ --availability-zone [.replaceable]`AZ` ---- -=== (Optional) Step 3: Attach VPC with {amazon-vpc} Transit Gateway (TGW) or {aws-direct-connect} virtual private gateway (VGW) +=== (Optional) Step 3: Attach VPC with Amazon VPC Transit Gateway (TGW) or {aws} Direct Connect virtual private gateway (VGW) -If you are using a TGW or VGW, attach your VPC to the TGW or VGW. For more information, see link:vpc/latest/tgw/tgw-vpc-attachments.html[{amazon-vpc} attachments in {amazon-vpc} Transit Gateways,type="documentation"] or link:vpn/latest/s2svpn/how_it_works.html#VPNGateway[{aws} Direct Connect virtual private gateway associations,type="documentation"]. +If you are using a TGW or VGW, attach your VPC to the TGW or VGW. For more information, see link:vpc/latest/tgw/tgw-vpc-attachments.html[Amazon VPC attachments in Amazon VPC Transit Gateways,type="documentation"] or link:vpn/latest/s2svpn/how_it_works.html#VPNGateway[{aws} Direct Connect virtual private gateway associations,type="documentation"]. *Transit Gateway* @@ -317,7 +352,7 @@ aws ec2 create-route \ ---- -*Remote [.noloc]`Pod` network* +*Remote Pod network* [source,cli,subs="verbatim,attributes,quotes"] ---- @@ -340,12 +375,12 @@ aws ec2 associate-route-table --route-table-id [.replaceable]`RT_ID` --subnet-id ---- -[[hybrid-nodes-networking-cluster-sg,hybrid-nodes-networking-cluster-sg.title]] +[#hybrid-nodes-networking-cluster-sg] == Cluster security group configuration -The following access for your Amazon EKS cluster security group is required for ongoing cluster operations. +The following access for your EKS cluster security group is required for ongoing cluster operations. Amazon EKS automatically creates the required *inbound* security group rules for hybrid nodes when you create or update your cluster with remote node and pod networks configured. Because security groups allow all *outbound* traffic by default, Amazon EKS doesn’t automatically modify the *outbound* rules of the cluster security group for hybrid nodes. If you want to customize the cluster security group, you can limit traffic to the rules in the following table. -[cols="1,1,1,1,1,1,1", options="header"] +[%header,cols="7"] |=== |Type |Protocol @@ -388,12 +423,23 @@ The following access for your Amazon EKS cluster security group is required for |Kubernetes API server to webhook (if running webhooks on hybrid nodes) |=== -To create a security group with the inbound access rules, run the following commands. This security group must be passed when you create your Amazon EKS cluster. By default, the command below creates a security group that allows all outbound access. You can restrict outbound access to include only the rules above. If you're considering limiting the outbound rules, we recommend that you thoroughly test all of your applications and pod connectivity before you apply your changed rules to a production cluster. +[IMPORTANT] +==== +**Security group rule limits**: Amazon EC2 security groups have a maximum of 60 inbound rules by default. The security group inbound rules may not apply if your cluster security group approaches this limit. In this case, it may be required to manually add in the missing inbound rules. + +**CIDR cleanup responsibility**: If you remove remote node or pod networks from EKS clusters, EKS does not automatically remove the corresponding security group rules. You are responsible for manually removing unused remote node or pod networks from your security group rules. +==== + +For more information about the cluster security group that Amazon EKS creates, see <>. + +=== (Optional) Manual security group configuration + +If you need to create additional security groups or modify the automatically created rules, you can use the following commands as reference. By default, the command below creates a security group that allows all outbound access. You can restrict outbound access to include only the rules above. If you're considering limiting the outbound rules, we recommend that you thoroughly test all of your applications and pod connectivity before you apply your changed rules to a production cluster. -* In the first command, replace SG_NAME with a name for your security group -* In the first command, replace VPC_ID with the ID of the VPC you created in the previous step -* In the second command, replace SG_ID with the ID of the security group you create in the first command -* In the second command, replace REMOTE_NODE_CIDR and REMOTE_POD_CIDR with the values for your hybrid nodes and on-premises network. +* In the first command, replace `SG_NAME` with a name for your security group +* In the first command, replace `VPC_ID` with the ID of the VPC you created in the previous step +* In the second command, replace `SG_ID` with the ID of the security group you create in the first command +* In the second command, replace `REMOTE_NODE_CIDR` and `REMOTE_POD_CIDR` with the values for your hybrid nodes and on-premises network. [source,cli,subs="verbatim,attributes,quotes"] ---- diff --git a/latest/ug/nodes/hybrid-nodes-nodeadm.adoc b/latest/ug/nodes/hybrid-nodes-nodeadm.adoc index 1e07b179a..0cdf264ad 100644 --- a/latest/ug/nodes/hybrid-nodes-nodeadm.adoc +++ b/latest/ug/nodes/hybrid-nodes-nodeadm.adoc @@ -1,67 +1,68 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[hybrid-nodes-nodeadm,hybrid-nodes-nodeadm.title]] +[#hybrid-nodes-nodeadm] = Hybrid nodes `nodeadm` reference -:info_doctype: section -:info_title: Hybrid nodes nodeadm reference -:info_titleabbrev: Hybrid nodes nodeadm reference -:keywords: on-premises, hybrid -:info_abstract: Hybrid nodes nodeadm reference - -include::../attributes.txt[] +:info_titleabbrev: Hybrid nodes nodeadm [abstract] -- Hybrid nodes nodeadm reference -- -The Amazon EKS Hybrid Nodes CLI (`nodeadm`) used for hybrid nodes lifecycle management differs from the `nodeadm` version used for bootstrapping Amazon EC2 instances as nodes in Amazon EKS clusters. Follow the documentation and references for the appropriate `nodeadm` version. This documentation page is for the hybrid nodes `nodeadm` version and the hybrid nodes `nodeadm` version is available in the https://github.com/aws/eks-hybrid[eks-hybrid repository on GitHub]. See the https://awslabs.github.io/amazon-eks-ami/nodeadm/[nodeadm - Amazon EKS AMI documentation] for the `nodeadm` version used for Amazon EC2 instances. +The Amazon EKS Hybrid Nodes CLI (`nodeadm`) simplifies the installation, configuration, registration, and uninstallation of the hybrid nodes components. You can include `nodeadm` in your operating system images to automate hybrid node bootstrap, see <> for more information. + +The `nodeadm` version for hybrid nodes differs from the `nodeadm` version used for bootstrapping Amazon EC2 instances as nodes in Amazon EKS clusters. Follow the documentation and references for the appropriate `nodeadm` version. This documentation page is for the hybrid nodes `nodeadm` version. + +The source code for the hybrid nodes `nodeadm` is published in the https://github.com/aws/eks-hybrid +GitHub repository. + +[IMPORTANT] +==== +You must run `nodeadm` with a user that has root/sudo privileges. +==== == Download `nodeadm` The hybrid nodes version of `nodeadm` is hosted in Amazon S3 fronted by Amazon CloudFront. To install `nodeadm` on each on-premises host, you can run the following command from your on-premises hosts. -*For x86_64 hosts:* -[source,yaml,subs="verbatim,attributes,quotes"] +*For x86_64 hosts* +[source,bash,subs="verbatim,attributes"] ---- curl -OL 'https://hybrid-assets.eks.amazonaws.com/releases/latest/bin/linux/amd64/nodeadm' ---- *For ARM hosts* -[source,yaml,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- curl -OL 'https://hybrid-assets.eks.amazonaws.com/releases/latest/bin/linux/arm64/nodeadm' ---- Add executable file permission to the downloaded binary on each host. -[source,yaml,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- chmod +x nodeadm ---- -== Commands +== `nodeadm install` -[IMPORTANT] -==== -You must run `nodeadm` with a user that has root/sudo privileges. -==== - -=== Install +The `nodeadm install` command is used to install the artifacts and dependencies required to run and join hybrid nodes to an Amazon EKS cluster. The `nodeadm install` command can be run individually on each hybrid node or can be run during image build pipelines to preinstall the hybrid nodes dependencies in operating system images. -The `install` command is used to install the artifacts and dependencies required to run and join hybrid nodes to an Amazon EKS cluster. The `install` command can be run individually on each hybrid node or can be run during image build pipelines to preinstall the hybrid nodes dependencies in operating system images. +*Usage* -==== Usage - -`nodeadm install [KUBERNETES_VERSION] [flags]` +[source,bash,subs="verbatim,attributes"] +---- +nodeadm install [KUBERNETES_VERSION] [flags] +---- -==== Positional Arguments +*Positional Arguments* -(Required) `KUBERNETES_VERSION` The major.minor version of EKS Kubernetes to install, for example `1.31` +(Required) `KUBERNETES_VERSION` The major.minor version of EKS Kubernetes to install, for example `1.32` -==== Flags +*Flags* -[cols="1,1,1", options="header"] +[%header,cols="3"] |=== |Name |Required @@ -81,11 +82,18 @@ The `install` command is used to install the artifacts and dependencies required *Values* -`distro` - This is the default value. `nodeadm` will install `containerd` package distributed by the node OS. `distro` is not a supported value for Red Hat Enterprise Linux (RHEL) operating systems. +`distro` - This is the default value. `nodeadm` will install the latest `containerd` package distributed by the node OS that is compatible with the EKS Kubernetes version. `distro` is not a supported value for Red Hat Enterprise Linux (RHEL) operating systems. -`docker` - `nodeadm` will install `containerd` package built and distributed by Docker. `docker` is not a supported value for Amazon Linux 2023 +`docker` - `nodeadm` will install the latest `containerd` package built and distributed by Docker that is compatible with the EKS Kubernetes version. `docker` is not a supported value for Amazon Linux 2023. `none` - `nodeadm` will not install `containerd` package. You must manually install `containerd` before running `nodeadm init`. + +|`-r`, + +`--region` +|FALSE +|Specifies the {aws} Region for downloading artifacts such as the SSM Agent. Defaults to `us-west-2`. + |`-t`, `--timeout` @@ -97,92 +105,43 @@ The `install` command is used to install the artifacts and dependencies required |Displays help message with available flag, subcommand and positional value parameters. |=== -==== Examples +*Examples* -Install Kubernetes version `1.31` with {aws} Systems Manager (SSM) as the credential provider +Install Kubernetes version `1.32` with {aws} Systems Manager (SSM) as the credential provider -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- -nodeadm install 1.31 --credential-provider ssm +nodeadm install 1.32 --credential-provider ssm ---- -Install Kubernetes version `1.31` with {aws} Systems Manager (SSM) as the credential provider, Docker as the containerd source, with a download timeout of 20 minutes. +Install Kubernetes version `1.32` with {aws} Systems Manager (SSM) as the credential provider, Docker as the containerd source, with a download timeout of 20 minutes. -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- -nodeadm install 1.31 --credential-provider ssm --containerd-source docker --timeout 20m +nodeadm install 1.32 --credential-provider ssm --containerd-source docker --timeout 20m ---- -Install Kubernetes version `1.31` with {aws} IAM Roles Anywhere as the credential provider +Install Kubernetes version `1.32` with {aws} IAM Roles Anywhere as the credential provider -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- -nodeadm install 1.31 --credential-provider iam-ra +nodeadm install 1.32 --credential-provider iam-ra ---- -==== Files installed - -[cols="1,1", options="header"] -|=== - -|Artifact -|Path - -|IAM Roles Anywhere CLI -|/usr/local/bin/aws_signing_helper - -|Kubelet binary -|/usr/bin/kubelet - -|Kubectl binary -|usr/local/bin/kubectl - -|ECR Credentials Provider -|/etc/eks/image-credential-provider/ecr-credential-provider - -|{aws} IAM Authenticator -|/usr/local/bin/aws-iam-authenticator - -|SSM Setup CLI -|/opt/ssm/ssm-setup-cli - -|SSM Agent -|On Ubuntu - /snap/amazon-ssm-agent/current/amazon-ssm-agent - -On RHEL & AL2023 - /usr/bin/amazon-ssm-agent - -|Containerd -|On Ubuntu & AL2023 - /usr/bin/containerd - -On RHEL - /bin/containerd - -|Iptables -|On Ubuntu & AL2023 - /usr/sbin/iptables - -On RHEL - /sbin/iptables - -|CNI plugins -|/opt/cni/bin +== `nodeadm config check` -|installed artifacts tracker -|/opt/nodeadm/tracker - -|=== +The `nodeadm config check` command checks the provided node configuration for errors. This command can be used to verify and validate the correctness of a hybrid node configuration file. -=== Config check - -The `config check` command checks the provided node configuration for errors. This command can be used to verify and validate the correctness of a hybrid node configuration file. - -==== Usage +*Usage* [source,bash,subs="verbatim,attributes"] ---- nodeadm config check [flags] ---- -==== Flags +*Flags* -[cols="1,1,1", options="header"] +[%header,cols="3"] |=== |Name @@ -201,27 +160,27 @@ nodeadm config check [flags] |Displays help message with available flag, subcommand and positional value parameters. |=== -==== Examples +*Examples* -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- -nodeadm config check --config-source file:///root/nodeConfig.yaml +nodeadm config check -c file://nodeConfig.yaml ---- -=== Init +== `nodeadm init` -The `init` command starts and connects the hybrid node with the configured Amazon EKS cluster. +The `nodeadm init` command starts and connects the hybrid node with the configured Amazon EKS cluster. See <> or <> for details of how to configure the `nodeConfig.yaml` file. -==== Usage +*Usage* -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- nodeadm init [flags] ---- -==== Flags +*Flags* -[cols="1,1,1", options="header"] +[%header,cols="3"] |=== |Name @@ -242,96 +201,42 @@ nodeadm init [flags] *Values* -`install-validation` skips checking if the proceding install command ran successfully. +`install-validation` skips checking if the preceding install command ran successfully. -|`-h, `--help` +`cni-validation` skips checking if either Cilium or Calico CNI's VXLAN ports are opened if firewall is enabled on the node + +`node-ip-validation` skips checking if the node IP falls within a CIDR in the remote node networks + +|`-h`, `--help` |FALSE |Displays help message with available flag, subcommand and positional value parameters. |=== +*Examples* -==== Examples - -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- -nodeadm init --config-source file://root/nodeConfig.yaml +nodeadm init -c file://nodeConfig.yaml ---- -==== Files installed - -[cols="1,1", options="header"] -|=== - -|Name -|Path - -|Kubelet kubeconfig -|/var/lib/kubelet/kubeconfig - -|Kubelet config -|/etc/kubernetes/kubelet/config.json - -|Kubelet systemd unit -|/etc/systemd/system/kubelet.service - -|Image credentials provider config -|/etc/eks/image-credential-provider/config.json - -|Kubelet env file -|/etc/eks/kubelet/environment - -|Kubelet Certs -|/etc/kubernetes/pki/ca.crt - -|Containerd config -|/etc/containerd/config.toml - -|Containerd kernel modules config -|/etc/modules-load.d/contianerd.conf - -|{aws} config file -|/etc/aws/hybrid/config - -|{aws} credentials file (if enable credentials file) -|/eks-hybrid/.aws/credentials - -|{aws} signing helper system unit -|/etc/systemd/system/aws_signing_helper_update.service - -|Sysctl conf file -|/etc/sysctl.d/99-nodeadm.conf - -|Apt manager files for docker repo (if containerd source is docker) -| - -|Ca-certificates -|/etc/ssl/certs/ca-certificates.crt - -|Gpg key file -|/etc/apt/keyrings/docker.asc - -|Docker repo source file -|/etc/apt/sources.list.d/docker.list -|=== - -=== Upgrade +== `nodeadm upgrade` The `nodeadm upgrade` command upgrades all the installed artifacts to the latest version and bootstraps the node to configure the upgraded artifacts and join the EKS cluster on {aws}. Upgrade is a disruptive command to the workloads running on the node. Please move your workloads to another node before running upgrade. -==== Usage +*Usage* -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- nodeadm upgrade [KUBERNETES_VERSION] [flags] ---- -==== Positional Arguments +*Positional Arguments* -(Required) `KUBERNETES_VERSION` The major.minor version of EKS Kubernetes to install, for example `1.31` +(Required) `KUBERNETES_VERSION` The major.minor version of EKS Kubernetes to install, for example `1.32` -==== Flags +*Flags* -[cols="1,1,1", options="header"] +[%header,cols="3"] |=== |Name @@ -364,37 +269,39 @@ nodeadm upgrade [KUBERNETES_VERSION] [flags] `init-validation` skips checking if the node has been initialized successfully before running upgrade. +`containerd-major-version-upgrade` prevents containerd major version upgrades during node upgrade. + |`-h`, `--help` |FALSE |Displays help message with available flag, subcommand and positional value parameters. |=== -==== Examples +*Examples* -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- -nodeadm upgrade 1.31 --config-source file:///root/nodeConfig.yaml +nodeadm upgrade 1.32 -c file://nodeConfig.yaml ---- -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- -nodeadm upgrade 1.31 --config-source file:///root/nodeConfig.yaml --timeout 20m +nodeadm upgrade 1.32 -c file://nodeConfig.yaml --timeout 20m ---- -=== Uninstall +== `nodeadm uninstall` The `nodeadm uninstall` command stops and removes the artifacts `nodeadm` installs during `nodeadm install`, including the kubelet and containerd. Note, the uninstall command does not drain or delete your hybrid nodes from your cluster. You must run the drain and delete operations separately, see <> for more information. By default, `nodeadm uninstall` will not proceed if there are pods remaining on the node. Similarly, `nodeadm uninstall` does not remove CNI dependencies or dependencies of other Kubernetes add-ons you run on your cluster. To fully remove the CNI installation from your host, see the instructions at <>. If you are using {aws} SSM hybrid activations as your on-premises credentials provider, the `nodeadm uninstall` command deregisters your hosts as {aws} SSM managed instances. -==== Usage +*Usage* -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- nodeadm uninstall [flags] ---- -==== Flags +*Flags* -[cols="1,1,1", options="header"] +[%header,cols="3"] |=== |Name @@ -405,7 +312,7 @@ nodeadm uninstall [flags] `--skip` |FALSE -|Phases of upgrade to be skipped. It is not recommended to skip any of the phase unless it helps to fix an issue. +|Phases of uninstall to be skipped. It is not recommended to skip any of the phases unless it helps to fix an issue. *Values* @@ -413,28 +320,45 @@ nodeadm uninstall [flags] `node-validation` skips checking if the node has been cordoned. -`init-validation` skips checking if the node has been initialized successfully before running upgrade. +`init-validation` skips checking if the node has been initialized successfully before running uninstall. |`-h`, `--help` |FALSE |Displays help message with available flag, subcommand and positional value parameters. + +|`-f`, + +`--force` +|FALSE +|Force delete additional directories that might contain remaining files from Kubernetes and CNI components. + +*WARNING* + +This will delete all contents in default Kubernetes and CNI directories (`/var/lib/cni`, `/etc/cni/net.d`, etc). Do not use this flag if you store your own data in these locations. + +Starting from nodeadm `v1.0.9`, the `./nodeadm uninstall --skip node-validation,pod-validation --force` command no longer deletes the `/var/lib/kubelet` directory. This is because it may contain Pod volumes and volume-subpath directories that sometimes include the mounted node filesystem. + +*Safe handling tips* + +- Deleting mounted paths can lead to accidental deletion of the actual mounted node filesystem. Before manually deleting the `/var/lib/kubelet` directory, carefully inspect all active mounts and unmount volumes safely to avoid data loss. + |=== -==== Examples +*Examples* -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- nodeadm uninstall ---- -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- nodeadm uninstall --skip node-validation,pod-validation ---- -=== Debug +== `nodeadm debug` The `nodeadm debug` command can be used to troubleshoot unhealthy or misconfigured hybrid nodes. It validates the following requirements are in-place. @@ -445,15 +369,16 @@ The `nodeadm debug` command can be used to troubleshoot unhealthy or misconfigur If errors are found, the command's output suggests troubleshooting steps. Certain validation steps show child processes. If these fail, the output is showed in a stderr section under the validation error. -==== Usage -[source,yaml,subs="verbatim,attributes,quotes"] +*Usage* + +[source,yaml,subs="verbatim,attributes"] ---- nodeadm debug [flags] ---- -==== Flags +*Flags* -[cols="1,1,1", options="header"] +[%header,cols="3"] |=== |Name @@ -464,24 +389,136 @@ nodeadm debug [flags] |TRUE |Source of `nodeadm` configuration. For hybrid nodes the input should follow a URI with file scheme. +|`--no-color` +|FALSE +|Disables color output. Useful for automation. + |`-h`, `--help` |FALSE |Displays help message with available flag, subcommand and positional value parameters. |=== -==== Examples -[source,yaml,subs="verbatim,attributes,quotes"] +*Examples* +[source,yaml,subs="verbatim,attributes"] ---- -nodeadm debug --config-source file://nodeConfig.yaml +nodeadm debug -c file://nodeConfig.yaml ---- -== Node Config API Reference +== Nodeadm file locations + +=== nodeadm install + +When running `nodeadm install`, the following files and file locations are configured. + +[%header,cols="2"] +|=== + +|Artifact +|Path + +|IAM Roles Anywhere CLI +|/usr/local/bin/aws_signing_helper + +|Kubelet binary +|/usr/bin/kubelet + +|Kubectl binary +|usr/local/bin/kubectl -*{aws} SSM hybrid activations* +|ECR Credentials Provider +|/etc/eks/image-credential-provider/ecr-credential-provider + +|{aws} IAM Authenticator +|/usr/local/bin/aws-iam-authenticator + +|SSM Setup CLI +|/opt/ssm/ssm-setup-cli + +|SSM Agent +|On Ubuntu - /snap/amazon-ssm-agent/current/amazon-ssm-agent + +On RHEL & AL2023 - /usr/bin/amazon-ssm-agent + +|Containerd +|On Ubuntu & AL2023 - /usr/bin/containerd + +On RHEL - /bin/containerd + +|Iptables +|On Ubuntu & AL2023 - /usr/sbin/iptables + +On RHEL - /sbin/iptables + +|CNI plugins +|/opt/cni/bin + +|installed artifacts tracker +|/opt/nodeadm/tracker + +|=== + +=== nodeadm init + +When running `nodeadm init`, the following files and file locations are configured. + +[%header,cols="2"] +|=== + +|Name +|Path + +|Kubelet kubeconfig +|/var/lib/kubelet/kubeconfig + +|Kubelet config +|/etc/kubernetes/kubelet/config.json + +|Kubelet systemd unit +|/etc/systemd/system/kubelet.service + +|Image credentials provider config +|/etc/eks/image-credential-provider/config.json + +|Kubelet env file +|/etc/eks/kubelet/environment + +|Kubelet Certs +|/etc/kubernetes/pki/ca.crt + +|Containerd config +|/etc/containerd/config.toml + +|Containerd kernel modules config +|/etc/modules-load.d/contianerd.conf + +|{aws} config file +|/etc/aws/hybrid/config + +|{aws} credentials file (if enable credentials file) +|/eks-hybrid/.aws/credentials + +|{aws} signing helper system unit +|/etc/systemd/system/aws_signing_helper_update.service + +|Sysctl conf file +|/etc/sysctl.d/99-nodeadm.conf + +|Ca-certificates +|/etc/ssl/certs/ca-certificates.crt + +|Gpg key file +|/etc/apt/keyrings/docker.asc + +|Docker repo source file +|/etc/apt/sources.list.d/docker.list +|=== + +[#hybrid-nodes-node-config-ssm] +== Node Config for SSM hybrid activations The following is a sample `nodeConfig.yaml` when using {aws} SSM hybrid activations for hybrid nodes credentials. -[source,yaml,subs="verbatim,attributes,quotes"] +[source,yaml,subs="verbatim,attributes"] ---- apiVersion: node.eks.aws/v1alpha1 kind: NodeConfig @@ -495,13 +532,14 @@ spec: activationId: # SSM hybrid activation id ---- -*{aws} IAM Roles Anywhere* +[#hybrid-nodes-node-config-iamra] +== Node Config for IAM Roles Anywhere The following is a sample `nodeConfig.yaml` for {aws} IAM Roles Anywhere for hybrid nodes credentials. When using {aws} IAM Roles Anywhere as your on-premises credentials provider, the `nodeName` you use in your `nodeadm` configuration must align with the permissions you scoped for your Hybrid Nodes IAM role. For example, if your permissions for the Hybrid Nodes IAM role only allow {aws} IAM Roles Anywhere to assume the role when the role session name is equal to the CN of the host certificate, then the `nodeName` in your `nodeadm` configuration must be the same as the CN of your certificates. The `nodeName` that you use can't be longer than 64 characters. For more information, see <>. -[source,yaml,subs="verbatim,attributes,quotes"] +[source,yaml,subs="verbatim,attributes"] ---- apiVersion: node.eks.aws/v1alpha1 kind: NodeConfig @@ -519,11 +557,12 @@ spec: privateKeyPath: # Path to the private key file for the certificate ---- -=== (Optional) Kubelet configuration +[#hybrid-nodes-nodeadm-kubelet] +== Node Config for customizing kubelet (Optional) You can pass kubelet configuration and flags in your `nodeadm` configuration. See the example below for how to add an additional node label `abc.amazonaws.com/test-label` and config for setting `shutdownGracePeriod` to 30 seconds. -[source,yaml,subs="verbatim,attributes,quotes"] +[source,yaml,subs="verbatim,attributes"] ---- apiVersion: node.eks.aws/v1alpha1 kind: NodeConfig @@ -542,11 +581,11 @@ spec: activationId: # SSM hybrid activation id ---- -=== (Optional) Containerd configuration +== Node Config for customizing containerd (Optional) -You can pass custom containerd configuration in your `nodeadm` configuration. The containerd configuration for `nodeadm` accepts in-line TOML. See the example below for how to configure containerd to disable deletion of unpacked image layers in the containerd content store. +You can pass custom containerd configuration in your `nodeadm` configuration. The containerd configuration for `nodeadm` accepts in-line TOML. See the example below for how to configure containerd to disable deletion of unpacked image layers in the containerd content store. -[source,yaml,subs="verbatim,attributes,quotes"] +[source,yaml,subs="verbatim,attributes"] ---- apiVersion: node.eks.aws/v1alpha1 kind: NodeConfig @@ -564,6 +603,11 @@ spec: activationId: # SSM hybrid activation id ---- +[NOTE] +==== +Containerd versions 1.x and 2.x use different configuration formats. Containerd 1.x uses config version 2, while containerd 2.x uses config version 3. Although containerd 2.x remains backward compatible with config version 2, config version 3 is recommended for optimal performance. Check your containerd version with `containerd --version` or review `nodeadm` install logs. For more details on config versioning, see https://containerd.io/releases/ +==== + You can also use the containerd configuration to enable SELinux support. With SELinux enabled on containerd, ensure pods scheduled on the node have the proper securityContext and seLinuxOptions enabled. More information on configuring a security context can be found on the https://kubernetes.io/docs/tasks/configure-pod-container/security-context/[Kubernetes documentation]. @@ -572,7 +616,7 @@ You can also use the containerd configuration to enable SELinux support. With SE Red Hat Enterprise Linux (RHEL) 8 and RHEL 9 have SELinux enabled by default and set to strict on the host. Amazon Linux 2023 has SELinux enabled by default and set to permissive mode. When SELinux is set to permissive mode on the host, enabling it on containerd will not block requests but will log it according to the SELinux configuration on the host. ==== -[source,yaml,subs="verbatim,attributes,quotes"] +[source,yaml,subs="verbatim,attributes"] ---- apiVersion: node.eks.aws/v1alpha1 kind: NodeConfig diff --git a/latest/ug/nodes/hybrid-nodes-os.adoc b/latest/ug/nodes/hybrid-nodes-os.adoc index 60be83bd0..a054d5678 100644 --- a/latest/ug/nodes/hybrid-nodes-os.adoc +++ b/latest/ug/nodes/hybrid-nodes-os.adoc @@ -1,28 +1,23 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[hybrid-nodes-os,hybrid-nodes-os.title]] +[#hybrid-nodes-os] = Prepare operating system for hybrid nodes -:info_doctype: section -:info_title: Prepare operating system for hybrid nodes :info_titleabbrev: Prepare operating system -:keywords: on-premises, hybrid -:info_abstract: Prepare operating system for hybrid nodes - -include::../attributes.txt[] [abstract] -- Prepare operating system for use with Hybrid Nodes -- -Amazon Linux 2023 (AL2023), Ubuntu, and Red Hat Enterprise Linux (RHEL) are validated on an ongoing basis for use as the node operating system for hybrid nodes. {aws} supports the hybrid nodes integration with these operating systems but does not provide support for the operating systems itself. AL2023 is not covered by {aws} Support Plans when run outside of Amazon EC2. AL2023 can only be used in on-premises virtualized environments, reference the link:linux/al2023/ug/outside-ec2.html[Amazon Linux 2023 User Guide,type="documentation"] for more information. +Bottlerocket, Amazon Linux 2023 (AL2023), Ubuntu, and RHEL are validated on an ongoing basis for use as the node operating system for hybrid nodes. Bottlerocket is supported by {aws}in VMware vSphere environments only. AL2023 is not covered by {aws} Support Plans when run outside of Amazon EC2. AL2023 can only be used in on-premises virtualized environments, see the link:linux/al2023/ug/outside-ec2.html[Amazon Linux 2023 User Guide,type="documentation"] for more information. {aws} supports the hybrid nodes integration with Ubuntu and RHEL operating systems but does not provide support for the operating system itself. -You are responsible for operating system provisioning and management. When you are testing hybrid nodes for the first time, it is easiest to run the Amazon EKS Hybrid Nodes CLI (`nodeadm`) on an already provisioned host. For production deployments, it is recommended to include `nodeadm` in your operating system images with it configured to run as a systemd service to automatically join hosts to Amazon EKS clusters at host startup. +You are responsible for operating system provisioning and management. When you are testing hybrid nodes for the first time, it is easiest to run the Amazon EKS Hybrid Nodes CLI (`nodeadm`) on an already provisioned host. For production deployments, we recommend that you include `nodeadm` in your operating system images with it configured to run as a systemd service to automatically join hosts to Amazon EKS clusters at host startup. If you are using Bottlerocket as your node operating system on vSphere, you do not need to use `nodeadm` as Bottlerocket already contains the dependencies required for hybrid nodes and will automatically connect to the cluster you configure upon host startup. == Version compatibility The table below represents the operating system versions that are compatible and validated to use as the node operating system for hybrid nodes. If you are using other operating system variants or versions that are not included in this table, then the compatibility of hybrid nodes with your operating system variant or version is not covered by {aws} Support. Hybrid nodes are agnostic to the underlying infrastructure and support x86 and ARM architectures. -[cols="1,1", options="header"] +[%header,cols="2"] |=== |Operating System @@ -31,6 +26,9 @@ The table below represents the operating system versions that are compatible and |Amazon Linux |Amazon Linux 2023 (AL2023) +|Bottlerocket +|v1.37.0 and above VMware variants running Kubernetes v1.28 and above + |Ubuntu |Ubuntu 20.04, Ubuntu 22.04, Ubuntu 24.04 @@ -46,22 +44,32 @@ The table below represents the operating system versions that are compatible and * The Amazon EKS Hybrid Nodes CLI (`nodeadm`) can be used to simplify the installation and configuration of the hybrid nodes components and dependencies. You can run the `nodeadm install` process during your operating system image build pipelines or at runtime on each on-premises host. For more information on the components that `nodeadm` installs, see the <>. * If you are using a proxy in your on-premises environment to reach the internet, there is additional operating system configuration required for the install and upgrade processes to configure your package manager to use the proxy. See <> for instructions. +=== Bottlerocket + +* The steps and tools to connect a Bottlerocket node are different than the steps for other operating systems and are covered separately in <>, instead of the steps in <>. +* The steps for Bottlerocket don't use the hybrid nodes CLI tool, `nodeadm`. +* Only VMware variants of Bottlerocket version v1.37.0 and above are supported with EKS Hybrid Nodes. VMware variants of Bottlerocket are available for Kubernetes versions v1.28 and above. https://bottlerocket.dev/en/os/1.36.x/concepts/variants[Other Bottlerocket variants] are not supported as the hybrid nodes operating system. NOTE: VMware variants of Bottlerocket are only available for the x86_64 architecture. + === Containerd * Containerd is the standard Kubernetes container runtime and is a dependency for hybrid nodes, as well as all Amazon EKS node compute types. The Amazon EKS Hybrid Nodes CLI (`nodeadm`) attempts to install containerd during the `nodeadm install` process. You can configure the containerd installation at `nodeadm install` runtime with the `--containerd-source` command line option. Valid options are `none`, `distro`, and `docker`. If you are using RHEL, `distro` is not a valid option and you can either configure `nodeadm` to install the containerd build from Docker's repos or you can manually install containerd. When using AL2023 or Ubuntu, `nodeadm` defaults to installing containerd from the operating system distribution. If you do not want nodeadm to install containerd, use the `--containerd-source none` option. === Ubuntu -* If you are using Ubuntu 20.04, you must use {aws} Systems Manager hybrid activations as your credential provider. {aws} IAM Roles Anywhere is not supported on Ubuntu 20.04. * If you are using Ubuntu 24.04, you may need to update your version of containerd or change your AppArmor configuration to adopt a fix that allows pods to properly terminate, see https://bugs.launchpad.net/ubuntu/\+source/containerd-app/\+bug/2065423[Ubuntu #2065423]. A reboot is required to apply changes to the AppArmor profile. The latest version of Ubuntu 24.04 has an updated containerd version in its package manager with the fix (containerd version 1.7.19+). -=== RHEL +=== ARM -* If you are using RHEL 8, you must use {aws} Systems Manager hybrid activations as your credential provider. {aws} IAM Roles Anywhere isn't supported on RHEL 8. +* If you are using ARM hardware, an ARMv8.2 compliant processor with the Cryptography Extension (ARMv8.2+crypto) is required to run version 1.31 and above of the EKS kube-proxy add-on. All Raspberry Pi systems prior to the Raspberry Pi 5, as well as Cortex-A72 based processors, do not meet this requirement. As a workaround, you can continue to use version 1.30 of the EKS kube-proxy add-on until it reaches end of extended support in July of 2026, see link:eks/latest/userguide/kubernetes-versions.html[Kubernetes release calendar,type="documentation"], or use a custom kube-proxy image from upstream. +* The following error message in the kube-proxy log indicates this incompatibility: +[source,none] +---- +Fatal glibc error: This version of Amazon Linux requires a newer ARM64 processor compliant with at least ARM architecture 8.2-a with Cryptographic extensions. On EC2 this is Graviton 2 or later. +---- == Building operating system images -Amazon EKS provides https://github.com/aws/eks-hybrid/tree/main/example/packer[example Packer templates] you can use to create operating system images that include `nodeadm` and configure it to run at host-startup. This process is recommended to avoid pulling the hybrid nodes dependencies individually on each host and to automate the hybrid nodes bootstrap process. You can use the example Packer templates with an Ubuntu 22.04, Ubuntu 24.04, RHEL 8 or RHEL 9 ISO image and can output images with these formats: OVA, [.noloc]`Qcow2`, or raw. +Amazon EKS provides https://github.com/aws/eks-hybrid/tree/main/example/packer[example Packer templates] you can use to create operating system images that include `nodeadm` and configure it to run at host-startup. This process is recommended to avoid pulling the hybrid nodes dependencies individually on each host and to automate the hybrid nodes bootstrap process. You can use the example Packer templates with an Ubuntu 22.04, Ubuntu 24.04, RHEL 8 or RHEL 9 ISO image and can output images with these formats: OVA, Qcow2, or raw. === Prerequisites @@ -79,7 +87,7 @@ Before running the Packer build, set the following environment variables on the The following environment variables must be set for building images with all operating systems and output formats. -[cols="1,1,1", options="header"] +[%header,cols="3"] |=== |Environment Variable @@ -104,7 +112,7 @@ The following environment variables must be set for building images with all ope |K8S_VERSION |String -|Kubernetes version for hybrid nodes (for example `1.31`). For supported Kubernetes versions, see <>. +|Kubernetes version for hybrid nodes (for example `1.31`). For supported Kubernetes versions, see link:eks/latest/userguide/kubernetes-versions.html[Amazon EKS supported versions,type="documentation"]. |NODEADM_ARCH |String @@ -116,7 +124,7 @@ The following environment variables must be set for building images with all ope If you are using RHEL, the following environment variables must be set. -[cols="1,1,1", options="header"] +[%header,cols="3"] |=== |Environment Variable @@ -145,7 +153,7 @@ There are no Ubuntu-specific environment variables required. If you are building a VMware vSphere OVA, the following environment variables must be set. -[cols="1,1,1", options="header"] +[%header,cols="3"] |=== |Environment Variable @@ -188,7 +196,7 @@ If you are building a VMware vSphere OVA, the following environment variables mu *QEMU* -[cols="1,1,1", options="header"] +[%header,cols="3"] |=== |Environment Variable @@ -364,4 +372,4 @@ govc vm.change -dc="YOUR_DATASTORE" -vm "$NODE_NAME" -e guestinfo.metadata.encod [source,yaml,subs="verbatim,attributes,quotes"] ---- govc vm.power -on "${NODE_NAME}" ----- +---- \ No newline at end of file diff --git a/latest/ug/nodes/hybrid-nodes-overview.adoc b/latest/ug/nodes/hybrid-nodes-overview.adoc new file mode 100644 index 000000000..6936de923 --- /dev/null +++ b/latest/ug/nodes/hybrid-nodes-overview.adoc @@ -0,0 +1,65 @@ +include::../attributes.txt[] + +[.topic] +[#hybrid-nodes-overview] += Amazon EKS Hybrid Nodes overview +:info_titleabbrev: Hybrid nodes + +[abstract] +-- +Join nodes from your data centers to Amazon EKS Kubernetes clusters with Amazon EKS Hybrid Nodes. +-- + +With _Amazon EKS Hybrid Nodes_, you can use your on-premises and edge infrastructure as nodes in Amazon EKS clusters. {aws} manages the {aws}-hosted Kubernetes control plane of the Amazon EKS cluster, and you manage the hybrid nodes that run in your on-premises or edge environments. This unifies Kubernetes management across your environments and offloads Kubernetes control plane management to {aws} for your on-premises and edge applications. + +Amazon EKS Hybrid Nodes works with any on-premises hardware or virtual machines, bringing the efficiency, scalability, and availability of Amazon EKS to wherever your applications need to run. You can use a wide range of Amazon EKS features with Amazon EKS Hybrid Nodes including Amazon EKS add-ons, Amazon EKS Pod Identity, cluster access entries, cluster insights, and extended Kubernetes version support. Amazon EKS Hybrid Nodes natively integrates with {aws} services including {aws} Systems Manager, {aws} IAM Roles Anywhere, Amazon Managed Service for Prometheus, and Amazon CloudWatch for centralized monitoring, logging, and identity management. + +With Amazon EKS Hybrid Nodes, there are no upfront commitments or minimum fees, and you are charged per hour for the vCPU resources of your hybrid nodes when they are attached to your Amazon EKS clusters. For more pricing information, see link:eks/pricing/[Amazon EKS Pricing,type="marketing"]. + +video::tFn9IdlddBw[youtube,align=center,height= 405,width=720,fileref = https://www.youtube.com/embed/tFn9IdlddBw] + +[#hybrid-nodes-features] +== Features + +EKS Hybrid Nodes has the following high-level features: + +* *Managed Kubernetes control plane*: {aws} manages the {aws}-hosted Kubernetes control plane of the EKS cluster, and you manage the hybrid nodes that run in your on-premises or edge environments. This unifies Kubernetes management across your environments and offloads Kubernetes control plane management to {aws} for your on-premises and edge applications. By moving the Kubernetes control plane to {aws}, you can conserve on-premises capacity for your applications and trust that the Kubernetes control plane scales with your workloads. +* *Consistent EKS experience*: Most EKS features are supported with EKS Hybrid Nodes for a consistent EKS experience across your on-premises and cloud environments including EKS add-ons, EKS Pod Identity, cluster access entries, cluster insights, extended Kubernetes version support, and more. See <> for more information on the EKS add-ons supported with EKS Hybrid Nodes. +* *Centralized observability and identity management*: EKS Hybrid Nodes natively integrates with {aws} services including {aws} Systems Manager, {aws} IAM Roles Anywhere, Amazon Managed Service for Prometheus, and Amazon CloudWatch for centralized monitoring, logging, and identity management. +* *Burst-to-cloud or add on-premises capacity*: A single EKS cluster can be used to run hybrid nodes and nodes in {aws} Regions, {aws} Local Zones, or {aws} Outposts to burst-to-cloud or add on-premises capacity to your EKS clusters. See <> for more information. +* *Flexible infrastructure*: EKS Hybrid Nodes follows a _bring your own infrastructure_ approach and is agnostic to the infrastructure you use for hybrid nodes. You can run hybrid nodes on physical or virtual machines, and x86 and ARM architectures, making it possible to migrate on-premises workloads running on hybrid nodes across different infrastructure types. +* *Flexible networking*: With EKS Hybrid Nodes, communication between the EKS control plane and hybrid nodes is routed through the VPC and subnets you pass during cluster creation, which builds on the link:eks/latest/best-practices/subnets.html[existing mechanism,type="documentation"] in EKS for control plane to node networking. This is flexible to your preferred method of connecting your on-premises networks to a VPC in {aws}. There are several link:whitepapers/latest/aws-vpc-connectivity-options/network-to-amazon-vpc-connectivity-options.html[documented options,type="documentation"] available including {aws} Site-to-Site VPN, {aws} Direct Connect, or your own VPN solution, and you can choose the method that best fits your use case. + +[#hybrid-node-limits] +== Limits + +* Up to 15 CIDRs for Remote Node Networks and 15 CIDRs for Remote Pod Networks per cluster are supported. + +[#hybrid-nodes-general] +== Considerations + +* EKS Hybrid Nodes can be used with new or existing EKS clusters. +* EKS Hybrid Nodes is available in all {aws} Regions, except the {aws} GovCloud (US) Regions and the {aws} China Regions. +* EKS Hybrid Nodes must have a reliable connection between your on-premises environment and {aws}. EKS Hybrid Nodes is not a fit for disconnected, disrupted, intermittent or limited (DDIL) environments. If you are running in a DDIL environment, consider link:eks/eks-anywhere/[Amazon EKS Anywhere,type="marketing"]. Reference the link:eks/latest/best-practices/hybrid-nodes-network-disconnections.html[Best Practices for EKS Hybrid Nodes,type="documentation"] for information on how hybrid nodes behave during network disconnection scenarios. +* Running EKS Hybrid Nodes on cloud infrastructure, including {aws} Regions, {aws} Local Zones, {aws} Outposts, or in other clouds, is not supported. You will be charged the hybrid nodes fee if you run hybrid nodes on Amazon EC2 instances. +* Billing for hybrid nodes starts when the nodes join the EKS cluster and stops when the nodes are removed from the cluster. Be sure to remove your hybrid nodes from your EKS cluster if you are not using them. + +[#hybrid-nodes-resources] +== Additional resources + +* link:https://www.eksworkshop.com/docs/networking/eks-hybrid-nodes/[**EKS Hybrid Nodes workshop**]: Step-by-step instructions for deploying EKS Hybrid Nodes in a demo environment. +* link:https://www.youtube.com/watch?v=ZxC7SkemxvU[**{aws} re:Invent: EKS Hybrid Nodes**]: {aws} re:Invent session introducing the EKS Hybrid Nodes launch with a customer showing how they are using EKS Hybrid Nodes in their environment. +* link:https://repost.aws/articles/ARL44xuau6TG2t-JoJ3mJ5Mw/unpacking-the-cluster-networking-for-amazon-eks-hybrid-nodes[**{aws} re:Post: Cluster networking for EKS Hybrid Nodes**]: Article explaining various methods for setting up networking for EKS Hybrid Nodes. +* link:https://aws.amazon.com/blogs/containers/run-genai-inference-across-environments-with-amazon-eks-hybrid-nodes/[**{aws} blog: Run GenAI inference across environments with EKS Hybrid Nodes**]: Blog post showing how to run GenAI inference across environments with EKS Hybrid Nodes. + +include::hybrid-nodes-prereqs.adoc[leveloffset=+1] + +include::hybrid-nodes-tutorial.adoc[leveloffset=+1] + +include::hybrid-nodes-configure.adoc[leveloffset=+1] + +include::hybrid-nodes-concepts.adoc[leveloffset=+1] + +include::hybrid-nodes-nodeadm.adoc[leveloffset=+1] + +include::hybrid-nodes-troubleshooting.adoc[leveloffset=+1] diff --git a/latest/ug/nodes/hybrid-nodes-prereqs.adoc b/latest/ug/nodes/hybrid-nodes-prereqs.adoc index 4f8711190..5f20aacd2 100644 --- a/latest/ug/nodes/hybrid-nodes-prereqs.adoc +++ b/latest/ug/nodes/hybrid-nodes-prereqs.adoc @@ -1,18 +1,13 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[hybrid-nodes-prereqs,hybrid-nodes-prereqs.title]] +[#hybrid-nodes-prereqs] = Prerequisite setup for hybrid nodes -:info_doctype: section -:info_title: Prerequisite setup for hybrid nodes :info_titleabbrev: Prerequisites -:keywords: on-premises prerequisites, hybrid prerequisites -:info_abstract: Prerequisites and requirements for Amazon EKS hybrid nodes - -include::../attributes.txt[] [abstract] -- -Learn about the prerequisites and requirements for joining nodes from your data centers to Amazon EKS [.noloc]`Kubernetes` clusters with Amazon EKS Hybrid Nodes. +Learn about the prerequisites and requirements for joining nodes from your data centers to Amazon EKS Kubernetes clusters with Amazon EKS Hybrid Nodes. -- To use Amazon EKS Hybrid Nodes, you must have private connectivity from your on-premises environment to/from {aws}, bare metal servers or virtual machines with a supported operating system, and {aws} IAM Roles Anywhere or {aws} Systems Manager (SSM) hybrid activations configured. You are responsible for managing these prerequisites throughout the hybrid nodes lifecycle. @@ -25,40 +20,39 @@ To use Amazon EKS Hybrid Nodes, you must have private connectivity from your on- image::images/hybrid-prereq-diagram.png[Hybrid node network connectivity.,scaledwidth=50%] -[[hybrid-nodes-prereqs-connect,hybrid-nodes-prereqs-connect.title]] +[#hybrid-nodes-prereqs-connect] == Hybrid network connectivity -The communication between the Amazon EKS control plane and hybrid nodes is routed through the VPC and subnets you pass during cluster creation, which builds on the https://aws.github.io/aws-eks-best-practices/networking/subnets/[existing mechanism] in Amazon EKS for control plane to node networking. There are several https://docs.aws.amazon.com/whitepapers/latest/aws-vpc-connectivity-options/network-to-amazon-vpc-connectivity-options.html[documented options] available for you to connect your on-premises environment with your VPC including {aws} Site-to-Site VPN, {aws} Direct Connect, or your own VPN connection. Reference the https://docs.aws.amazon.com/vpn/latest/s2svpn/VPC_VPN.html[{aws} Site-to-Site VPN] and https://docs.aws.amazon.com/directconnect/latest/UserGuide/Welcome.html[{aws} Direct Connect] user guides for more information on how to use those solutions for your hybrid network connection. +The communication between the Amazon EKS control plane and hybrid nodes is routed through the VPC and subnets you pass during cluster creation, which builds on the https://aws.github.io/aws-eks-best-practices/networking/subnets/[existing mechanism] in Amazon EKS for control plane to node networking. There are several link:whitepapers/latest/aws-vpc-connectivity-options/network-to-amazon-vpc-connectivity-options.html[documented options,type="documentation"] available for you to connect your on-premises environment with your VPC including {aws} Site-to-Site VPN, {aws} Direct Connect, or your own VPN connection. Reference the link:vpn/latest/s2svpn/VPC_VPN.html[{aws} Site-to-Site VPN,type="documentation"] and link:directconnect/latest/UserGuide/Welcome.html[{aws} Direct Connect,type="documentation"] user guides for more information on how to use those solutions for your hybrid network connection. -For an optimal experience, {aws} recommends reliable network connectivity of at least 100 Mbps and a maximum of 200ms round trip latency for the hybrid nodes connection to the {aws} Region. The bandwidth and latency requirements can vary depending on the number of hybrid nodes and your workload characteristics, such as application image size, application elasticity, monitoring and logging configurations, and application dependencies on accessing data stored in other {aws} services. We recommend that you test with your own applications and environments before deploying to production to validate that your networking setup meets the requirements for your workloads. +For an optimal experience, we recommend that you have reliable network connectivity of at least 100 Mbps and a maximum of 200ms round trip latency for the hybrid nodes connection to the {aws} Region. This is general guidance that accommodates most use cases but is not a strict requirement. The bandwidth and latency requirements can vary depending on the number of hybrid nodes and your workload characteristics, such as application image size, application elasticity, monitoring and logging configurations, and application dependencies on accessing data stored in other {aws} services. We recommend that you test with your own applications and environments before deploying to production to validate that your networking setup meets the requirements for your workloads. -[[hybrid-nodes-prereqs-onprem,hybrid-nodes-prereqs-onprem.title]] +[#hybrid-nodes-prereqs-onprem] == On-premises network configuration -You must enable inbound network access from the Amazon EKS control plane to your on-premises environment to allow the Amazon EKS control plane to communicate with the `kubelet` running on hybrid nodes and optionally with webhooks running on your hybrid nodes. Additionally, you must enable outbound network access for your hybrid nodes and components running on them to communicate with the Amazon EKS control plane. You can configure this communication to stay fully private to your {aws} Direct Connect, {aws} Site-to-Site VPN, or your own VPN connection. For a full list of the required ports and protocols that you must enable in your firewall and on-premises environment, see <>. +You must enable inbound network access from the Amazon EKS control plane to your on-premises environment to allow the Amazon EKS control plane to communicate with the `kubelet` running on hybrid nodes and optionally with webhooks running on your hybrid nodes. Additionally, you must enable outbound network access for your hybrid nodes and components running on them to communicate with the Amazon EKS control plane. You can configure this communication to stay fully private to your {aws} Direct Connect, {aws} Site-to-Site VPN, or your own VPN connection. -The Classless Inter-Domain Routing (CIDR) ranges you use for your on-premises node and pod networks must use IPv4 RFC1918 address ranges. When you create your hybrid nodes-enabled Amazon EKS cluster, you pass your on-premises node and optionally pod CIDRs to enable communication from the Amazon EKS control plane to your hybrid nodes and the resources running on them. Your on-premises router must be configured with routes to your on-premises nodes and optionally pods. You can use Border Gateway Protocol (BGP) or static configurations to advertise pod IPs to your router. +The Classless Inter-Domain Routing (CIDR) ranges you use for your on-premises node and pod networks must use IPv4 RFC-1918 address ranges. Your on-premises router must be configured with routes to your on-premises nodes and optionally pods. See <> for more information on the on-premises network requirements, including the full list of required ports and protocols that must be enabled in your firewall and on-premises environment. - -[[hybrid-nodes-prereqs-cluster,hybrid-nodes-prereqs-cluster.title]] +[#hybrid-nodes-prereqs-cluster] == EKS cluster configuration -To minimize latency, it is recommended to create your Amazon EKS cluster in the {aws} Region closest to your on-premises or edge environment. You pass your on-premises node and pod CIDRs during Amazon EKS cluster creation via two API fields: `RemoteNodeNetwork` and `RemotePodNetwork`. You may need to discuss with your on-premises network team to identify your on-premises node and pod CIDRs. The node CIDR is allocated from your on-premises network and the pod CIDR is allocated from the Container Network Interface (CNI) you use if you are using an overlay network for your CNI. +To minimize latency, we recommend that you create your Amazon EKS cluster in the {aws} Region closest to your on-premises or edge environment. You pass your on-premises node and pod CIDRs during Amazon EKS cluster creation via two API fields: `RemoteNodeNetwork` and `RemotePodNetwork`. You may need to discuss with your on-premises network team to identify your on-premises node and pod CIDRs. The node CIDR is allocated from your on-premises network and the pod CIDR is allocated from the Container Network Interface (CNI) you use if you are using an overlay network for your CNI. Cilium and Calico use overlay networks by default. -The on-premises node and pod CIDRs are used to configure the Amazon EKS control plane to route traffic through your VPC to the `kubelet` and the pods running on your hybrid nodes. Your on-premises node and pod CIDRs cannot overlap with each other, the VPC CIDR you pass during cluster creation, or the service IPv4 configuration for your Amazon EKS cluster. The pod CIDR is optional. You must configure your pod CIDR if your CNI does not use Network Address Translation (NAT) or masquerading for pod IP addresses when pod traffic leaves your on-premises hosts. You additionally must configure your pod CIDR if you are running _Kubernetes webhooks_ on hybrid nodes. For example, {aws} Distro for Open Telemetry (ADOT) uses webhooks. +The on-premises node and pod CIDRs you configure via the `RemoteNodeNetwork` and `RemotePodNetwork` fields are used to configure the Amazon EKS control plane to route traffic through your VPC to the `kubelet` and the pods running on your hybrid nodes. Your on-premises node and pod CIDRs cannot overlap with each other, the VPC CIDR you pass during cluster creation, or the service IPv4 configuration for your Amazon EKS cluster. Also, Pod CIDRs must be unique to each EKS cluster so that your on-premises router can route traffic. -It is recommended to use either public or private endpoint access for the Amazon EKS Kubernetes API server endpoint. If you choose “Public and Private”, the Amazon EKS Kubernetes API server endpoint will always resolve to the public IPs for hybrid nodes running outside of your VPC, which can prevent your hybrid nodes from joining the cluster. You can use either public or private endpoint access for the Amazon EKS Kubernetes API server endpoint. You cannot choose “Public and Private”. When you use public endpoint access, the Kubernetes API server endpoint is resolved to public IPs and the communication from hybrid nodes to the Amazon EKS control plane will be routed over the internet. When you choose private endpoint access, the Kubernetes API server endpoint is resolved to private IPs and the communication from hybrid nodes to the Amazon EKS control plane will be routed over your private connectivity link, in most cases {aws} Direct Connect or {aws} Site-to-Site VPN. +We recommend that you use either public or private endpoint access for the Amazon EKS Kubernetes API server endpoint. If you choose “Public and Private”, the Amazon EKS Kubernetes API server endpoint will always resolve to the public IPs for hybrid nodes running outside of your VPC, which can prevent your hybrid nodes from joining the cluster. When you use public endpoint access, the Kubernetes API server endpoint is resolved to public IPs and the communication from hybrid nodes to the Amazon EKS control plane will be routed over the internet. When you choose private endpoint access, the Kubernetes API server endpoint is resolved to private IPs and the communication from hybrid nodes to the Amazon EKS control plane will be routed over your private connectivity link, in most cases {aws} Direct Connect or {aws} Site-to-Site VPN. -[[hybrid-nodes-prereqs-vpc,hybrid-nodes-prereqs-vpc.title]] +[#hybrid-nodes-prereqs-vpc] == VPC configuration You must configure the VPC you pass during Amazon EKS cluster creation with routes in its routing table for your on-premises node and optionally pod networks with your virtual private gateway (VGW) or transit gateway (TGW) as the target. An example is shown below. Replace `REMOTE_NODE_CIDR` and `REMOTE_POD_CIDR` with the values for your on-premises network. -[cols="1,1,1", options="header"] +[%header,cols="3"] |=== |Destination |Target @@ -78,12 +72,12 @@ You must configure the VPC you pass during Amazon EKS cluster creation with rout |=== -[[hybrid-nodes-prereqs-sg,hybrid-nodes-prereqs-sg.title]] +[#hybrid-nodes-prereqs-sg] == Security group configuration When you create a cluster, Amazon EKS creates a security group that's named `eks-cluster-sg--`. You cannot alter the inbound rules of this Cluster Security Group but you can restrict the outbound rules. You must add an additional security group to your cluster to enable the kubelet and optionally webhooks running on your hybrid nodes to contact the Amazon EKS control plane. The required inbound rules for this additional security group are shown below. Replace `REMOTE_NODE_CIDR` and `REMOTE_POD_CIDR` with the values for your on-premises network. -[cols="1,1,1,1,1,1,1", options="header"] +[%header,cols="7"] |=== |Name |Security group rule ID @@ -111,24 +105,24 @@ When you create a cluster, Amazon EKS creates a security group that's named `eks |=== -[[hybrid-nodes-prereqs-infra,hybrid-nodes-prereqs-infra.title]] +[#hybrid-nodes-prereqs-infra] == Infrastructure -You must have bare metal servers or virtual machines available to use as hybrid nodes. Hybrid nodes are agnostic to the underlying infrastructure and support x86 and ARM architectures. Amazon EKS Hybrid Nodes follows a “bring your own infrastructure” approach, where you are responsible for provisioning and managing the bare metal servers or virtual machines that you use for hybrid nodes. While there is not a strict minimum resource requirement, it is recommended to use hosts with at least 1 vCPU and 1GiB RAM for hybrid nodes. +You must have bare metal servers or virtual machines available to use as hybrid nodes. Hybrid nodes are agnostic to the underlying infrastructure and support x86 and ARM architectures. Amazon EKS Hybrid Nodes follows a “bring your own infrastructure” approach, where you are responsible for provisioning and managing the bare metal servers or virtual machines that you use for hybrid nodes. While there is not a strict minimum resource requirement, we recommend that you use hosts with at least 1 vCPU and 1GiB RAM for hybrid nodes. -[[hybrid-nodes-prereqs-os,hybrid-nodes-prereqs-os.title]] +[#hybrid-nodes-prereqs-os] == Operating system -Amazon Linux 2023 (AL2023), Ubuntu, and RHEL are validated on an ongoing basis for use as the node operating system for hybrid nodes. {aws} supports the hybrid nodes integration with these operating systems but does not provide support for the operating systems itself. AL2023 is not covered by {aws} Support Plans when run outside of Amazon EC2. AL2023 can only be used in on-premises virtualized environments, see the link:linux/al2023/ug/outside-ec2.html[Amazon Linux 2023 User Guide,type="documentation"] for more information. +Bottlerocket, Amazon Linux 2023 (AL2023), Ubuntu, and RHEL are validated on an ongoing basis for use as the node operating system for hybrid nodes. Bottlerocket is supported by {aws}in VMware vSphere environments only. AL2023 is not covered by {aws} Support Plans when run outside of Amazon EC2. AL2023 can only be used in on-premises virtualized environments, see the link:linux/al2023/ug/outside-ec2.html[Amazon Linux 2023 User Guide,type="documentation"] for more information. {aws} supports the hybrid nodes integration with Ubuntu and RHEL operating systems but does not provide support for the operating system itself. -You are responsible for operating system provisioning and management. When you are testing hybrid nodes for the first time, it is easiest to run the Amazon EKS Hybrid Nodes CLI (`nodeadm`) on an already provisioned host. For production deployments, it is recommended to include `nodeadm` in your golden operating system images with it configured to run as a systemd service to automatically join hosts to Amazon EKS clusters at host startup. +You are responsible for operating system provisioning and management. When you are testing hybrid nodes for the first time, it is easiest to run the Amazon EKS Hybrid Nodes CLI (`nodeadm`) on an already provisioned host. For production deployments, we recommend that you include `nodeadm` in your golden operating system images with it configured to run as a systemd service to automatically join hosts to Amazon EKS clusters at host startup. -[[hybrid-nodes-prereqs-iam,hybrid-nodes-prereqs-iam.title]] +[#hybrid-nodes-prereqs-iam] == On-premises IAM credentials provider -Amazon EKS Hybrid Nodes use temporary IAM credentials provisioned by {aws} SSM hybrid activations or {aws} IAM Roles Anywhere to authenticate with the Amazon EKS cluster. You must use either {aws} SSM hybrid activations or {aws} IAM Roles Anywhere with the Amazon EKS Hybrid Nodes CLI (`nodeadm`). It is recommended to use {aws} SSM hybrid activations if you do not have existing Public Key Infrastructure (PKI) with a Certificate Authority (CA) and certificates for your on-premises environments. If you do have existing PKI and certificates on-premises, use {aws} IAM Roles Anywhere. +Amazon EKS Hybrid Nodes use temporary IAM credentials provisioned by {aws} SSM hybrid activations or {aws} IAM Roles Anywhere to authenticate with the Amazon EKS cluster. You must use either {aws} SSM hybrid activations or {aws} IAM Roles Anywhere with the Amazon EKS Hybrid Nodes CLI (`nodeadm`). We recommend that you use {aws} SSM hybrid activations if you do not have existing Public Key Infrastructure (PKI) with a Certificate Authority (CA) and certificates for your on-premises environments. If you do have existing PKI and certificates on-premises, use {aws} IAM Roles Anywhere. Similar to the <> for nodes running on Amazon EC2, you will create a Hybrid Nodes IAM Role with the required permissions to join hybrid nodes to Amazon EKS clusters. If you are using {aws} IAM Roles Anywhere, configure a trust policy that allows {aws} IAM Roles Anywhere to assume the Hybrid Nodes IAM Role and configure your {aws} IAM Roles Anywhere profile with the Hybrid Nodes IAM Role as an assumable role. If you are using {aws} SSM, configure a trust policy that allows {aws} SSM to assume the Hybrid Nodes IAM Role and create the hybrid activation with the Hybrid Nodes IAM Role. See <> for how to create the Hybrid Nodes IAM Role with the required permissions. @@ -145,4 +139,7 @@ include::hybrid-nodes-creds.adoc[leveloffset=+1] include::hybrid-nodes-cluster-create.adoc[leveloffset=+1] -include::hybrid-nodes-cluster-prep.adoc[leveloffset=+1] +include::hybrid-nodes-cluster-update.adoc[leveloffset=+1] + + +include::hybrid-nodes-cluster-prep.adoc[leveloffset=+1] \ No newline at end of file diff --git a/latest/ug/nodes/hybrid-nodes-proxy.adoc b/latest/ug/nodes/hybrid-nodes-proxy.adoc index e9053a4ae..3432fbcd4 100644 --- a/latest/ug/nodes/hybrid-nodes-proxy.adoc +++ b/latest/ug/nodes/hybrid-nodes-proxy.adoc @@ -1,25 +1,26 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[hybrid-nodes-proxy,hybrid-nodes-proxy.title]] +[#hybrid-nodes-proxy] = Configure proxy for hybrid nodes -:info_doctype: section -:info_title: Configure proxy for hybrid nodes :info_titleabbrev: Configure proxy -:keywords: on-premises proxy, hybrid proxy -:info_abstract: Configure HTTP/S proxies for Amazon EKS hybrid nodes - -include::../attributes.txt[] [abstract] -- Configure HTTP/S proxies for Amazon EKS hybrid nodes -- -If you are using a proxy server in your on-premises environment for traffic leaving your data center or edge environment, you need to configure your operating system, `containerd`, `kubelet`, and `kube-proxy` to use your proxy server. You must configure `kube-proxy` after creating your Amazon EKS cluster. You can make the changes for your operating system, `containerd`, and the `kubelet` during the build process for your operating system images or before you run `nodeadm init` on each hybrid node. +If you are using a proxy server in your on-premises environment for traffic leaving your data center or edge environment, you need to separately configure your nodes and your cluster to use your proxy server. + +Cluster:: +On your cluster, you need to configure `kube-proxy` to use your proxy server. You must configure `kube-proxy` after creating your Amazon EKS cluster. + +Nodes:: +On your nodes, you must configure the operating system, `containerd`, `kubelet`, and the Amazon SSM agent to use your proxy server. You can make these changes during the build process for your operating system images or before you run `nodeadm init` on each hybrid node. == Node-level configuration -The configurations in this section must be applied in your operating system images or before running `nodeadm init` on each hybrid node. +You must apply the following configurations either in your operating system images or before running `nodeadm init` on each hybrid node. === `containerd` proxy configuration @@ -30,12 +31,27 @@ Create a file on each hybrid node called `http-proxy.conf` in the `/etc/systemd/ [source,yaml,subs="verbatim,attributes,quotes"] ---- [Service] -Environment="HTTP_PROXY=http://proxy-domain:port" -Environment="HTTPS_PROXY=http://proxy-domain:port" +Environment="HTTP_PROXY=http://[.replaceable]#proxy-domain:port#" +Environment="HTTPS_PROXY=http://[.replaceable]#proxy-domain:port#" Environment="NO_PROXY=localhost" ---- -=== Kubelet proxy configuration +==== `containerd` configuration from user data + +The `containerd.service.d` directory will need to be created for this file. You will need to reload systemd to pick up the configuration file without a reboot. In AL2023, the service will likely already be running when your script executes, so you will also need to restart it. + +[source,yaml,subs="verbatim,attributes,quotes"] +---- +mkdir -p /etc/systemd/system/containerd.service.d +echo '[Service]' > /etc/systemd/system/containerd.service.d/http-proxy.conf +echo 'Environment="HTTP_PROXY=http://[.replaceable]#proxy-domain:port#"' >> /etc/systemd/system/containerd.service.d/http-proxy.conf +echo 'Environment="HTTPS_PROXY=http://[.replaceable]#proxy-domain:port#"' >> /etc/systemd/system/containerd.service.d/http-proxy.conf +echo 'Environment="NO_PROXY=localhost"' >> /etc/systemd/system/containerd.service.d/http-proxy.conf +systemctl daemon-reload +systemctl restart containerd +---- + +=== `kubelet` proxy configuration `kubelet` is the Kubernetes node agent that runs on each Kubernetes node and is responsible for managing the node and pods running on it. If you are using a proxy in your on-premises environment, you must configure the `kubelet` so it can communicate with your Amazon EKS cluster's public or private endpoints. @@ -44,11 +60,68 @@ Create a file on each hybrid node called `http-proxy.conf` in the `/etc/systemd/ [source,yaml,subs="verbatim,attributes,quotes"] ---- [Service] -Environment="HTTP_PROXY=http://proxy-domain:port" -Environment="HTTPS_PROXY=http://proxy-domain:port" +Environment="HTTP_PROXY=http://[.replaceable]#proxy-domain:port#" +Environment="HTTPS_PROXY=http://[.replaceable]#proxy-domain:port#" +Environment="NO_PROXY=localhost" +---- + +==== `kubelet` configuration from user data + +The `kubelet.service.d` directory must be created for this file. You will need to reload systemd to pick up the configuration file without a reboot. In AL2023, the service will likely already be running when your script executes, so you will also need to restart it. + +[source,yaml,subs="verbatim,attributes,quotes"] +---- +mkdir -p /etc/systemd/system/kubelet.service.d +echo '[Service]' > /etc/systemd/system/kubelet.service.d/http-proxy.conf +echo 'Environment="HTTP_PROXY=http://[.replaceable]#proxy-domain:port#"' >> /etc/systemd/system/kubelet.service.d/http-proxy.conf +echo 'Environment="HTTPS_PROXY=http://[.replaceable]#proxy-domain:port#"' >> /etc/systemd/system/kubelet.service.d/http-proxy.conf +echo 'Environment="NO_PROXY=localhost"' >> /etc/systemd/system/kubelet.service.d/http-proxy.conf +systemctl daemon-reload +systemctl restart kubelet +---- + +=== `ssm` proxy configuration + +`ssm` is one of the credential providers that can be used to initialize a hybrid node. `ssm` is responsible for authenticating with {aws} and generating temporary credentials that is used by `kubelet`. If you are using a proxy in your on-premises environment and using `ssm` as your credential provider on the node, you must configure the `ssm` so it can communicate with Amazon SSM service endpoints. + +Create a file on each hybrid node called `http-proxy.conf` in the path below depending on the operating system + +* Ubuntu - `/etc/systemd/system/snap.amazon-ssm-agent.amazon-ssm-agent.service.d/http-proxy.conf` +* Amazon Linux 2023 and Red Hat Enterprise Linux - `/etc/systemd/system/amazon-ssm-agent.service.d/http-proxy.conf` + +Populate the file with the following contents. Replace `proxy-domain` and `port` with the values for your environment. +[source,yaml,subs="verbatim,attributes,quotes"] +---- +[Service] +Environment="HTTP_PROXY=http://[.replaceable]#proxy-domain:port#" +Environment="HTTPS_PROXY=http://[.replaceable]#proxy-domain:port#" Environment="NO_PROXY=localhost" ---- +==== `ssm` configuration from user data + +The `ssm` systemd service file directory must be created for this file. The directory path depends on the operating system used on the node. + +* Ubuntu - `/etc/systemd/system/snap.amazon-ssm-agent.amazon-ssm-agent.service.d` +* Amazon Linux 2023 and Red Hat Enterprise Linux - `/etc/systemd/system/amazon-ssm-agent.service.d` + +Replace the systemd service name in the restart command below depending on the operating system used on the node + +* Ubuntu - `snap.amazon-ssm-agent.amazon-ssm-agent` +* Amazon Linux 2023 and Red Hat Enterprise Linux - `amazon-ssm-agent` + +[source,yaml,subs="verbatim,attributes,quotes"] +---- +mkdir -p [.replaceable]#systemd-service-file-directory +echo '[Service]' > [.replaceable]#systemd-service-file-directory/http-proxy.conf +echo 'Environment="HTTP_PROXY=http://[.replaceable]#proxy-domain:port#"' >> [.replaceable]#systemd-service-file-directory/http-proxy.conf +echo 'Environment="HTTPS_PROXY=http://[.replaceable]#proxy-domain:port#"' >> [.replaceable]#systemd-service-file-directory/http-proxy.conf +echo 'Environment="NO_PROXY=localhost"' >> [.replaceable]#systemd-service-file-directory/http-proxy.conf +systemctl daemon-reload +systemctl restart [.replaceable]#systemd-service-name +---- + + === Operating system proxy configuration If you are using a proxy for internet access, you must configure your operating system to be able to pull the hybrid nodes dependencies from your operating systems' package manager. @@ -59,25 +132,48 @@ If you are using a proxy for internet access, you must configure your operating + [source,yaml,subs="verbatim,attributes,quotes"] ---- -sudo snap set system proxy.https=http://proxy-domain:port -sudo snap set system proxy.http=http://proxy-domain:port +sudo snap set system proxy.https=http://[.replaceable]#proxy-domain:port# +sudo snap set system proxy.http=http://[.replaceable]#proxy-domain:port# ---- . To enable proxy for `apt`, create a file called `apt.conf` in the `/etc/apt/` directory. Replace proxy-domain and port with the values for your environment. + [source,yaml,subs="verbatim,attributes,quotes"] ---- -Acquire::http::Proxy "http://proxy-domain:port"; -Acquire::https::Proxy "http://proxy-domain:port"; +Acquire::http::Proxy "http://[.replaceable]#proxy-domain:port#"; +Acquire::https::Proxy "http://[.replaceable]#proxy-domain:port#"; ---- -*Amazon Linux 2023 and Red Hat Enterprise Linux* +*Amazon Linux 2023* + +. Configure `dnf` to use your proxy. Create a file `/etc/dnf/dnf.conf` with the proxy-domain and port values for your environment. ++ +[source,yaml,subs="verbatim,attributes,quotes"] +---- +proxy=http://[.replaceable]#proxy-domain:port# +---- + +*Red Hat Enterprise Linux* . Configure `yum` to use your proxy. Create a file `/etc/yum.conf` with the proxy-domain and port values for your environment. + [source,yaml,subs="verbatim,attributes,quotes"] ---- -proxy=http://proxy-domain:port +proxy=http://[.replaceable]#proxy-domain:port# +---- + +=== IAM Roles Anywhere proxy configuration + +The IAM Roles Anywhere credential provider service is responsible for refreshing credentials when using IAM Roles Anywhere with the `enableCredentialsFile` flag (see <>). If you are using a proxy in your on-premises environment, you must configure the service so it can communicate with IAM Roles Anywhere endpoints. + +Create a file called `http-proxy.conf` in the `/etc/systemd/system/aws_signing_helper_update.service.d/` directory with the following content. Replace `proxy-domain` and `port` with the values for your environment. + +[source,yaml,subs="verbatim,attributes,quotes"] +---- +[Service] +Environment="HTTP_PROXY=http://[.replaceable]#proxy-domain:port#" +Environment="HTTPS_PROXY=http://[.replaceable]#proxy-domain:port#" +Environment="NO_PROXY=localhost" ---- == Cluster wide configuration @@ -108,12 +204,13 @@ containers: - --config=/var/lib/kube-proxy-config/config - --hostname-override=$(NODE_NAME) env: - name: HTTP_PROXY - value: http://proxy-domain:port + value: http://[.replaceable]#proxy-domain:port# - name: HTTPS_PROXY - value: http://proxy-domain:port + value: http://[.replaceable]#proxy-domain:port# - name: NODE_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: spec.nodeName ---- + diff --git a/latest/ug/nodes/hybrid-nodes-remove.adoc b/latest/ug/nodes/hybrid-nodes-remove.adoc index bed35c5cd..3fe1f010f 100644 --- a/latest/ug/nodes/hybrid-nodes-remove.adoc +++ b/latest/ug/nodes/hybrid-nodes-remove.adoc @@ -1,32 +1,27 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[hybrid-nodes-remove,hybrid-nodes-remove.title]] +[#hybrid-nodes-remove] = Remove hybrid nodes -:info_doctype: section -:info_title: Delete hybrid nodes from your EKS cluster :info_titleabbrev: Delete hybrid nodes -:keywords: Delete hybrid nodes from your EKS cluster -:info_abstract: Delete hybrid nodes from your EKS cluster - -include::../attributes.txt[] [abstract] -- Delete hybrid nodes from your EKS cluster -- -This topic describes how to delete hybrid nodes from your Amazon EKS cluster. You must delete your hybrid nodes with your choice of Kubernetes-compatible tooling such as https://kubernetes.io/docs/reference/kubectl/[kubectl]. Charges for hybrid nodes stop when the node object is removed from the Amazon EKS cluster. For more information on hybrid nodes pricing, see https://aws.amazon.com/eks/pricing/[Amazon EKS Pricing]. +This topic describes how to delete hybrid nodes from your Amazon EKS cluster. You must delete your hybrid nodes with your choice of Kubernetes-compatible tooling such as https://kubernetes.io/docs/reference/kubectl/[kubectl]. Charges for hybrid nodes stop when the node object is removed from the Amazon EKS cluster. For more information on hybrid nodes pricing, see link:eks/pricing/[Amazon EKS Pricing,type="marketing"]. [IMPORTANT] ==== -Removing nodes is disruptive to workloads running on the node. Before deleting hybrid nodes, it is recommended to first drain the node to move pods to another active node. For more information on draining nodes, see https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/[Safely Drain a Node] in the Kubernetes documentation. +Removing nodes is disruptive to workloads running on the node. Before deleting hybrid nodes, we recommend that you first drain the node to move pods to another active node. For more information on draining nodes, see https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/[Safely Drain a Node] in the Kubernetes documentation. ==== Run the kubectl steps below from your local machine or instance that you use to interact with the Amazon EKS cluster's Kubernetes API endpoint. If you are using a specific `kubeconfig` file, use the `--kubeconfig` flag. == Step 1: List your nodes -[source,yaml,subs="verbatim,attributes,quotes"] +[source,yaml,subs="verbatim,attributes"] ---- kubectl get nodes ---- @@ -35,7 +30,7 @@ kubectl get nodes See https://kubernetes.io/docs/reference/kubectl/generated/kubectl_drain/[kubectl drain] in the Kubernetes documentation for more information on the `kubectl drain` command. -[source,yaml,subs="verbatim,attributes,quotes"] +[source,yaml,subs="verbatim,attributes"] ---- kubectl drain --ignore-daemonsets ---- @@ -44,7 +39,7 @@ kubectl drain --ignore-daemonsets You can use the Amazon EKS Hybrid Nodes CLI (`nodeadm`) to stop and remove the hybrid nodes artifacts from the host. You must run `nodeadm` with a user that has root/sudo privileges. By default, `nodeadm uninstall` will not proceed if there are pods remaining on the node. If you are using {aws} Systems Manager (SSM) as your credentials provider, the `nodeadm uninstall` command deregisters the host as an {aws} SSM managed instance. For more information, see <>. -[source,yaml,subs="verbatim,attributes,quotes"] +[source,yaml,subs="verbatim,attributes"] ---- nodeadm uninstall ---- @@ -53,10 +48,10 @@ nodeadm uninstall With the hybrid nodes artifacts stopped and uninstalled, remove the node resource from your cluster. -[source,yaml,subs="verbatim,attributes,quotes"] +[source,yaml,subs="verbatim,attributes"] ---- kubectl delete node ---- == Step 5: Check for remaining artifacts -Depending on your choice of CNI, there may be artifacts remaining on your hybrid nodes after running the above steps. See <> for more information. +Depending on your choice of CNI, there may be artifacts remaining on your hybrid nodes after running the above steps. See <> for more information. \ No newline at end of file diff --git a/latest/ug/nodes/hybrid-nodes-security.adoc b/latest/ug/nodes/hybrid-nodes-security.adoc new file mode 100644 index 000000000..5d48cec51 --- /dev/null +++ b/latest/ug/nodes/hybrid-nodes-security.adoc @@ -0,0 +1,101 @@ +[.topic] +[#hybrid-nodes-security] += Patch security updates for hybrid nodes +:info_titleabbrev: Patch hybrid nodes + +include::../attributes.txt[] + +[abstract] +-- +Perform security updates on your Hybrid nodes +-- + +This topic describes the procedure to perform in-place patching of security updates for specific packages and dependencies running on your hybrid nodes. As a best practice we recommend you to regularly update your hybrid nodes to receive CVEs and security patches. + +For steps to upgrade the Kubernetes version, see <>. + +One example of software that might need security patching is `containerd`. + +== `Containerd` + +`containerd` is the standard Kubernetes container runtime and core dependency for EKS Hybrid Nodes, used for managing container lifecycle, including pulling images and managing container execution. On an hybrid node, you can install `containerd` through the link:eks/latest/userguide/hybrid-nodes-nodeadm.html[nodeadm CLI,type="documentation"] or manually. Depending on the operating system of your node, `nodeadm` will install `containerd` from the OS-distributed package or Docker package. + +When a CVE in `containerd` has been published, you have the following options to upgrade to the patched version of `containerd` on your Hybrid nodes. + +== Step 1: Check if the patch published to package managers + +You can check whether the `containerd` CVE patch has been published to each respective OS package manager by referring to the corresponding security bulletins: + +* https://alas.aws.amazon.com/alas2023.html[Amazon Linux 2023] +* https://access.redhat.com/security/security-updates/security-advisories[RHEL] +* https://ubuntu.com/security/notices?order=newest&release=focal[Ubuntu 20.04] +* https://ubuntu.com/security/notices?order=newest&release=jammy[Ubuntu 22.04] +* https://ubuntu.com/security/notices?order=newest&release=noble[Ubuntu 24.04] + +If you use the Docker repo as the source of `containerd`, you can check the https://docs.docker.com/security/security-announcements/[Docker security announcements] to identify the availability of the patched version in the Docker repo. + +== Step 2: Choose the method to install the patch + +There are three methods to patch and install security upgrades in-place on nodes. Which method you can use depends on whether the patch is available from the operating system in the package manager or not: + +. Install patches with `nodeadm upgrade` that are published to package managers, see <>. +. Install patches with the package managers directly, see <>. +. Install custom patches that aren't published in package managers. Note that there are special considerations for custom patches for `containerd`, <>. + +[#hybrid-nodes-security-nodeadm] +== Step 2 a: Patching with `nodeadm upgrade` + +After you confirm that the `containerd` CVE patch has been published to the OS or Docker repos (either Apt or RPM), you can use the `nodeadm upgrade` command to upgrade to the latest version of `containerd`. Since this isn't a Kubernetes version upgrade, you must pass in your current Kubernetes version to the `nodeadm` upgrade command. + +[source,bash,subs="verbatim,attributes,quotes"] +---- +nodeadm upgrade [.replaceable]`K8S_VERSION` --config-source file:///root/nodeConfig.yaml +---- + +[#hybrid-nodes-security-package] +== Step 2 b: Patching with operating system package managers + +Alternatively you can also update through the respective package manager and use it to upgrade the `containerd` package as follows. + +*Amazon Linux 2023* +[source,bash,subs="verbatim,attributes"] +---- +sudo yum update -y +sudo yum install -y containerd +---- + +*RHEL* +[source,bash,subs="verbatim,attributes"] +---- +sudo yum install -y yum-utils +sudo yum-config-manager --add-repo https://download.docker.com/linux/rhel/docker-ce.repo +sudo yum update -y +sudo yum install -y containerd +---- + +*Ubuntu* +[source,bash,subs="verbatim,attributes"] +---- +sudo mkdir -p /etc/apt/keyrings +sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc +echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +sudo apt update -y +sudo apt install -y --only-upgrade containerd.io +---- + +[#hybrid-nodes-security-manual] +== Step 2 c: `Containerd` CVE patch not published in package managers + +If the patched `containerd` version is only available by other means instead of in the package manager, for example in GitHub releases, then you can install `containerd` from the official GitHub site. + +. If the machine has already joined the cluster as a hybrid node, then you need to run the `nodeadm uninstall` command. +. Install the official `containerd` binaries. You can use the steps https://github.com/containerd/containerd/blob/main/docs/getting-started.md#option-1-from-the-official-binaries[official installation steps] on GitHub. +. Run the `nodeadm install` command with the `--containerd-source` argument set to `none`, which will skip `containerd` installation through `nodeadm`. You can use the value of `none` in the `containerd` source for any operating system that the node is running. ++ +[source,bash,subs="verbatim,attributes,quotes"] +---- +nodeadm install [.replaceable]`K8S_VERSION` --credential-provider [.replaceable]`CREDS_PROVIDER` --containerd-source none +---- diff --git a/latest/ug/nodes/hybrid-nodes-troubleshooting.adoc b/latest/ug/nodes/hybrid-nodes-troubleshooting.adoc index 30293b86a..415b1c190 100644 --- a/latest/ug/nodes/hybrid-nodes-troubleshooting.adoc +++ b/latest/ug/nodes/hybrid-nodes-troubleshooting.adoc @@ -1,33 +1,31 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[hybrid-nodes-troubleshooting,hybrid-nodes-troubleshooting.title]] +[#hybrid-nodes-troubleshooting] = Troubleshooting hybrid nodes -:info_doctype: section -:info_title: Troubleshooting hybrid nodes :info_titleabbrev: Troubleshooting -:keywords: on-premises, hybrid -:info_abstract: Amazon EKS hybrid nodes - -include::../attributes.txt[] [abstract] -- -Troubleshoot, diagnose, and repair hybrid nodes from your data centers to Amazon EKS [.noloc]`Kubernetes` clusters. +Troubleshoot, diagnose, and repair hybrid nodes from your data centers to Amazon EKS Kubernetes clusters. -- -This topic covers some common errors that you may see while using Amazon EKS Hybrid Nodes and how to fix them. For other troubleshooting information, see <> and https://repost.aws/tags/knowledge-center/TA4IvCeWI1TE66q4jEj4Z9zg/amazon-elastic-kubernetes-service[Knowledge Center tag for Amazon EKS] on _{aws} re:Post_. If you cannot resolve the issue, contact {aws} Support. +This topic covers some common errors that you might see while using Amazon EKS Hybrid Nodes and how to fix them. For other troubleshooting information, see <> and https://repost.aws/tags/knowledge-center/TA4IvCeWI1TE66q4jEj4Z9zg/amazon-elastic-kubernetes-service[Knowledge Center tag for Amazon EKS] on _{aws} re:Post_. If you cannot resolve the issue, contact {aws} Support. +*Node troubleshooting with `nodeadm debug`* You can run the `nodeadm debug` command from your hybrid nodes to validate networking and credential requirements are met. For more information on the `nodeadm debug` command, see <>. +*Detect issues with your hybrid nodes with cluster insights* +Amazon EKS cluster insights includes __insight checks__ that detect common issues with the configuration of EKS Hybrid Nodes in your cluster. You can view the results of all insight checks from the {aws-management-console}, {aws} CLI, and the {aws} SDKs. For more information about cluster insights, see <>. -[[hybrid-nodes-troubleshooting-install,hybrid-nodes-troubleshooting-install.title]] +[#hybrid-nodes-troubleshooting-install] == Installing hybrid nodes troubleshooting -The troubleshooting topics in this section are related to installing the hybrid nodes dependencies on hosts with the `nodeadm install` command. +The following troubleshooting topics are related to installing the hybrid nodes dependencies on hosts with the `nodeadm install` command. -*nodeadm command failed `must run as root`* +*`nodeadm` command failed `must run as root`* -The `nodeadm install` command must be run with a user that has root/sudo privileges on your host. If you run `nodeadm install` with a user that does not have root/sudo privileges, you will see the following error in the nodeadm output. +The `nodeadm install` command must be run with a user that has root or `sudo` privileges on your host. If you run `nodeadm install` with a user that does not have root or `sudo` privileges, you will see the following error in the `nodeadm` output. [source,bash,subs="verbatim,attributes"] ---- @@ -36,7 +34,7 @@ The `nodeadm install` command must be run with a user that has root/sudo privile *Unable to connect to dependencies* -The `nodeadm install` command installs the dependencies required for hybrid nodes. The hybrid nodes dependencies include `containerd`, `kubelet`, `kubectl`, and {aws} SSM or {aws} IAM Roles Anywhere components. You must have access from where you are running nodeadm install to download these dependencies. For more information on the list of locations that you must be able to access, see <>. If you do not have access, you will see errors similar to the following in the `nodeadm install` output. +The `nodeadm install` command installs the dependencies required for hybrid nodes. The hybrid nodes dependencies include `containerd`, `kubelet`, `kubectl`, and {aws} SSM or {aws} IAM Roles Anywhere components. You must have access from where you are running `nodeadm install` to download these dependencies. For more information on the list of locations that you must be able to access, see <>. If you do not have access, you will see errors similar to the following in the `nodeadm install` output. [source,bash,subs="verbatim,attributes"] ---- @@ -45,7 +43,7 @@ The `nodeadm install` command installs the dependencies required for hybrid node *Failed to update package manager* -The `nodeadm install` command runs apt update or yum/dnf update before installing the hybrid nodes dependencies. If this step does not succeed you may see errors similar to the following. To remediate, you can run apt update or yum/dnf update before running `nodeadm install` or you can attempt to re-run `nodeadm install`. +The `nodeadm install` command runs `apt update` or `yum update` or `dnf update` before installing the hybrid nodes dependencies. If this step does not succeed you might see errors similar to the following. To remediate, you can run `apt update` or `yum update` or `dnf update` before running `nodeadm install` or you can attempt to re-run `nodeadm install`. [source,bash,subs="verbatim,attributes"] ---- @@ -54,7 +52,7 @@ failed to run update using package manager *Timeout or context deadline exceeded* -When running `nodeadm install`, if you see issues at various stages of the install process with errors that indicate there was a timeout or context deadline exceeded, you may have a slow connection that is preventing the installation of the hybrid nodes dependencies before timeouts are met. To work around these issues, you can attempt to use the `--timeout` flag in `nodeadm` to extend the duration of the timeouts for downloading the dependencies. +When running `nodeadm install`, if you see issues at various stages of the install process with errors that indicate there was a timeout or context deadline exceeded, you might have a slow connection that is preventing the installation of the hybrid nodes dependencies before timeouts are met. To work around these issues, you can attempt to use the `--timeout` flag in `nodeadm` to extend the duration of the timeouts for downloading the dependencies. [source,bash,subs="verbatim,attributes,quotes"] ---- @@ -63,7 +61,7 @@ nodeadm install K8S_VERSION --credential-provider CREDS_PROVIDER --timeout [.rep ---- -[[hybrid-nodes-troubleshooting-connect,hybrid-nodes-troubleshooting-connect.title]] +[#hybrid-nodes-troubleshooting-connect] == Connecting hybrid nodes troubleshooting The troubleshooting topics in this section are related to the process of connecting hybrid nodes to EKS clusters with the `nodeadm init` command. @@ -86,10 +84,43 @@ When running `nodeadm init`, `nodeadm` attempts to gather information about your "msg":"Command failed","error":"operation error EKS: DescribeCluster, https response error StatusCode: 403 ... AccessDeniedException" ---- +*Node IP not in remote node network CIDR* + +When running `nodeadm init`, you might encounter an error if the node's IP address is not within the specified remote node network CIDRs. The error will look similar to the following example: + +[source,bash,subs="verbatim,attributes"] +---- +node IP 10.18.0.1 is not in any of the remote network CIDR blocks [10.0.0.0/16 192.168.0.0/16] +---- + +This example shows a node with IP 10.18.0.1 attempting to join a cluster with remote network CIDRs 10.0.0.0/16 and 192.168.0.0/16. The error occurs because 10.18.0.1 isn't within either of the ranges. + +Confirm that you've properly configured your `RemoteNodeNetworks` to include all node IP addresses. For more information on networking configuration, see <>. + +* Run the following command in the region your cluster is located to check your `RemoteNodeNetwork` configurations. Verify that the CIDR blocks listed in the output include the IP range of your node and is the same as the CIDR blocks listed in the error message. If they do not match, confirm the cluster name and region in your `nodeConfig.yaml` match your intended cluster. + +[source,bash,subs="verbatim,attributes,quotes"] +---- +aws eks describe-cluster --name [.replaceable]`CLUSTER_NAME` --region `REGION_NAME` --query cluster.remoteNetworkConfig.remoteNodeNetworks +---- + +* Verify you're working with the intended node: +** Confirm you're on the correct node by checking its hostname and IP address match the one you intend to register with the cluster. +** Confirm this node is in the correct on-premises network (the one whose CIDR range was registered as `RemoteNodeNetwork` during cluster setup). + +If your node IP is still not what you expected, check the following: + +* If you are using IAM Roles Anywhere, `kubelet` performs a DNS lookup on the IAM Roles Anywhere `nodeName` and uses an IP associated with the node name if available. If you maintain DNS entries for your nodes, confirm that these entries point to IPs within your remote node network CIDRs. +* If your node has multiple network interfaces, `kubelet` might select an interface with an IP address outside your remote node network CIDRs as default. To use a different interface, specify its IP address using the `--node-ip` `kubelet` flag in your `nodeConfig.yaml`. For more information, see <>. You can view your node's network interfaces and its IP addresses by running the following command on your node: + +[source,bash,subs="verbatim,attributes"] +---- +ip addr show +---- *Hybrid nodes are not appearing in EKS cluster* -If you ran `nodeadm init` and it completed but your hybrid nodes do not appear in your cluster, there may be issues with the network connection between your hybrid nodes and the EKS control plane, you may not have the required security group permissions configured, or you may not have the required mapping of your Hybrid Nodes IAM role to Kubernetes Role-Based Access Control (RBAC). You can start the debugging process by checking the status of kubelet and the kubelet logs with the following commands. Run the following commands from the hybrid nodes that failed to join your cluster. +If you ran `nodeadm init` and it completed but your hybrid nodes do not appear in your cluster, there might be issues with the network connection between your hybrid nodes and the EKS control plane, you might not have the required security group permissions configured, or you might not have the required mapping of your Hybrid Nodes IAM role to Kubernetes Role-Based Access Control (RBAC). You can start the debugging process by checking the status of `kubelet` and the `kubelet` logs with the following commands. Run the following commands from the hybrid nodes that failed to join your cluster. [source,bash,subs="verbatim,attributes"] ---- @@ -103,7 +134,7 @@ journalctl -u kubelet -f *Unable to communicate with cluster* -If your hybrid node was unable to communicate with the cluster control plane, you may see logs similar to the following. +If your hybrid node was unable to communicate with the cluster control plane, you might see logs similar to the following. [source,bash,subs="verbatim,attributes"] ---- @@ -128,7 +159,7 @@ If you see these messages, check the following to ensure it meets the hybrid nod *Unauthorized* -If your hybrid node was able to communicate with the EKS control plane but was not able to register, you may see logs similar to the following. Note the key difference in the log messages below is the `Unauthorized` error. This signals that the node was not able to perform its tasks because it does not have the required permissions. +If your hybrid node was able to communicate with the EKS control plane but was not able to register, you might see logs similar to the following. Note the key difference in the log messages below is the `Unauthorized` error. This signals that the node was not able to perform its tasks because it does not have the required permissions. [source,bash,subs="verbatim,attributes"] ---- @@ -152,10 +183,10 @@ If you see these messages, check the following to ensure it meets the hybrid nod * Confirm that in your cluster you have an EKS access entry for your Hybrid Nodes IAM role or confirm that your `aws-auth` ConfigMap has an entry for your Hybrid Nodes IAM role. If you are using EKS access entries, confirm your access entry for your Hybrid Nodes IAM role has the `HYBRID_LINUX` access type. If you are using the `aws-auth` ConfigMap, confirm your entry for the Hybrid Nodes IAM role meets the requirements and formatting detailed in <>. -[[hybrid-nodes-troubleshooting-not-ready,hybrid-nodes-troubleshooting-not-ready.title]] +[#hybrid-nodes-troubleshooting-not-ready] === Hybrid nodes registered with EKS cluster but show status `Not Ready` -If your hybrid nodes successfully registered with your EKS cluster, but the hybrid nodes show status `Not Ready`, the first thing to check is your Container Networking Interface (CNI) status. If you have not installed a CNI, then it is expected that your hybrid nodes have status `Not Ready`. Once a CNI is installed and running successfully, nodes transition to have the status Ready. If you attempted to install a CNI but it is not running successfully, see <> on this page. +If your hybrid nodes successfully registered with your EKS cluster, but the hybrid nodes show status `Not Ready`, the first thing to check is your Container Networking Interface (CNI) status. If you have not installed a CNI, then it is expected that your hybrid nodes have status `Not Ready`. Once a CNI is installed and running successfully, nodes are updated to the status `Ready`. If you attempted to install a CNI but it is not running successfully, see <> on this page. *Certificate Signing Requests (CSRs) are stuck Pending* @@ -164,7 +195,7 @@ After connecting hybrid nodes to your EKS cluster, if you see that there are pen *Hybrid profile already exists* -If you changed your `nodeadm` configuration and attempt to re-register the node with the new configuration, you may see an error that states that the hybrid profile already exists but its contents have changed. Instead of running `nodeadm init` in between configuration changes, run `nodeadm uninstall` followed by a `nodeadm install` and `nodeadm init`. This ensures a proper clean up with the changes in configuration. +If you changed your `nodeadm` configuration and attempt to reregister the node with the new configuration, you might see an error that states that the hybrid profile already exists but its contents have changed. Instead of running `nodeadm init` in between configuration changes, run `nodeadm uninstall` followed by a `nodeadm install` and `nodeadm init`. This ensures a proper clean up with the changes in configuration. [source,bash,subs="verbatim,attributes"] ---- @@ -173,7 +204,7 @@ If you changed your `nodeadm` configuration and attempt to re-register the node *Hybrid node failed to resolve Private API* -After running `nodeadm init`, if you see an error in the `kubelet` logs that shows failures to contact the EKS Kubernetes API server because there is `no such host`, you may have to change your DNS entry for the EKS Kubernetes API endpoint in your on-premises network or at the host level. See link:Route53/latest/DeveloperGuide/resolver-forwarding-inbound-queries.html[Forwarding inbound DNS queries to your VPC,type="documentation"] in the _{aws} Route53 documentation_. +After running `nodeadm init`, if you see an error in the `kubelet` logs that shows failures to contact the EKS Kubernetes API server because there is `no such host`, you might have to change your DNS entry for the EKS Kubernetes API endpoint in your on-premises network or at the host level. See link:Route53/latest/DeveloperGuide/resolver-forwarding-inbound-queries.html[Forwarding inbound DNS queries to your VPC,type="documentation"] in the _{aws} Route53 documentation_. [source,bash,subs="verbatim,attributes"] ---- @@ -186,7 +217,7 @@ If you have registered your hybrid nodes but are unable to view them in the EKS == Running hybrid nodes troubleshooting -If your hybrid nodes registered with your EKS cluster, had status `Ready`, and then transitioned to status `Not Ready`, there are a wide range of issues that may have contributed to the unhealthy status such as the node lacking sufficient resources for CPU, memory, or available disk space, or the node is disconnected from the EKS control plane. You can use the steps below to troubleshoot your nodes, and if you cannot resolve the issue, contact {aws} Support. +If your hybrid nodes registered with your EKS cluster, had status `Ready`, and then transitioned to status `Not Ready`, there are a wide range of issues that might have contributed to the unhealthy status such as the node lacking sufficient resources for CPU, memory, or available disk space, or the node is disconnected from the EKS control plane. You can use the steps below to troubleshoot your nodes, and if you cannot resolve the issue, contact {aws} Support. Run `nodeadm debug` from your hybrid nodes to validate networking and credential requirements are met. For more information on the `nodeadm debug` command, see <>. @@ -228,7 +259,7 @@ kubectl logs [.replaceable]`POD_NAME` ---- -*Check kubectl logs* +*Check `kubectl` logs* [source,bash,subs="verbatim,attributes"] ---- @@ -242,7 +273,7 @@ journalctl -u kubelet -f *Pod liveness probes failing or webhooks are not working* -If applications, add-ons, or webhooks running on your hybrid nodes are not starting properly, you may have networking issues that block the communication to the pods. For the EKS control plane to contact webhooks running on hybrid nodes, you must configure your EKS cluster with a remote pod network and have routes for your on-premises pod CIDR in your VPC routing table with the target as your Transit Gateway (TGW), virtual private gateway (VPW), or other gateway you are using to connect your VPC with your on-premises network. For more information on the networking requirements for hybrid nodes, see <>. You additionally must allow this traffic in your on-premises firewall and ensure your router can properly route to your pods. +If applications, add-ons, or webhooks running on your hybrid nodes are not starting properly, you might have networking issues that block the communication to the pods. For the EKS control plane to contact webhooks running on hybrid nodes, you must configure your EKS cluster with a remote pod network and have routes for your on-premises pod CIDR in your VPC routing table with the target as your Transit Gateway (TGW), virtual private gateway (VPW), or other gateway you are using to connect your VPC with your on-premises network. For more information on the networking requirements for hybrid nodes, see <>. You additionally must allow this traffic in your on-premises firewall and ensure your router can properly route to your pods. See <> for more information on the requirements for running webhooks on hybrid nodes. A common pod log message for this scenario is shown below the following where ip-address is the Cluster IP for the Kubernetes service. @@ -250,8 +281,25 @@ A common pod log message for this scenario is shown below the following where ip ---- dial tcp :443: connect: no route to host ---- +*`kubectl logs` or `kubectl exec` commands not working (`kubelet` API commands)* + +If `kubectl attach`, `kubectl cp`, `kubectl exec`, `kubectl logs`, and `kubectl port-forward` commands time out while other `kubectl` commands succeed, the issue is likely related to remote network configuration. These commands connect through the cluster to the `kubelet` endpoint on the node. For more information see <>. + +Verify that your node IPs and pod IPs fall within the remote node network and remote pod network CIDRs configured for your cluster. Use the commands below to examine IP assignments. + +[source,bash,subs="verbatim,attributes"] +---- +kubectl get nodes -o wide +---- -[[hybrid-nodes-troubleshooting-cni,hybrid-nodes-troubleshooting-cni.title]] +[source,bash,subs="verbatim,attributes"] +---- +kubectl get pods -A -o wide +---- + +Compare these IPs with your configured remote network CIDRs to ensure proper routing. For network configuration requirements, see <>. + +[#hybrid-nodes-troubleshooting-cni] == Hybrid nodes CNI troubleshooting If you run into issues with initially starting Cilium or Calico with hybrid nodes, it is most often due to networking issues between hybrid nodes or the CNI pods running on hybrid nodes, and the EKS control plane. Make sure your environment meets the requirements in Prepare networking for hybrid nodes. It's useful to break down the problem into parts. @@ -259,16 +307,16 @@ If you run into issues with initially starting Cilium or Calico with hybrid node EKS cluster configuration:: Are the RemoteNodeNetwork and RemotePodNetwork configurations correct? VPC configuration:: Are there routes for the RemoteNodeNetwork and RemotePodNetwork in the VPC routing table with the target of the Transit Gateway or Virtual Private Gateway? Security group configuration:: Are there inbound and outbound rules for the RemoteNodeNetwork and RemotePodNetwork ? -On-premises network:: Are there routes and access to/from the EKS control plane and to/from the hybrid nodes and the pods running on hybrid nodes? +On-premises network:: Are there routes and access to and from the EKS control plane and to and from the hybrid nodes and the pods running on hybrid nodes? CNI configuration:: If using an overlay network, does the IP pool configuration for the CNI match the RemotePodNetwork configured for the EKS cluster if using webhooks? *Hybrid node has status `Ready` without a CNI installed* -If your hybrid nodes are showing status `Ready`, but you have not installed a CNI on your cluster, it is possible that there are old CNI artifacts on your hybrid nodes. By default, when you uninstall Cilium and Calico with tools such as Helm, the on-disk resources are not removed from your physical or virtual machines. Additionally, the Custom Resource Definitions (CRDs) for these CNIs may still be present on your cluster from an old installation. For more information, see the Delete Cilium and Delete Calico sections of <>. +If your hybrid nodes are showing status `Ready`, but you have not installed a CNI on your cluster, it is possible that there are old CNI artifacts on your hybrid nodes. By default, when you uninstall Cilium and Calico with tools such as Helm, the on-disk resources are not removed from your physical or virtual machines. Additionally, the Custom Resource Definitions (CRDs) for these CNIs might still be present on your cluster from an old installation. For more information, see the Delete Cilium and Delete Calico sections of <>. *Cilium troubleshooting* -If you are having issues running Cilium on hybrid nodes, see https://docs.cilium.io/en/stable/operations/troubleshooting/[the troubleshooting steps] in the Cilium documentation. The sections below cover issues that may be unique to deploying Cilium on hybrid nodes. +If you are having issues running Cilium on hybrid nodes, see https://docs.cilium.io/en/stable/operations/troubleshooting/[the troubleshooting steps] in the Cilium documentation. The sections below cover issues that might be unique to deploying Cilium on hybrid nodes. *Cilium isn't starting* @@ -280,7 +328,7 @@ msg="Unable to contact k8s api-server" level=fatal msg="failed to start: Get \"https://:443/api/v1/namespaces/kube-system\": dial tcp :443: i/o timeout" ---- -The Cilium agent runs on the host network. Your EKS cluster must be configured with `RemoteNodeNetwork` for the Cilium connectivity. Confirm you have an additional security group for your EKS cluster with an inbound rule for your `RemoteNodeNetwork`, that you have routes in your VPC for your `RemodeNodeNetwork`, and that your on-premises network is configured correctly to allow connectivity to the EKS control plane. +The Cilium agent runs on the host network. Your EKS cluster must be configured with `RemoteNodeNetwork` for the Cilium connectivity. Confirm you have an additional security group for your EKS cluster with an inbound rule for your `RemoteNodeNetwork`, that you have routes in your VPC for your `RemoteNodeNetwork`, and that your on-premises network is configured correctly to allow connectivity to the EKS control plane. If the Cilium operator is running and some of your Cilium agents are running but not all, confirm that you have available pod IPs to allocate for all nodes in your cluster. You configure the size of your allocatable Pod CIDRs when using cluster pool IPAM with `clusterPoolIPv4PodCIDRList` in your Cilium configuration. The per-node CIDR size is configured with the `clusterPoolIPv4MaskSize` setting in your Cilium configuration. See https://docs.cilium.io/en/stable/network/concepts/ipam/cluster-pool/#expanding-the-cluster-pool[Expanding the cluster pool] in the Cilium documentation for more information. @@ -288,7 +336,7 @@ If the Cilium operator is running and some of your Cilium agents are running but If you are using Cilium BGP Control Plane to advertise your pod or service addresses to your on-premises network, you can use the following Cilium CLI commands to check if BGP is advertising the routes to your resources. For steps to install the Cilium CLI, see https://docs.cilium.io/en/stable/gettingstarted/k8s-install-default/#install-the-cilium-cli[Install the Cilium CLI] in the Cilium documentation. -If BGP is working correctly, you should your hybrid nodes with Session State `established` in the output. You may need to work with your networking team to identify the correct values for your environment's Local AS, Peer AS, and Peer Address. +If BGP is working correctly, you should your hybrid nodes with Session State `established` in the output. You might need to work with your networking team to identify the correct values for your environment's Local AS, Peer AS, and Peer Address. [source,bash,subs="verbatim,attributes"] ---- @@ -300,7 +348,7 @@ cilium bgp peers cilium bgp routes ---- -If you are using Cilium BGP to advertise the IPs of Services with type `LoadBalancer`, you must have the same label on both your `CiliumLoadBalancerIPPool` and Service, which should be used in the selector of your `CiliumBGPAdvertisement`. An example is shown below. Note, if you are using Cilium BGP to advertise the IPs of Services with type LoadBalancer, the BGP routes may be disrupted during Cilium agent restart. For more information, see https://docs.cilium.io/en/latest/network/bgp-control-plane/bgp-control-plane-operation/#failure-scenarios[Failure Scenarios] in the Cilium documentation. +If you are using Cilium BGP to advertise the IPs of Services with type `LoadBalancer`, you must have the same label on both your `CiliumLoadBalancerIPPool` and Service, which should be used in the selector of your `CiliumBGPAdvertisement`. An example is shown below. Note, if you are using Cilium BGP to advertise the IPs of Services with type LoadBalancer, the BGP routes might be disrupted during Cilium agent restart. For more information, see https://docs.cilium.io/en/latest/network/bgp-control-plane/bgp-control-plane-operation/#failure-scenarios[Failure Scenarios] in the Cilium documentation. *Service* @@ -363,11 +411,11 @@ spec: *Calico troubleshooting* -If you are having issues running Calico on hybrid nodes, see https://docs.tigera.io/calico/latest/operations/troubleshoot/[the troubleshooting steps] in the Calico documentation. The sections below cover issues that may be unique to deploying Calico on hybrid nodes. +If you are having issues running Calico on hybrid nodes, see https://docs.tigera.io/calico/latest/operations/troubleshoot/[the troubleshooting steps] in the Calico documentation. The sections below cover issues that might be unique to deploying Calico on hybrid nodes. The table below summarizes the Calico components and whether they run on the node or pod network by default. If you configured Calico to use NAT for outgoing pod traffic, your on-premises network must be configured to route traffic to your on-premises node CIDR and your VPC routing tables must be configured with a route for your on-premises node CIDR with your transit gateway (TGW) or virtual private gateway (VGW) as the target. If you are not configuring Calico to use NAT for outgoing pod traffic, your on-premises network must be configured to route traffic to your on-premises pod CIDR and your VPC routing tables must be configured with a route for your on-premises pod CIDR with your transit gateway (TGW) or virtual private gateway (VGW) as the target. -[cols="1,1", options="header"] +[%header,cols="2"] |=== |Component |Network @@ -375,13 +423,13 @@ The table below summarizes the Calico components and whether they run on the nod |Calico API server |Node -|Calico kube controllers +|Calico Controllers for Kubernetes |Pod |Calico node agent |Node -|Calico typha +|Calico `typha` |Node |Calico CSI node driver @@ -436,7 +484,7 @@ installation: tolerationSeconds: 300 ---- -[[hybrid-nodes-troubleshooting-creds,hybrid-nodes-troubleshooting-creds.title]] +[#hybrid-nodes-troubleshooting-creds] == Credentials troubleshooting For both {aws} SSM hybrid activations and {aws} IAM Roles Anywhere, you can validate that credentials for the Hybrid Nodes IAM role are correctly configured on your hybrid nodes by running the following command from your hybrid nodes. Confirm the node name and Hybrid Nodes IAM Role name are what you expect. @@ -451,15 +499,15 @@ sudo aws sts get-caller-identity { "UserId": "ABCDEFGHIJKLM12345678910:", "Account": "", - "Arn": "arn:aws:sts:::assumed-role/" + "Arn": "{arn-aws}sts:::assumed-role/" } ---- *{aws} Systems Manager (SSM) troubleshooting* -If you are using {aws} SSM hybrid activations for your hybrid nodes credentials, be aware of the following SSM directories and artifacts that are installed on your hybrid nodes by nodeadm. For more information on the SSM agent, see link:systems-manager/latest/userguide/ssm-agent.html[Working with the SSM agent,type="documentation"] in the _{aws} Systems Manager User Guide_. +If you are using {aws} SSM hybrid activations for your hybrid nodes credentials, be aware of the following SSM directories and artifacts that are installed on your hybrid nodes by `nodeadm`. For more information on the SSM agent, see link:systems-manager/latest/userguide/ssm-agent.html[Working with the SSM agent,type="documentation"] in the _{aws} Systems Manager User Guide_. -[cols="1,1", options="header"] +[%header,cols="2"] |=== |Description |Location @@ -498,7 +546,7 @@ systemctl restart snap.amazon-ssm-agent.amazon-ssm-agent *Check connectivity to SSM endpoints* -Confirm you can connect to the SSM endpoints from your hybrid nodes. For a list of the SSM endpoints, see link:general/latest/gr/ssm.html[{aws} Systems Manager endpoints and quotas,type="documentation"]. Replace `us-west-2` in the command below with the {aws-region} for your {aws} SSM hybrid activation. +Confirm you can connect to the SSM endpoints from your hybrid nodes. For a list of the SSM endpoints, see link:general/latest/gr/ssm.html[{aws} Systems Manager endpoints and quotas,type="documentation"]. Replace `us-west-2` in the command below with the {aws} Region for your {aws} SSM hybrid activation. [source,bash,subs="verbatim,attributes"] ---- @@ -507,7 +555,7 @@ ping ssm.us-west-2.amazonaws.com *View connection status of registered SSM instances* -You can check the connection status of the instances that are registered with SSM hybrid activations with the following {cli} command. Replace the machine ID with the machine ID of your instance. +You can check the connection status of the instances that are registered with SSM hybrid activations with the following {aws} CLI command. Replace the machine ID with the machine ID of your instance. [source,bash,subs="verbatim,attributes,quotes"] ---- @@ -526,7 +574,7 @@ Failed to perform agent-installation/on-prem registration: error while verifying *SSM `InvalidActivation`* -If you see an error registering your instance with {aws} SSM, confirm the `region`, `activationCode`, and `activationId` in your `nodeConfig.yaml` are correct. The {aws-region} for your EKS cluster must match the region of your SSM hybrid activation. If these values are misconfigured, you may see an error similar to the following. +If you see an error registering your instance with {aws} SSM, confirm the `region`, `activationCode`, and `activationId` in your `nodeConfig.yaml` are correct. The {aws} Region for your EKS cluster must match the region of your SSM hybrid activation. If these values are misconfigured, you might see an error similar to the following. [source,bash,subs="verbatim,attributes"] ---- @@ -535,7 +583,7 @@ ERROR Registration failed due to error registering the instance with {aws} SSM. *SSM `ExpiredTokenException`: The security token included in the request is expired* -If the SSM agent is not able to refresh credentials, you may see an `ExpiredTokenException`. In this scenario, if you are able to connect to the SSM endpoints from your hybrid nodes, you may need to restart the SSM agent to force a credential refresh. +If the SSM agent is not able to refresh credentials, you might see an `ExpiredTokenException`. In this scenario, if you are able to connect to the SSM endpoints from your hybrid nodes, you might need to restart the SSM agent to force a credential refresh. [source,bash,subs="verbatim,attributes"] ---- @@ -544,7 +592,7 @@ If the SSM agent is not able to refresh credentials, you may see an `ExpiredToke *SSM error in running register machine command* -If you see an error registering the machine with SSM, you may need to re-run `nodeadm install` to make sure all of the SSM dependencies are properly installed. +If you see an error registering the machine with SSM, you might need to re-run `nodeadm install` to make sure all of the SSM dependencies are properly installed. [source,bash,subs="verbatim,attributes"] ---- @@ -568,7 +616,7 @@ ERROR Registration failed due to error registering the instance with {aws} SSM. *SSM failed to refresh cached credentials* -If you see a failure to refresh cached credentials, the `/root/.aws/credentials` file may have been deleted on your host. First check your SSM hybrid activation and ensure it is active and your hybrid nodes are configured correctly to use the activation. Check the SSM agent logs at `/var/log/amazon/ssm` and re-run the nodeadm init command once you have resolved the issue on the SSM side. +If you see a failure to refresh cached credentials, the `/root/.aws/credentials` file might have been deleted on your host. First check your SSM hybrid activation and ensure it is active and your hybrid nodes are configured correctly to use the activation. Check the SSM agent logs at `/var/log/amazon/ssm` and re-run the `nodeadm init` command once you have resolved the issue on the SSM side. [source,bash,subs="verbatim,attributes"] ---- @@ -589,9 +637,9 @@ rm -rf /var/lib/amazon/ssm/Vault/Store/RegistrationKey *{aws} IAM Roles Anywhere troubleshooting* -If you are using {aws} IAM Roles Anywhere for your hybrid nodes credentials, be aware of the following directories and artifacts that are installed on your hybrid nodes by nodeadm. For more information on the troubleshooting IAM Roles Anywhere, see link:rolesanywhere/latest/userguide/security_iam_troubleshoot.html[Troubleshooting {aws} IAM Roles Anywhere identity and access,type="documentation"] in the _{aws} IAM Roles Anywhere User Guide_. +If you are using {aws} IAM Roles Anywhere for your hybrid nodes credentials, be aware of the following directories and artifacts that are installed on your hybrid nodes by `nodeadm`. For more information on the troubleshooting IAM Roles Anywhere, see link:rolesanywhere/latest/userguide/security_iam_troubleshoot.html[Troubleshooting {aws} IAM Roles Anywhere identity and access,type="documentation"] in the _{aws} IAM Roles Anywhere User Guide_. -[cols="1,1", options="header"] +[%header,cols="2"] |=== |Description |Location @@ -608,7 +656,7 @@ If you are using {aws} IAM Roles Anywhere for your hybrid nodes credentials, be *IAM Roles Anywhere failed to refresh cached credentials* -If you see a failure to refresh cached credentials, review the contents of `/etc/aws/hybrid/config` and confirm that IAM Roles Anywhere was configured correctly in your `nodeadm` configuration. Confirm that `/etc/iam/pki` exists. Each node must have a unique certificate and key. By default, when using IAM Roles Anywhere as the credential provider, `nodeadm` uses `/etc/iam/pki/server.pem` for the certificate location and name, and `/etc/iam/pki/server.key` for the private key. You may need to create the directories before placing the certificates and keys in the directories with `sudo mkdir -p /etc/iam/pki`. You can verify the content of your certificate with the command below. +If you see a failure to refresh cached credentials, review the contents of `/etc/aws/hybrid/config` and confirm that IAM Roles Anywhere was configured correctly in your `nodeadm` configuration. Confirm that `/etc/iam/pki` exists. Each node must have a unique certificate and key. By default, when using IAM Roles Anywhere as the credential provider, `nodeadm` uses `/etc/iam/pki/server.pem` for the certificate location and name, and `/etc/iam/pki/server.key` for the private key. You might need to create the directories before placing the certificates and keys in the directories with `sudo mkdir -p /etc/iam/pki`. You can verify the content of your certificate with the command below. [source,bash,subs="verbatim,attributes"] ---- @@ -631,7 +679,7 @@ In the `kubelet` logs, if you see an access denied issue for the `sts:AssumeRole could not get token: AccessDenied: User: ... is not authorized to perform: sts:AssumeRole on resource: ... ---- -*IAM Roles Anywhere not authorized to set roleSessionName* +*IAM Roles Anywhere not authorized to set `roleSessionName`* In the `kubelet` logs, if you see an access denied issue for setting the `roleSessionName`, confirm you have set `acceptRoleSessionName` to true for your IAM Roles Anywhere profile. @@ -640,10 +688,10 @@ In the `kubelet` logs, if you see an access denied issue for setting the `roleSe AccessDeniedException: Not authorized to set roleSessionName ---- -[[hybrid-nodes-troubleshooting-os,hybrid-nodes-troubleshooting-os.title]] +[#hybrid-nodes-troubleshooting-os] == Operating system troubleshooting -*RHEL* +=== RHEL *Entitlement or subscription manager registration failures* @@ -654,6 +702,8 @@ If you are running `nodeadm install` and encounter a failure to install the hybr This system is not registered with an entitlement server ---- +=== Ubuntu + *GLIBC not found* If you are using Ubuntu for your operating system and IAM Roles Anywhere for your credential provider with hybrid nodes and see an issue with GLIBC not found, you can install that dependency manually to resolve the issue. @@ -671,3 +721,65 @@ ldd --version sudo apt update && apt install libc6 sudo apt install glibc-source ---- + +=== Bottlerocket + +If you have the Bottlerocket admin container enabled, you can access it with SSH for advanced debugging and troubleshooting with elevated privileges. The following sections contain commands that need to be run on the context of the Bottlerocket host. Once you are on the admin container, you can run `sheltie` to get a full root shell in the Bottlerocket host. + +[source,bash,subs="verbatim,attributes"] +---- +sheltie +---- + +You can also run the commands in the following sections from the admin container shell by prefixing each command with `sudo chroot /.bottlerocket/rootfs`. + +[source,bash,subs="verbatim,attributes"] +---- +sudo chroot /.bottlerocket/rootfs +---- + +*Using logdog for log collection* + +Bottlerocket provides the `logdog` utility to efficiently collect logs and system information for troubleshooting purposes. + +[source,bash,subs="verbatim,attributes"] +---- +logdog +---- + +The `logdog` utility gathers logs from various locations on a Bottlerocket host and combines them into a tarball. By default, the tarball will be created at `/var/log/support/bottlerocket-logs.tar.gz`, and is accessible from host containers at `/.bottlerocket/support/bottlerocket-logs.tar.gz`. + +*Accessing system logs with journalctl* + +You can check the status of the various system services such as `kubelet`, `containerd`, etc and view their logs with the following commands. The `-f` flag will follow the logs in real time. + +For checking `kubelet` service status and retrieving `kubelet` logs, you can run: + +[source,bash,subs="verbatim,attributes"] +---- +systemctl status kubelet +journalctl -u kubelet -f +---- + +For checking `containerd` service status and retrieving the logs for the orchestrated `containerd` instance, you can run: + +[source,bash,subs="verbatim,attributes"] +---- +systemctl status containerd +journalctl -u containerd -f +---- + +For checking `host-containerd` service status and retrieving the logs for the host `containerd` instance, you can run: + +[source,bash,subs="verbatim,attributes"] +---- +systemctl status host-containerd +journalctl -u host-containerd -f +---- + +For retrieving the logs for the bootstrap containers and host containers, you can run: + +[source,bash,subs="verbatim,attributes"] +---- +journalctl _COMM=host-ctr -f +---- diff --git a/latest/ug/nodes/hybrid-nodes-tutorial.adoc b/latest/ug/nodes/hybrid-nodes-tutorial.adoc index dfb1817ad..5edf43791 100644 --- a/latest/ug/nodes/hybrid-nodes-tutorial.adoc +++ b/latest/ug/nodes/hybrid-nodes-tutorial.adoc @@ -1,18 +1,13 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[hybrid-nodes-tutorial,hybrid-nodes-tutorial.title]] +[#hybrid-nodes-tutorial] = Run on-premises workloads on hybrid nodes -:info_doctype: section -:info_title: Run and manage hybrid nodes :info_titleabbrev: Run hybrid nodes -:keywords: on-premises, hybrid -:info_abstract: Amazon EKS hybrid nodes - -include::../attributes.txt[] [abstract] -- -Join nodes from your data centers to Amazon EKS [.noloc]`Kubernetes` clusters with Amazon EKS Hybrid Nodes. +Join nodes from your data centers to Amazon EKS Kubernetes clusters with Amazon EKS Hybrid Nodes. -- In an EKS cluster with hybrid nodes enabled, you can run on-premises and edge applications on your own infrastructure with the same Amazon EKS clusters, features, and tools that you use in {aws} Cloud. @@ -25,6 +20,10 @@ The following sections contain step-by-step instructions for using hybrid nodes. include::hybrid-nodes-join.adoc[leveloffset=+1] +include::hybrid-nodes-bottlerocket.adoc[leveloffset=+1] + include::hybrid-nodes-upgrade.adoc[leveloffset=+1] -include::hybrid-nodes-remove.adoc[leveloffset=+1] +include::hybrid-nodes-security.adoc[leveloffset=+1] + +include::hybrid-nodes-remove.adoc[leveloffset=+1] \ No newline at end of file diff --git a/latest/ug/nodes/hybrid-nodes-upgrade.adoc b/latest/ug/nodes/hybrid-nodes-upgrade.adoc index e856544a4..eb2488464 100644 --- a/latest/ug/nodes/hybrid-nodes-upgrade.adoc +++ b/latest/ug/nodes/hybrid-nodes-upgrade.adoc @@ -1,21 +1,16 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[hybrid-nodes-upgrade,hybrid-nodes-upgrade.title]] +[#hybrid-nodes-upgrade] = Upgrade hybrid nodes for your cluster -:info_doctype: section -:info_title: Upgrade hybrid nodes for your cluster :info_titleabbrev: Upgrade hybrid nodes -:keywords: upgrade on-premises nodes, upgrade hybrid nodes -:info_abstract: Upgrade Kubernetes versions on hybrid nodes - -include::../attributes.txt[] [abstract] -- Upgrade Kubernetes versions on hybrid nodes -- -The guidance for upgrading hybrid nodes is similar to self-managed Amazon EKS nodes that run in Amazon EC2. It is recommended to can create new hybrid nodes on your target Kubernetes version, gracefully migrate your existing applications to the hybrid nodes on the new Kubernetes version, and remove the hybrid nodes on the old Kubernetes version from your cluster. Be sure to review the link:eks/latest/best-practices/cluster-upgrades.html[Amazon EKS Best Practices,type="documentation"] for upgrades before initiating an upgrade. Amazon EKS Hybrid Nodes have the same <> for Amazon EKS clusters with cloud nodes, including standard and extended support. +The guidance for upgrading hybrid nodes is similar to self-managed Amazon EKS nodes that run in Amazon EC2. We recommend that you create new hybrid nodes on your target Kubernetes version, gracefully migrate your existing applications to the hybrid nodes on the new Kubernetes version, and remove the hybrid nodes on the old Kubernetes version from your cluster. Be sure to review the link:eks/latest/best-practices/cluster-upgrades.html[Amazon EKS Best Practices,type="documentation"] for upgrades before initiating an upgrade. Amazon EKS Hybrid Nodes have the same link:eks/latest/userguide/kubernetes-versions.html[Kubernetes version support,type="documentation"] for Amazon EKS clusters with cloud nodes, including standard and extended support. Amazon EKS Hybrid Nodes follow the same https://kubernetes.io/releases/version-skew-policy/#supported-version-skew[version skew policy] for nodes as upstream Kubernetes. Amazon EKS Hybrid Nodes cannot be on a newer version than the Amazon EKS control plane, and hybrid nodes may be up to three Kubernetes minor versions older than the Amazon EKS control plane minor version. @@ -36,33 +31,33 @@ Before upgrading, make sure you have completed the following prerequisites. * You must have kubectl installed on your local machine or instance you are using to interact with your Amazon EKS Kubernetes API endpoint. * The version of your CNI must support the Kubernetes version you are upgrading to. If it does not, upgrade your CNI version before upgrading your hybrid nodes. See <> for more information. -== Cutover migration upgrades +[#hybrid-nodes-upgrade-cutover] +== Cutover migration (blue-green) upgrades -Cutover migration upgrades refer to the process of creating new hybrid nodes on new hosts with your target Kubernetes version, gracefully migrating your existing applications to the new hybrid nodes on your target Kubernetes version, and removing the hybrid nodes on the old Kubernetes version from your cluster. +_Cutover migration upgrades_ refer to the process of creating new hybrid nodes on new hosts with your target Kubernetes version, gracefully migrating your existing applications to the new hybrid nodes on your target Kubernetes version, and removing the hybrid nodes on the old Kubernetes version from your cluster. This strategy is also called a blue-green migration. . Connect your new hosts as hybrid nodes following the <> steps. When running the `nodeadm install` command, use your target Kubernetes version. . Enable communication between the new hybrid nodes on the target Kubernetes version and your hybrid nodes on the old Kubernetes version. This configuration allows pods to communicate with each other while you are migrating your workload to the hybrid nodes on the target Kubernetes version. . Confirm your hybrid nodes on your target Kubernetes version successfully joined your cluster and have status Ready. -. Use the following command to taint each of the nodes that you want to remove with `NoSchedule`. This is so that new pods aren't scheduled or rescheduled on the nodes that you are replacing. For more information, see https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/[Taints and Tolerations] in the Kubernetes documentation. Replace `NODE_NAME` with the name of the hybrid nodes on the old Kubernetes version. +. Use the following command to mark each of the nodes that you want to remove as unschedulable. This is so that new pods aren't scheduled or rescheduled on the nodes that you are replacing. For more information, see https://kubernetes.io/docs/reference/kubectl/generated/kubectl_cordon/[kubectl cordon] in the Kubernetes documentation. Replace `NODE_NAME` with the name of the hybrid nodes on the old Kubernetes version. + [source,yaml,subs="verbatim,attributes,quotes"] ---- -kubectl taint nodes [.replaceable]`NODE_NAME` key=value:NoSchedule +kubectl cordon [.replaceable]`NODE_NAME` ---- + -You can identify and taint all of the nodes of a particular Kubernetes version (in this case, `1.28`) with the following code snippet. +You can identify and cordon all of the nodes of a particular Kubernetes version (in this case, `1.28`) with the following code snippet. + [source,yaml,subs="verbatim,attributes"] ---- K8S_VERSION=1.28 -nodes=$(kubectl get nodes -o jsonpath="{.items[?(@.status.nodeInfo.kubeletVersion==\"v$K8S_VERSION\")].metadata.name}") -for node in ${nodes[@]} +for node in $(kubectl get nodes -o json | jq --arg K8S_VERSION "$K8S_VERSION" -r '.items[] | select(.status.nodeInfo.kubeletVersion | match("\($K8S_VERSION)")).metadata.name') do - echo "Tainting $node" - kubectl taint nodes $node key=value:NoSchedule + echo "Cordoning $node" + kubectl cordon $node done ---- -. If your current deployment is running fewer than two CoreDNS replicas on your hybrid nodes, scale out the deployment to at least two replicas. It is recommended to run at least two CoreDNS replicas on hybrid nodes for resiliency during normal operations. +. If your current deployment is running fewer than two CoreDNS replicas on your hybrid nodes, scale out the deployment to at least two replicas. We recommend that you run at least two CoreDNS replicas on hybrid nodes for resiliency during normal operations. + [source,yaml,subs="verbatim,attributes"] ---- @@ -80,8 +75,7 @@ You can identify and drain all of the nodes of a particular Kubernetes version ( [source,yaml,subs="verbatim,attributes"] ---- K8S_VERSION=1.28 -nodes=$(kubectl get nodes -o jsonpath="{.items[?(@.status.nodeInfo.kubeletVersion==\"v$K8S_VERSION\")].metadata.name}") -for node in ${nodes[@]} +for node in $(kubectl get nodes -o json | jq --arg K8S_VERSION "$K8S_VERSION" -r '.items[] | select(.status.nodeInfo.kubeletVersion | match("\($K8S_VERSION)")).metadata.name') do echo "Draining $node" kubectl drain $node --ignore-daemonsets --delete-emptydir-data @@ -107,44 +101,50 @@ You can identify and delete all of the nodes of a particular Kubernetes version [source,yaml,subs="verbatim,attributes"] ---- K8S_VERSION=1.28 -nodes=$(kubectl get nodes -o jsonpath="{.items[?(@.status.nodeInfo.kubeletVersion==\"v$K8S_VERSION\")].metadata.name}") -for node in ${nodes[@]} +for node in $(kubectl get nodes -o json | jq --arg K8S_VERSION "$K8S_VERSION" -r '.items[] | select(.status.nodeInfo.kubeletVersion | match("\($K8S_VERSION)")).metadata.name') do echo "Deleting $node" - kubectl delete node $node + kubectl delete node $node done ---- . Depending on your choice of CNI, there may be artifacts remaining on your hybrid nodes after running the above steps. See <> for more information. +[#hybrid-nodes-upgrade-inplace] == In-place upgrades The in-place upgrade process refers to using `nodeadm upgrade` to upgrade the Kubernetes version for hybrid nodes without using new physical or virtual hosts and a cutover migration strategy. The `nodeadm upgrade` process shuts down the existing older Kubernetes components running on the hybrid node, uninstalls the existing older Kubernetes components, installs the new target Kubernetes components, and starts the new target Kubernetes components. It is strongly recommend to upgrade one node at a time to minimize impact to applications running on the hybrid nodes. The duration of this process depends on your network bandwidth and latency. -. Use the following command to taint the node you are upgrading with `NoSchedule`. This is so that new pods aren't scheduled or rescheduled on the node that you are upgrading. For more information, see https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/[Taints and Tolerations] in the Kubernetes documentation. Replace `NODE_NAME` with the name of the hybrid node you are upgrading +. Use the following command to mark the node you are upgrading as unschedulable. This is so that new pods aren't scheduled or rescheduled on the node that you are upgrading. For more information, see https://kubernetes.io/docs/reference/kubectl/generated/kubectl_cordon/[kubectl cordon] in the Kubernetes documentation. Replace `NODE_NAME` with the name of the hybrid node you are upgrading + -[source,yaml,subs="verbatim,attributes,quotes"] +[source,yaml,subs="verbatim,attributes"] ---- -kubectl taint nodes NODE_NAME key=value:NoSchedule +kubectl cordon NODE_NAME ---- . Drain the node you are upgrading with the following command. For more information on draining nodes, see https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/[Safely Drain a Node] in the Kubernetes documentation. Replace `NODE_NAME` with the name of the hybrid node you are upgrading. + -[source,yaml,subs="verbatim,attributes,quotes"] +[source,yaml,subs="verbatim,attributes"] ---- kubectl drain NODE_NAME --ignore-daemonsets --delete-emptydir-data ---- . Run `nodeadm upgrade` on the hybrid node you are upgrading. You must run `nodeadm` with a user that has root/sudo privileges. The name of the node is preserved through upgrade for both {aws} SSM and {aws} IAM Roles Anywhere credential providers. You cannot change credentials providers during the upgrade process. See <> for configuration values for `nodeConfig.yaml`. Replace `K8S_VERSION` with the target Kubernetes version you upgrading to. + -[source,yaml,subs="verbatim,attributes,quotes"] +[source,yaml,subs="verbatim,attributes"] ---- nodeadm upgrade K8S_VERSION -c file://nodeConfig.yaml ---- +. To allow pods to be scheduled on the node after you have upgraded, type the following. Replace `NODE_NAME` with the name of the node. ++ +[source,yaml,subs="verbatim,attributes,quotes"] +---- +kubectl uncordon NODE_NAME +---- . Watch the status of your hybrid nodes and wait for your nodes to shutdown and restart on the new Kubernetes version with the Ready status. + -[source,yaml,subs="verbatim,attributes,quotes"] +[source,yaml,subs="verbatim,attributes"] ---- -kubectl get nodes -o -w +kubectl get nodes -o wide -w ---- diff --git a/latest/ug/nodes/hybrid-nodes-webhooks.adoc b/latest/ug/nodes/hybrid-nodes-webhooks.adoc new file mode 100644 index 000000000..2e01a1bc8 --- /dev/null +++ b/latest/ug/nodes/hybrid-nodes-webhooks.adoc @@ -0,0 +1,357 @@ +[.topic] +[#hybrid-nodes-webhooks] += Configure webhooks for hybrid nodes +:info_titleabbrev: Configure webhooks +:keywords: hybrid nodes webhooks, hybrid nodes webhook configuration + +include::../attributes.txt[] + +[abstract] +-- +Configure webhooks for hybrid nodes +-- + +This page details considerations for running webhooks with hybrid nodes. Webhooks are used in Kubernetes applications and open source projects, such as the {aws} Load Balancer Controller and CloudWatch Observability Agent, to perform mutating and validation capabilities at runtime. + +*Routable pod networks* + +If you are able to make your on-premises pod CIDR routable on your on-premises network, you can run webhooks on hybrid nodes. There are several techniques you can use to make your on-premises pod CIDR routable on your on-premises network including Border Gateway Protocol (BGP), static routes, or other custom routing solutions. BGP is the recommended solution as it is more scalable and easier to manage than alternative solutions that require custom or manual route configuration. {aws} supports the BGP capabilities of Cilium and Calico for advertising pod CIDRs, see <> and <> for more information. + +*Unroutable pod networks* + +If you _cannot_ make your on-premises pod CIDR routable on your on-premises network and need to run webhooks, we recommend that you run all webhooks on cloud nodes in the same EKS cluster as your hybrid nodes. + +[#hybrid-nodes-considerations-mixed-mode] +== Considerations for mixed mode clusters + +_Mixed mode clusters_ are defined as EKS clusters that have both hybrid nodes and nodes running in {aws} Cloud. When running a mixed mode cluster, consider the following recommendations: + +- Run the VPC CNI on nodes in {aws} Cloud and either Cilium or Calico on hybrid nodes. Cilium and Calico are not supported by {aws} when running on nodes in {aws} Cloud. +- Configure webhooks to run on nodes in {aws} Cloud. See <> for how to configure the webhooks for {aws} and community add-ons. +- If your applications require pods running on nodes in {aws} Cloud to directly communicate with pods running on hybrid nodes ("east-west communication"), and you are using the VPC CNI on nodes in {aws} Cloud, and Cilium or Calico on hybrid nodes, then your on-premises pod CIDR must be routable on your on-premises network. +- Run at least one replica of CoreDNS on nodes in {aws} Cloud and at least one replica of CoreDNS on hybrid nodes. +- Configure Service Traffic Distribution to keep Service traffic local to the zone it is originating from. For more information on Service Traffic Distribution, see <>. +- If you are using {aws} Application Load Balancers (ALB) or Network Load Balancers (NLB) for workload traffic running on hybrid nodes, then the IP target(s) used with the ALB or NLB must be routable from {aws}. +- The Metrics Server add-on requires connectivity from the EKS control plane to the Metrics Server pod IP address. If you are running the Metrics Server add-on on hybrid nodes, then your on-premises pod CIDR must be routable on your on-premises network. +- To collect metrics for hybrid nodes using Amazon Managed Service for Prometheus (AMP) managed collectors, your on-premises pod CIDR must be routable on your on-premises network. Or, you can use the AMP managed collector for EKS control plane metrics and resources running in {aws} Cloud, and the {aws} Distro for OpenTelemetry (ADOT) add-on to collect metrics for hybrid nodes. + +[#hybrid-nodes-mixed-mode] +== Configure mixed mode clusters + +To view the mutating and validating webhooks running on your cluster, you can view the *Extensions* resource type in the *Resources* panel of the EKS console for your cluster, or you can use the following commands. EKS also reports webhook metrics in the cluster observability dashboard, see <> for more information. + +[source,bash,subs="verbatim,attributes"] +---- +kubectl get mutatingwebhookconfigurations +---- + +[source,bash,subs="verbatim,attributes"] +---- +kubectl get validatingwebhookconfigurations +---- + +[#hybrid-nodes-mixed-service-traffic-distribution] +=== Configure Service Traffic Distribution + +When running mixed mode clusters, we recommend that you use link:https://kubernetes.io/docs/reference/networking/virtual-ips/#traffic-distribution[_Service Traffic Distribution_] to keep Service traffic local to the zone it is originating from. Service Traffic Distribution (available for Kubernetes versions 1.31 and later in EKS) is the recommended solution over link:https://kubernetes.io/docs/concepts/services-networking/topology-aware-routing/[Topology Aware Routing] because it is more predictable. With Service Traffic Distribution, healthy endpoints in the zone will receive all of the traffic for that zone. With Topology Aware Routing, each service must meet several conditions in that zone to apply the custom routing, otherwise it routes traffic evenly to all endpoints. + +If you are using Cilium as your CNI, you must run the CNI with the `enable-service-topology` set to `true` to enable Service Traffic Distribution. You can pass this configuration with the Helm install flag `--set loadBalancer.serviceTopology=true` or you can update an existing installation with the Cilium CLI command `cilium config set enable-service-topology true`. The Cilium agent running on each node must be restarted after updating the configuration for an existing installation. + +An example of how to configure Service Traffic Distribution for the CoreDNS Service is shown in the following section, and we recommend that you enable the same for all Services in your cluster to avoid unintended cross-environment traffic. + +[#hybrid-nodes-mixed-coredns] +=== Configure CoreDNS replicas + +If you are running a mixed mode cluster with both hybrid nodes and nodes in {aws} Cloud, we recommend that you have at least one CoreDNS replica on hybrid nodes and at least one CoreDNS replica on your nodes in {aws} Cloud. To prevent latency and network issues in a mixed mode cluster setup, you can configure the CoreDNS Service to prefer the closest CoreDNS replica with link:https://kubernetes.io/docs/reference/networking/virtual-ips/#traffic-distribution[Service Traffic Distribution]. + +_Service Traffic Distribution_ (available for Kubernetes versions 1.31 and later in EKS) is the recommended solution over link:https://kubernetes.io/docs/concepts/services-networking/topology-aware-routing/[Topology Aware Routing] because it is more predictable. In Service Traffic Distribution, healthy endpoints in the zone will receive all of the traffic for that zone. In Topology Aware Routing, each service must meet several conditions in that zone to apply the custom routing, otherwise it routes traffic evenly to all endpoints. The following steps configure Service Traffic Distribution. + +If you are using Cilium as your CNI, you must run the CNI with the `enable-service-topology` set to `true` to enable Service Traffic Distribution. You can pass this configuration with the Helm install flag `--set loadBalancer.serviceTopology=true` or you can update an existing installation with the Cilium CLI command `cilium config set enable-service-topology true`. The Cilium agent running on each node must be restarted after updating the configuration for an existing installation. + +. Add a topology zone label for each of your hybrid nodes, for example `topology.kubernetes.io/zone: onprem`. Or, you can set the label at the `nodeadm init` phase by specifying the label in your `nodeadm` configuration, see <>. Note, nodes running in {aws} Cloud automatically get a topology zone label applied to them that corresponds to the availability zone (AZ) of the node. ++ +[source,bash,subs="verbatim,attributes,quotes"] +---- +kubectl label node [.replaceable]`hybrid-node-name` topology.kubernetes.io/zone=[.replaceable]`zone` +---- ++ +. Add `podAntiAffinity` to the CoreDNS deployment with the topology zone key. Or, you can configure the CoreDNS deployment during installation with EKS add-ons. ++ +[source,bash,subs="verbatim,attributes,quotes"] +---- +kubectl edit deployment coredns -n kube-system +---- ++ +[source,yaml,subs="verbatim,attributes"] +---- +spec: + template: + spec: + affinity: + ... + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: k8s-app + operator: In + values: + - kube-dns + topologyKey: kubernetes.io/hostname + weight: 100 + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: k8s-app + operator: In + values: + - kube-dns + topologyKey: topology.kubernetes.io/zone + weight: 50 + ... +---- ++ +. Add the setting `trafficDistribution: PreferClose` to the `kube-dns` Service configuration to enable Service Traffic Distribution. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl patch svc kube-dns -n kube-system --type=merge -p '{ + "spec": { + "trafficDistribution": "PreferClose" + } +}' +---- ++ +. You can confirm that Service Traffic Distribution is enabled by viewing the endpoint slices for the `kube-dns` Service. Your endpoint slices must show the `hints` for your topology zone labels, which confirms that Service Traffic Distribution is enabled. If you do not see the `hints` for each endpoint address, then Service Traffic Distribution is not enabled. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get endpointslice -A | grep "kube-dns" +---- ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get endpointslice [.replaceable]`kube-dns-` -n kube-system -o yaml +---- ++ +[source,yaml,subs="verbatim,attributes"] +---- +addressType: IPv4 +apiVersion: discovery.k8s.io/v1 +endpoints: +- addresses: + - + hints: + forZones: + - name: onprem + nodeName: + zone: onprem +- addresses: + - + hints: + forZones: + - name: us-west-2a + nodeName: + zone: us-west-2a +---- + +[#hybrid-nodes-webhooks-add-ons] +=== Configure webhooks for add-ons + +The following add-ons use webhooks and are supported for use with hybrid nodes. + +- {aws} Load Balancer Controller +- CloudWatch Observability Agent +- {aws} Distro for OpenTelemetry (ADOT) +- `cert-manager` + +See the following sections for configuring the webhooks used by these add-ons to run on nodes in {aws} Cloud. + +[#hybrid-nodes-mixed-lbc] +==== {aws} Load Balancer Controller + +To use the {aws} Load Balancer Controller in a mixed mode cluster setup, you must run the controller on nodes in {aws} Cloud. To do so, add the following to your Helm values configuration or specify the values by using EKS add-on configuration. + +[source,yaml,subs="verbatim,attributes"] +---- +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - hybrid +---- + +[#hybrid-nodes-mixed-cwagent] +==== CloudWatch Observability Agent + +The CloudWatch Observability Agent add-on has a Kubernetes Operator that uses webhooks. To run the operator on nodes in {aws} Cloud in a mixed mode cluster setup, edit the CloudWatch Observability Agent operator configuration. You can't configure the operator affinity during installation with Helm and EKS add-ons (see link:https://github.com/aws/containers-roadmap/issues/2431[containers-roadmap issue #2431]). + +[source,bash,subs="verbatim,attributes"] +---- +kubectl edit -n amazon-cloudwatch deployment amazon-cloudwatch-observability-controller-manager +---- + +[source,yaml,subs="verbatim,attributes"] +---- +spec: + ... + template: + ... + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - hybrid +---- + +[#hybrid-nodes-mixed-adot] +==== {aws} Distro for OpenTelemetry (ADOT) + +The {aws} Distro for OpenTelemetry (ADOT) add-on has a Kubernetes Operator that uses webhooks. To run the operator on nodes in {aws} Cloud in a mixed mode cluster setup, add the following to your Helm values configuration or specify the values by using EKS add-on configuration. + +[source,yaml,subs="verbatim,attributes"] +---- +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - hybrid +---- + +If your pod CIDR is not routable on your on-premises network, then the ADOT collector must run on hybrid nodes to scrape the metrics from your hybrid nodes and the workloads running on them. To do so, edit the Custom Resource Definition (CRD). + +[source,bash,subs="verbatim,attributes"] +---- +kubectl -n opentelemetry-operator-system edit opentelemetrycollectors.opentelemetry.io adot-col-prom-metrics +---- + +[source,yaml,subs="verbatim,attributes"] +---- +spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: In + values: + - hybrid +---- + +You can configure the ADOT collector to only scrape metrics from hybrid nodes and the resources running on hybrid nodes by adding the following `relabel_configs` to each `scrape_configs` in the ADOT collector CRD configuration. + +[source,yaml,subs="verbatim,attributes"] +---- +relabel_configs: + - action: keep + regex: hybrid + source_labels: + - __meta_kubernetes_node_label_eks_amazonaws_com_compute_type +---- + +The ADOT add-on has a prerequisite requirement to install `cert-manager` for the TLS certificates used by the ADOT operator webhook. `cert-manager` also runs webhooks and you can configure it to run on nodes in {aws} Cloud with the following Helm values configuration. + +[source,yaml,subs="verbatim,attributes"] +---- +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - hybrid +webhook: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - hybrid +cainjector: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - hybrid +startupapicheck: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - hybrid +---- + +[#hybrid-nodes-mixed-cert-manager] +==== `cert-manager` +The `cert-manager` add-on runs webhooks and you can configure it to run on nodes in {aws} Cloud with the following Helm values configuration. + +[source,yaml,subs="verbatim,attributes"] +---- +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - hybrid +webhook: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - hybrid +cainjector: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - hybrid +startupapicheck: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - hybrid +---- diff --git a/latest/ug/nodes/hybrid-nodes.adoc b/latest/ug/nodes/hybrid-nodes.adoc deleted file mode 100644 index dc678f56d..000000000 --- a/latest/ug/nodes/hybrid-nodes.adoc +++ /dev/null @@ -1,100 +0,0 @@ -//!!NODE_ROOT
-[.topic] -[[hybrid-nodes-overview,hybrid-nodes-overview.title]] -= Amazon EKS Hybrid Nodes overview -:info_doctype: section -:info_title: Amazon EKS Hybrid Nodes overview -:info_titleabbrev: Hybrid nodes -:keywords: on-premises, hybrid -:info_abstract: Amazon EKS Hybrid Nodes overview - -include::../attributes.txt[] - -[abstract] --- -Join nodes from your data centers to Amazon EKS [.noloc]`Kubernetes` clusters with Amazon EKS Hybrid Nodes. --- - -With _Amazon EKS Hybrid Nodes_, you can use your on-premises and edge infrastructure as nodes in Amazon EKS clusters. {aws} manages the {aws}-hosted Kubernetes control plane of the Amazon EKS cluster, and you manage the hybrid nodes that run in your on-premises or edge environments. This unifies Kubernetes management across your environments and offloads Kubernetes control plane management to {aws} for your on-premises and edge applications. - -Amazon EKS Hybrid Nodes works with any on-premises hardware or virtual machines, bringing the efficiency, scalability, and availability of Amazon EKS to wherever your applications need to run. You can use a wide range of Amazon EKS features with Amazon EKS Hybrid Nodes including Amazon EKS add-ons, Amazon EKS [.noloc]`Pod` Identity, cluster access entries, cluster insights, and extended Kubernetes version support. Amazon EKS Hybrid Nodes natively integrates with {aws-services} including {aws} Systems Manager, {aws} IAM Roles Anywhere, Amazon Managed Service for Prometheus, Amazon CloudWatch, and Amazon GuardDuty for centralized monitoring, logging, and identity management. - -With Amazon EKS Hybrid Nodes, there are no upfront commitments or minimum fees, and you are charged per hour for the vCPU resources of your hybrid nodes when they are attached to your Amazon EKS clusters. For more pricing information, see link:eks/pricing/[Amazon EKS Pricing,type="marketing"]. - -For an overview of the other Amazon EKS options for on-premises and edge deployments, see <>. - - -[[hybrid-nodes-general,hybrid-nodes-general.title]] -== General concepts of Amazon EKS Hybrid Nodes - -* Amazon EKS Hybrid Nodes must have a reliable connection between your on-premises environment and {aws}. Amazon EKS Hybrid Nodes aren't a fit for disconnected, disrupted, intermittent or limited (DDIL) environments. If you are running in a DDIL environment, consider link:eks/eks-anywhere/[Amazon EKS Anywhere,type="marketing"]. -* Running Amazon EKS Hybrid Nodes on cloud infrastructure, including {aws-regions}, {aws} Local Zones, {outposts}, or in other clouds, is not supported. Use Amazon EKS Auto Mode, Karpenter, Amazon EC2 managed node groups, self-managed nodes, or {aws} Fargate when running in {aws-regions}. Use Amazon EC2 managed node groups or Amazon EC2 self-managed nodes when running on {aws} Local Zones. Only Amazon EC2 self-managed nodes can be used on {outposts} or {aws} Wavelength Zones. -* A single Amazon EKS cluster can be used to run hybrid nodes and nodes in {aws-regions}, {aws} Local Zones, or {outposts}. -* Amazon EKS Hybrid Nodes is available in all {aws-regions}, except the {aws} GovCloud (US) Regions and the {aws} China Regions. -* You will be charged the hybrid nodes fee if you run hybrid nodes on Amazon EC2 instances. -* Billing for hybrid nodes starts when the nodes join the Amazon EKS cluster and stops when the nodes are removed from the cluster. Be sure to remove your hybrid nodes from your Amazon EKS cluster if you are not using them. - -*Infrastructure Management* - -* Amazon EKS Hybrid Nodes follows a _bring your own infrastructure_ approach where it is your responsibility to provision and manage the physical or virtual machines and the operating system you use for hybrid nodes. -* Amazon EKS Hybrid Nodes are agnostic to the infrastructure they run on. You can run hybrid nodes on physical or virtual machines, and x86 and ARM architectures. - -*Operating Systems for hybrid nodes* - -* *Amazon Linux 2023 (AL2023)*: You can use Amazon Linux 2023 (AL2023) as the node operating system for hybrid nodes, but only in virtualized environments such as VMWare, KVM, and Hyper-V. {aws} supports the integration of hybrid nodes with AL2023, but AL2023 isn't covered by the {aws} Support Plans when you run it outside of Amazon EC2. -* *Ubuntu*: You can use Ubuntu 20.04, Ubuntu 22.04, and Ubuntu 24.04 as the node operating system for hybrid nodes. -* *Red Hat Enterprise Linux (RHEL)*: You can use RHEL 8 and RHEL 9 as the node operating system for hybrid nodes. - - -*Kubernetes and platform versions* - -* Amazon EKS Hybrid Nodes supports the same Kubernetes versions and deprecation schedule as Amazon EKS, including standard and extended Kubernetes version support. For more information on Kubernetes versions in Amazon EKS, see <>. For more information about Amazon EKS platform versions, see <>. -* You must create new Amazon EKS clusters to use Amazon EKS Hybrid Nodes. Hybrid nodes can't be used with existing Amazon EKS clusters. - -*Networking* - -* The communication between the Amazon EKS control plane and hybrid nodes is routed through the VPC and subnets you pass during cluster creation, which builds on the link:eks/latest/best-practices/subnets.html[existing mechanism,type="documentation"] in Amazon EKS for control plane to node networking. -* Amazon EKS Hybrid Nodes is flexible to your preferred method of connecting your on-premises networks to a VPC in {aws}. There are several link:whitepapers/latest/aws-vpc-connectivity-options/network-to-amazon-vpc-connectivity-options.html[documented options,type="documentation"] available including {aws} Site-to-Site VPN and {aws} Direct Connect, and you can choose the method that best fits your use case. -* *IP address family*: Hybrid nodes can be used with Amazon EKS clusters configured with the `IPv4` IP address family only. You can't use Amazon EKS clusters configured with the `IPv6` IP address family. Similarly, your on-premises node and [.noloc]`Pod` CIDRs must be `IPv4` RFC1918 CIDR blocks. -* You must enable the required domains, protocols, and ports for Amazon EKS Hybrid Nodes in your on-premises environments and firewalls. For more information, including minimum networking requirements, see <>. -* *Cluster endpoint access*: You can use “Public” or “Private” cluster endpoint access. You should not use “Public and Private” cluster endpoint access, as the endpoint DNS resolution will always resolve to the public addresses for queries originating from your on-premises environment. -* For information and best practices during scenarios where there are network disconnections between hybrid nodes and the {aws-region}, see the link:eks/latest/best-practices/hybrid-nodes.html[hybrid nodes,type="documentation"] section of the _Amazon EKS Best Practices Guide_. -* *Application load balancing*: Kubernetes has a https://kubernetes.io/docs/concepts/services-networking/service/[Service] object to define the names and domain names for your applications and resolve and load balance to them. By default, the `type:LoadBalancer` type of Service additionally creates an {aws} Classic Load Balancer for traffic from outside the cluster. You can change this behavior with add-ons. Specifically, we recommend the {aws} Application Load Balancer and {aws} Network Load Balancer which are created by the {aws} Load Balancer Controller, instead of the {aws} Classic Load Balancer. For steps to install the {aws} Load Balancer Controller in a hybrid environment, see <>. - -*Security for hybrid nodes* - -* Amazon EKS Hybrid Nodes use temporary IAM credentials to authenticate with your Amazon EKS cluster. You can use either {aws} IAM Roles Anywhere or {aws} Systems Manager (SSM) hybrid activations for provisioning the on-premises IAM credentials for hybrid nodes. It is recommended to use {aws} SSM hybrid activations if you do not have existing Public Key Infrastructure (PKI) with a Certificate Authority (CA) and certificates for your on-premises environments. If you do have existing PKI and certificates on-premises, use {aws} IAM Roles Anywhere. -* You can use `API` or `API_AND_CONFIG_MAP` cluster authentication modes for your hybrid nodes-enabled Amazon EKS clusters. Use the cluster access entry type called `HYBRID_LINUX` with your hybrid nodes IAM role to enable hybrid nodes to join the Amazon EKS cluster. -* OIDC authentication is supported for hybrid nodes-enabled Amazon EKS clusters. -* You can use Amazon EKS [.noloc]`Pod` Identities and IAM Roles for Service Accounts (IRSA) with applications running on hybrid nodes to enable granular access for your [.noloc]`Pods` running on hybrid nodes with other {aws-services}. -* You can use Amazon GuardDuty EKS Protection with hybrid nodes-enabled Amazon EKS clusters to analyze activities of users and applications accessing your cluster. - -*Add-ons for hybrid nodes* - -For detailed information, see <>. - -* *Container Networking Interface (CNI)*: The {aws} VPC CNI can't be used with hybrid nodes. The core capabilities of [.noloc]`Cilium` and [.noloc]`Calico` are supported for use with hybrid nodes. You can manage your CNI with your choice of tooling such as [.noloc]`Helm`. For more information, see <>. -* *`kube-proxy` and CoreDNS*: `kube-proxy` and CoreDNS are installed automatically when hybrid nodes join the Amazon EKS cluster. These add-ons can be managed as Amazon EKS add-ons after cluster creation. -* *Ingress and Load Balancing*: You can use the {aws} Load Balancer Controller and Application Load Balancer (ALB) or Network Load Balancer (NLB) with the target type `ip` for workloads on hybrid nodes connected with {aws} Direct Connect or {aws} Site-to-Site VPN. You can alternatively use your choice of Ingress controller or load balancer for application traffic that stays local to your on-premises environment. -* *Metrics*: You can use Amazon Managed Prometheus (AMP) agent-less scrapers, {aws} Distro for Open Telemetry (ADOT), and the Amazon CloudWatch Observability Agent with hybrid nodes. To use AMP agent-less scrapers for [.noloc]`Pod` metrics on hybrid nodes, your [.noloc]`Pods` must be accessible from the VPC that you use for the Amazon EKS cluster. -* *Logs*: You can enable Amazon EKS control plane logging for hybrid nodes-enabled clusters. You can use the ADOT EKS add-on and the Amazon CloudWatch Observability Agent EKS add-on for hybrid node and [.noloc]`Pod` logging. - -*User interfaces* - -* *Node management*: The Amazon EKS Hybrid Nodes CLI is called `nodeadm` and is run on each on-premises host to simplify the installation, configuration, registration, and uninstall of the hybrid nodes components. The hybrid nodes `nodeadm` version is different than the `nodeadm` version used in the AL2023 Amazon EKS-optimized AMIs. You should not use the hybrid nodes `nodeadm` version for nodes running in Amazon EC2. -* *Cluster management*: The Amazon EKS user interfaces for cluster management are the same with hybrid nodes-enabled Amazon EKS clusters. This includes the {aws-management-console}, {aws} API, {aws} SDKs, {cli}, [.noloc]`eksctl` CLI, {cloudformation}, and Terraform. - - -include::hybrid-nodes-prereqs.adoc[leveloffset=+1] - -include::hybrid-nodes-tutorial.adoc[leveloffset=+1] - -include::hybrid-nodes-cni.adoc[leveloffset=+1] - -include::hybrid-nodes-add-ons.adoc[leveloffset=+1] - -include::hybrid-nodes-proxy.adoc[leveloffset=+1] - -include::hybrid-nodes-nodeadm.adoc[leveloffset=+1] - -include::hybrid-nodes-troubleshooting.adoc[leveloffset=+1] diff --git a/latest/ug/nodes/launch-node-bottlerocket.adoc b/latest/ug/nodes/launch-node-bottlerocket.adoc index 732dca9b2..b19993068 100644 --- a/latest/ug/nodes/launch-node-bottlerocket.adoc +++ b/latest/ug/nodes/launch-node-bottlerocket.adoc @@ -1,13 +1,13 @@ -//!!NODE_ROOT
include::../attributes.txt[] + [.topic] -[[launch-node-bottlerocket,launch-node-bottlerocket.title]] -= Create self-managed [.noloc]`Bottlerocket` nodes +[#launch-node-bottlerocket] += Create self-managed Bottlerocket nodes :info_titleabbrev: Bottlerocket [abstract] -- -This topic describes how to launch Auto Scaling groups of [.noloc]`Bottlerocket` nodes that register with your Amazon EKS cluster +This topic describes how to launch Auto Scaling groups of Bottlerocket nodes that register with your Amazon EKS cluster -- [NOTE] @@ -17,9 +17,9 @@ Managed node groups might offer some advantages for your use case. For more info ==== -This topic describes how to launch Auto Scaling groups of link:bottlerocket/[Bottlerocket,type="marketing"] nodes that register with your Amazon EKS cluster. [.noloc]`Bottlerocket` is a [.noloc]`Linux`-based open-source operating system from {aws} that you can use for running containers on virtual machines or bare metal hosts. After the nodes join the cluster, you can deploy [.noloc]`Kubernetes` applications to them. For more information about [.noloc]`Bottlerocket`, see https://github.com/bottlerocket-os/bottlerocket/blob/develop/QUICKSTART-EKS.md[Using a Bottlerocket AMI with Amazon EKS] on [.noloc]`GitHub` and https://eksctl.io/usage/custom-ami-support/[Custom AMI support] in the `eksctl` documentation. +This topic describes how to launch Auto Scaling groups of link:bottlerocket/[Bottlerocket,type="marketing"] nodes that register with your Amazon EKS cluster. Bottlerocket is a Linux-based open-source operating system from {aws} that you can use for running containers on virtual machines or bare metal hosts. After the nodes join the cluster, you can deploy Kubernetes applications to them. For more information about Bottlerocket, see https://github.com/bottlerocket-os/bottlerocket/blob/develop/QUICKSTART-EKS.md[Using a Bottlerocket AMI with Amazon EKS] on GitHub and https://eksctl.io/usage/custom-ami-support/[Custom AMI support] in the `eksctl` documentation. -For information about in-place upgrades, see https://github.com/bottlerocket-os/bottlerocket-update-operator[Bottlerocket Update Operator] on [.noloc]`GitHub`. +For information about in-place upgrades, see https://github.com/bottlerocket-os/bottlerocket-update-operator[Bottlerocket Update Operator] on GitHub. [IMPORTANT] ==== @@ -27,9 +27,9 @@ For information about in-place upgrades, see https://github.com/bottlerocket-os/ * Amazon EKS nodes are standard Amazon EC2 instances, and you are billed for them based on normal Amazon EC2 instance prices. For more information, see link:ec2/pricing/[Amazon EC2 pricing,type="marketing"]. * You can launch Bottlerocket nodes in Amazon EKS extended clusters on {aws} Outposts, but you can't launch them in local clusters on {aws} Outposts. For more information, see <>. -* You can deploy to Amazon EC2 instances with `x86` or [.noloc]`Arm` processors. However, you can't deploy to instances that have [.noloc]`Inferentia` chips. -* [.noloc]`Bottlerocket` is compatible with {aws} CloudFormation. However, there is no official CloudFormation template that can be copied to deploy [.noloc]`Bottlerocket` nodes for Amazon EKS. -* [.noloc]`Bottlerocket` images don't come with an [.noloc]`SSH` server or a shell. You can use out-of-band access methods to allow [.noloc]`SSH` enabling the admin container and to pass some bootstrapping configuration steps with user data. For more information, see these sections in the https://github.com/bottlerocket-os/bottlerocket[bottlerocket README.md] on [.noloc]`GitHub`: +* You can deploy to Amazon EC2 instances with `x86` or Arm processors. However, you can't deploy to instances that have Inferentia chips. +* Bottlerocket is compatible with {aws} CloudFormation. However, there is no official CloudFormation template that can be copied to deploy Bottlerocket nodes for Amazon EKS. +* Bottlerocket images don't come with an SSH server or a shell. You can use out-of-band access methods to allow SSH enabling the admin container and to pass some bootstrapping configuration steps with user data. For more information, see these sections in the https://github.com/bottlerocket-os/bottlerocket[bottlerocket README.md] on GitHub: + ** https://github.com/bottlerocket-os/bottlerocket#exploration[Exploration] ** https://github.com/bottlerocket-os/bottlerocket#admin-container[Admin container] @@ -45,10 +45,10 @@ eksctl version ---- For instructions on how to install or upgrade `eksctl`, see https://eksctl.io/installation[Installation] in the `eksctl` documentation.NOTE: This procedure only works for clusters that were created with `eksctl`. + +. Copy the following contents to your device. Replace [.replaceable]`my-cluster` with the name of your cluster. The name can contain only alphanumeric characters (case-sensitive) and hyphens. It must start with an alphanumeric character and can't be longer than 100 characters. The name must be unique within the {aws} Region and {aws} account that you're creating the cluster in. Replace [.replaceable]`ng-bottlerocket` with a name for your node group. The node group name can't be longer than 63 characters. It must start with letter or digit, but can also include hyphens and underscores for the remaining characters. To deploy on Arm instances, replace [.replaceable]`m5.large` with an Arm instance type. Replace [.replaceable]`my-ec2-keypair-name` with the name of an Amazon EC2 SSH key pair that you can use to connect using SSH into your nodes with after they launch. If you don't already have an Amazon EC2 key pair, you can create one in the {aws-management-console}. For more information, see link:AWSEC2/latest/UserGuide/ec2-key-pairs.html[Amazon EC2 key pairs,type="documentation"] in the _Amazon EC2 User Guide_. Replace all remaining example values with your own values. Once you've made the replacements, run the modified command to create the `bottlerocket.yaml` file. + -. Copy the following contents to your device. Replace [.replaceable]`my-cluster` with the name of your cluster. The name can contain only alphanumeric characters (case-sensitive) and hyphens. It must start with an alphanumeric character and can't be longer than 100 characters. The name must be unique within the {aws} Region and {aws} account that you're creating the cluster in. Replace [.replaceable]`ng-bottlerocket` with a name for your node group. The node group name can't be longer than 63 characters. It must start with letter or digit, but can also include hyphens and underscores for the remaining characters. To deploy on Arm instances, replace [.replaceable]`m5.large` with an Arm instance type. Replace [.replaceable]`my-ec2-keypair-name` with the name of an Amazon EC2 SSH key pair that you can use to connect using SSH into your nodes with after they launch. If you don't already have an Amazon EC2 key pair, you can create one in the {aws-management-console}. For more information, see link:AWSEC2/latest/UserGuide/ec2-key-pairs.html[Amazon EC2 key pairs,type="documentation"] in the _Amazon EC2 User Guide_. Replace all remaining [.replaceable]`example values` with your own values. Once you've made the replacements, run the modified command to create the `bottlerocket.yaml` file. -+ -If specifying an Arm Amazon EC2 instance type, then review the considerations in <> before deploying. For instructions on how to deploy using a custom AMI, see https://github.com/bottlerocket-os/bottlerocket/blob/develop/BUILDING.md[Building Bottlerocket] on [.noloc]`GitHub` and https://eksctl.io/usage/custom-ami-support/[Custom AMI support] in the `eksctl` documentation. To deploy a managed node group, deploy a custom AMI using a launch template. For more information, see <>. +If specifying an Arm Amazon EC2 instance type, then review the considerations in <> before deploying. For instructions on how to deploy using a custom AMI, see https://github.com/bottlerocket-os/bottlerocket/blob/develop/BUILDING.md[Building Bottlerocket] on GitHub and https://eksctl.io/usage/custom-ami-support/[Custom AMI support] in the `eksctl` documentation. To deploy a managed node group, deploy a custom AMI using a launch template. For more information, see <>. + IMPORTANT: To deploy a node group to {aws} Outposts, {aws} Wavelength, or {aws} Local Zone subnets, don't pass {aws} Outposts, {aws} Wavelength, or {aws} Local Zone subnets when you create the cluster. You must specify the subnets in the following example. For more information see https://eksctl.io/usage/nodegroups/#creating-a-nodegroup-from-a-config-file[Create a nodegroup from a config file] and https://eksctl.io/usage/schema/[Config file schema] in the `eksctl` documentation. Replace [.replaceable]`region-code` with the {aws} Region that your cluster is in. + @@ -62,7 +62,7 @@ kind: ClusterConfig metadata: name: my-cluster region: region-code - version: '1.30' + version: '{k8s-n}' iam: withOIDC: true @@ -99,8 +99,8 @@ Several lines are output while the nodes are created. One of the last lines of o ---- [✔] created 1 nodegroup(s) in cluster "my-cluster" ---- -. (Optional) Create a [.noloc]`Kubernetes` https://kubernetes.io/docs/concepts/storage/persistent-volumes/[persistent volume] on a [.noloc]`Bottlerocket` node using the https://github.com/kubernetes-sigs/aws-ebs-csi-driver[Amazon EBS CSI Plugin]. The default Amazon EBS driver relies on file system tools that aren't included with [.noloc]`Bottlerocket`. For more information about creating a storage class using the driver, see <>. -. (Optional) By default, `kube-proxy` sets the `nf_conntrack_max` kernel parameter to a default value that may differ from what [.noloc]`Bottlerocket` originally sets at boot. To keep [.noloc]`Bottlerocket`'s https://github.com/bottlerocket-os/bottlerocket/blob/develop/packages/release/release-sysctl.conf[default setting], edit the `kube-proxy` configuration with the following command. +. (Optional) Create a Kubernetes https://kubernetes.io/docs/concepts/storage/persistent-volumes/[persistent volume] on a Bottlerocket node using the https://github.com/kubernetes-sigs/aws-ebs-csi-driver[Amazon EBS CSI Plugin]. The default Amazon EBS driver relies on file system tools that aren't included with Bottlerocket. For more information about creating a storage class using the driver, see <>. +. (Optional) By default, `kube-proxy` sets the `nf_conntrack_max` kernel parameter to a default value that may differ from what Bottlerocket originally sets at boot. To keep Bottlerocket's https://github.com/bottlerocket-os/bottlerocket-core-kit/blob/develop/packages/release/release-sysctl.conf[default setting], edit the `kube-proxy` configuration with the following command. + [source,bash,subs="verbatim,attributes"] ---- @@ -119,11 +119,11 @@ Add `--conntrack-max-per-core` and `--conntrack-min` to the `kube-proxy` argumen - --conntrack-max-per-core=0 - --conntrack-min=0 ---- -. (Optional) Deploy a <> to test your [.noloc]`Bottlerocket` nodes. -. We recommend blocking [.noloc]`Pod` access to IMDS if the following conditions are true: +. (Optional) Deploy a <> to test your Bottlerocket nodes. +. We recommend blocking Pod access to IMDS if the following conditions are true: + -** You plan to assign IAM roles to all of your [.noloc]`Kubernetes` service accounts so that [.noloc]`Pods` only have the minimum permissions that they need. -** No [.noloc]`Pods` in the cluster require access to the Amazon EC2 instance metadata service (IMDS) for other reasons, such as retrieving the current {aws} Region. +** You plan to assign IAM roles to all of your Kubernetes service accounts so that Pods only have the minimum permissions that they need. +** No Pods in the cluster require access to the Amazon EC2 instance metadata service (IMDS) for other reasons, such as retrieving the current {aws} Region. + -For more information, see https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node[Restrict access to the instance profile assigned to the worker node]. +For more information, see https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node[Restrict access to the instance profile assigned to the worker node]. \ No newline at end of file diff --git a/latest/ug/nodes/launch-node-ubuntu.adoc b/latest/ug/nodes/launch-node-ubuntu.adoc index 35909f2a2..c20ac7ecb 100644 --- a/latest/ug/nodes/launch-node-ubuntu.adoc +++ b/latest/ug/nodes/launch-node-ubuntu.adoc @@ -1,13 +1,13 @@ -//!!NODE_ROOT
include::../attributes.txt[] + [.topic] -[[launch-node-ubuntu,launch-node-ubuntu.title]] -= Create self-managed [.noloc]`Ubuntu Linux` nodes +[#launch-node-ubuntu] += Create self-managed Ubuntu Linux nodes :info_titleabbrev: Ubuntu Linux [abstract] -- -This topic describes how to launch Auto Scaling groups of [.noloc]`Ubuntu` nodes that register with your Amazon EKS cluster +This topic describes how to launch Auto Scaling groups of Ubuntu nodes that register with your Amazon EKS cluster -- [NOTE] @@ -17,7 +17,7 @@ Managed node groups might offer some advantages for your use case. For more info ==== -This topic describes how to launch Auto Scaling groups of https://cloud-images.ubuntu.com/aws-eks/[Ubuntu on Amazon Elastic Kubernetes Service (EKS)] or https://ubuntu.com/blog/ubuntu-pro-for-eks-is-now-generally-available[Ubuntu Pro on Amazon Elastic Kubernetes Service (EKS)] nodes that register with your Amazon EKS cluster. [.noloc]`Ubuntu` and [.noloc]`Ubuntu Pro` for EKS are based on the official [.noloc]`Ubuntu` Minimal LTS, include the custom {aws} kernel that is jointly developed with {aws}, and have been built specifically for EKS. [.noloc]`Ubuntu Pro` adds additional security coverage by supporting EKS extended support periods, kernel [.noloc]`livepatch`, FIPS compliance and the ability to run unlimited [.noloc]`Pro` containers. +This topic describes how to launch Auto Scaling groups of https://cloud-images.ubuntu.com/aws-eks/[Ubuntu on Amazon Elastic Kubernetes Service (EKS)] or https://ubuntu.com/blog/ubuntu-pro-for-eks-is-now-generally-available[Ubuntu Pro on Amazon Elastic Kubernetes Service (EKS)] nodes that register with your Amazon EKS cluster. Ubuntu and Ubuntu Pro for EKS are based on the official Ubuntu Minimal LTS, include the custom {aws} kernel that is jointly developed with {aws}, and have been built specifically for EKS. Ubuntu Pro adds additional security coverage by supporting EKS extended support periods, kernel livepatch, FIPS compliance and the ability to run unlimited Pro containers. After the nodes join the cluster, you can deploy containerized applications to them. For more information, visit the documentation for https://documentation.ubuntu.com/aws/en/latest/[Ubuntu on {aws}] and https://eksctl.io/usage/custom-ami-support/[Custom AMI support] in the `eksctl` documentation. @@ -26,8 +26,8 @@ After the nodes join the cluster, you can deploy containerized applications to t * Amazon EKS nodes are standard Amazon EC2 instances, and you are billed for them based on normal Amazon EC2 instance prices. For more information, see link:ec2/pricing/[Amazon EC2 pricing,type="marketing"]. -* You can launch [.noloc]`Ubuntu` nodes in Amazon EKS extended clusters on {aws} Outposts, but you can't launch them in local clusters on {aws} Outposts. For more information, see <>. -* You can deploy to Amazon EC2 instances with `x86` or [.noloc]`Arm` processors. However, instances that have [.noloc]`Inferentia` chips might need to install the https://awsdocs-neuron.readthedocs-hosted.com/en/latest/[Neuron SDK] first. +* You can launch Ubuntu nodes in Amazon EKS extended clusters on {aws} Outposts, but you can't launch them in local clusters on {aws} Outposts. For more information, see <>. +* You can deploy to Amazon EC2 instances with `x86` or Arm processors. However, instances that have Inferentia chips might need to install the https://awsdocs-neuron.readthedocs-hosted.com/en/latest/[Neuron SDK] first. ==== @@ -39,8 +39,8 @@ eksctl version ---- For instructions on how to install or upgrade `eksctl`, see https://eksctl.io/installation[Installation] in the `eksctl` documentation.NOTE: This procedure only works for clusters that were created with `eksctl`. -+ -. Copy the following contents to your device. Replace `my-cluster` with the name of your cluster. The name can contain only alphanumeric characters (case-sensitive) and hyphens. It must start with an alphabetic character and can't be longer than 100 characters. Replace `ng-ubuntu` with a name for your node group. The node group name can't be longer than 63 characters. It must start with letter or digit, but can also include hyphens and underscores for the remaining characters. To deploy on [.noloc]`Arm` instances, replace `m5.large` with an [.noloc]`Arm` instance type. Replace `my-ec2-keypair-name` with the name of an Amazon EC2 SSH key pair that you can use to connect using SSH into your nodes with after they launch. If you don't already have an Amazon EC2 key pair, you can create one in the {aws-management-console}. For more information, see link:AWSEC2/latest/UserGuide/ec2-key-pairs.html[Amazon EC2 key pairs,type="documentation"] in the Amazon EC2 User Guide. Replace all remaining [.replaceable]`example values` with your own values. Once you've made the replacements, run the modified command to create the `ubuntu.yaml` file. + +. Copy the following contents to your device. Replace `my-cluster` with the name of your cluster. The name can contain only alphanumeric characters (case-sensitive) and hyphens. It must start with an alphabetic character and can't be longer than 100 characters. Replace `ng-ubuntu` with a name for your node group. The node group name can't be longer than 63 characters. It must start with letter or digit, but can also include hyphens and underscores for the remaining characters. To deploy on Arm instances, replace `m5.large` with an Arm instance type. Replace `my-ec2-keypair-name` with the name of an Amazon EC2 SSH key pair that you can use to connect using SSH into your nodes with after they launch. If you don't already have an Amazon EC2 key pair, you can create one in the {aws-management-console}. For more information, see link:AWSEC2/latest/UserGuide/ec2-key-pairs.html[Amazon EC2 key pairs,type="documentation"] in the Amazon EC2 User Guide. Replace all remaining example values with your own values. Once you've made the replacements, run the modified command to create the `ubuntu.yaml` file. + IMPORTANT: To deploy a node group to {aws} Outposts, {aws} Wavelength, or {aws} Local Zone subnets, don't pass {aws} Outposts, {aws} Wavelength, or {aws} Local Zone subnets when you create the cluster. You must specify the subnets in the following example. For more information see https://eksctl.io/usage/nodegroups/#creating-a-nodegroup-from-a-config-file[Create a nodegroup from a config file] and https://eksctl.io/usage/schema/[Config file schema] in the `eksctl` documentation. Replace [.replaceable]`region-code` with the {aws} Region that your cluster is in. + @@ -54,7 +54,7 @@ kind: ClusterConfig metadata: name: my-cluster region: region-code - version: '1.30' + version: '{k8s-n}' iam: withOIDC: true @@ -64,7 +64,6 @@ nodeGroups: instanceType: m5.large desiredCapacity: 3 amiFamily: Ubuntu2204 - ami: auto-ssm iam: attachPolicyARNs: - {arn-aws}iam::aws:policy/AmazonEKSWorkerNodePolicy @@ -77,7 +76,7 @@ nodeGroups: EOF ---- + -To create an [.noloc]`Ubuntu Pro` node group, just change the `amiFamily` value to `UbuntuPro2204`. +To create an Ubuntu Pro node group, just change the `amiFamily` value to `UbuntuPro2204`. . Deploy your nodes with the following command. + [source,bash,subs="verbatim,attributes"] @@ -93,11 +92,11 @@ Several lines are output while the nodes are created. One of the last lines of o ---- [✔] created 1 nodegroup(s) in cluster "my-cluster" ---- -. (Optional) Deploy a <> to test your [.noloc]`Ubuntu` nodes. -. We recommend blocking [.noloc]`Pod` access to IMDS if the following conditions are true: +. (Optional) Deploy a <> to test your Ubuntu nodes. +. We recommend blocking Pod access to IMDS if the following conditions are true: + -** You plan to assign IAM roles to all of your [.noloc]`Kubernetes` service accounts so that [.noloc]`Pods` only have the minimum permissions that they need. -** No [.noloc]`Pods` in the cluster require access to the Amazon EC2 instance metadata service (IMDS) for other reasons, such as retrieving the current {aws} Region. +** You plan to assign IAM roles to all of your Kubernetes service accounts so that Pods only have the minimum permissions that they need. +** No Pods in the cluster require access to the Amazon EC2 instance metadata service (IMDS) for other reasons, such as retrieving the current {aws} Region. + -For more information, see https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node[Restrict access to the instance profile assigned to the worker node]. +For more information, see https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node[Restrict access to the instance profile assigned to the worker node]. \ No newline at end of file diff --git a/latest/ug/nodes/launch-templates.adoc b/latest/ug/nodes/launch-templates.adoc index 1919066da..136fea7bf 100644 --- a/latest/ug/nodes/launch-templates.adoc +++ b/latest/ug/nodes/launch-templates.adoc @@ -1,7 +1,7 @@ -//!!NODE_ROOT
include::../attributes.txt[] + [.topic] -[[launch-templates,launch-templates.title]] +[#launch-templates] = Customize managed nodes with launch templates :info_titleabbrev: Launch templates @@ -12,10 +12,8 @@ For the highest level of customization, you can deploy managed nodes using your For the highest level of customization, you can deploy managed nodes using your own launch template. Using a launch template allows capabilities such as the following: - - * Provide bootstrap arguments at deployment of a node, such as extra https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/[kubelet] arguments. -* Assign IP addresses to [.noloc]`Pods` from a different CIDR block than the IP address assigned to the node. +* Assign IP addresses to Pods from a different CIDR block than the IP address assigned to the node. * Deploy your own custom AMI to nodes. * Deploy your own custom CNI to nodes. @@ -23,93 +21,84 @@ When you give your own launch template upon first creating a managed node group, Managed node groups are always deployed with a launch template to be used with the Amazon EC2 Auto Scaling group. When you don't provide a launch template, the Amazon EKS API creates one automatically with default values in your account. However, we don't recommend that you modify auto-generated launch templates. Furthermore, existing node groups that don't use a custom launch template can't be updated directly. Instead, you must create a new node group with a custom launch template to do so. -[[launch-template-basics,launch-template-basics.title]] +[#launch-template-basics] == Launch template configuration basics You can create an Amazon EC2 Auto Scaling launch template with the {aws-management-console}, {aws} CLI, or an {aws} SDK. For more information, see link:autoscaling/ec2/userguide/create-launch-template.html[Creating a Launch Template for an Auto Scaling group,type="documentation"] in the _Amazon EC2 Auto Scaling User Guide_. Some of the settings in a launch template are similar to the settings used for managed node configuration. When deploying or updating a node group with a launch template, some settings must be specified in either the node group configuration or the launch template. Don't specify a setting in both places. If a setting exists where it shouldn't, then operations such as creating or updating a node group fail. The following table lists the settings that are prohibited in a launch template. It also lists similar settings, if any are available, that are required in the managed node group configuration. The listed settings are the settings that appear in the console. They might have similar but different names in the {aws} CLI and SDK. -[cols="1,1", options="header"] +[%header,cols="2"] |=== |Launch template – Prohibited |Amazon EKS node group configuration -|*Subnet* under *Network interfaces* (*Add network interface*) -|*Subnets* under *Node group network configuration* on the *Specify networking* page +|*Subnet* under *Network interfaces* (*Add network interface*) +|*Subnets* under *Node group network configuration* on the *Specify networking* page -|*IAM instance profile* under *Advanced details* -|*Node IAM role* under *Node group configuration* on the *Configure Node group* page +|*IAM instance profile* under *Advanced details* +|*Node IAM role* under *Node group configuration* on the *Configure Node group* page -|*Shutdown behavior* and *Stop - Hibernate behavior* under *Advanced details*. Retain default *Don't include in launch template setting* in launch template for both settings. +|*Shutdown behavior* and *Stop - Hibernate behavior* under *Advanced details*. Retain default *Don't include in launch template setting* in launch template for both settings. |No equivalent. Amazon EKS must control the instance lifecycle, not the Auto Scaling group. |=== The following table lists the prohibited settings in a managed node group configuration. It also lists similar settings, if any are available, which are required in a launch template. The listed settings are the settings that appear in the console. They might have similar names in the {aws} CLI and SDK. -[cols="1,1", options="header"] +[%header,cols="2"] |=== |Amazon EKS node group configuration – Prohibited |Launch template -|(Only if you specified a custom AMI in a launch template) *AMI type* under *Node group compute configuration* on *Set compute and scaling configuration* page – Console displays *Specified in launch template* and the AMI ID that was specified. +|(Only if you specified a custom AMI in a launch template) *AMI type* under *Node group compute configuration* on *Set compute and scaling configuration* page – Console displays *Specified in launch template* and the AMI ID that was specified. If *Application and OS Images (Amazon Machine Image)* wasn't specified in the launch template, you can select an AMI in the node group configuration. -|*Application and OS Images (Amazon Machine Image)* under *Launch template contents* – You must specify an ID if you have either of the following requirements: - - +a|*Application and OS Images (Amazon Machine Image)* under *Launch template contents* – You must specify an ID if you have either of the following requirements: * Using a custom AMI. If you specify an AMI that doesn't meet the requirements listed in <>, the node group deployment will fail. -* Want to provide user data to provide arguments to the `bootstrap.sh` file included with an Amazon EKS optimized AMI. You can enable your instances to assign a significantly higher number of IP addresses to [.noloc]`Pods`, assign IP addresses to [.noloc]`Pods` from a different CIDR block than the instance's, or deploy a private cluster without outbound internet access. For more information, see the following topics: -+ -** <> -** <> -** <> -** <> +* Want to provide additional arguments to the NodeConfig specification of an Amazon EKS optimized AMI. For more information, see https://awslabs.github.io/amazon-eks-ami/nodeadm/doc/api/#nodeconfig[NodeConfig API reference] and https://awslabs.github.io/amazon-eks-ami/nodeadm/doc/examples/[NodeConfig examples]. -|*Disk size* under *Node group compute configuration* on *Set compute and scaling configuration* page – Console displays *Specified in launch template*. -|*Size* under *Storage (Volumes)* (*Add new volume*). You must specify this in the launch template. -|*SSH key pair* under *Node group configuration* on the *Specify Networking* page – The console displays the key that was specified in the launch template or displays *Not specified in launch template*. -|*Key pair name* under *Key pair (login)*. +|*Disk size* under *Node group compute configuration* on *Set compute and scaling configuration* page – Console displays *Specified in launch template*. +|*Size* under *Storage (Volumes)* (*Add new volume*). You must specify this in the launch template. + +|*SSH key pair* under *Node group configuration* on the *Specify Networking* page – The console displays the key that was specified in the launch template or displays *Not specified in launch template*. +|*Key pair name* under *Key pair (login)*. |You can't specify source security groups that are allowed remote access when using a launch template. -|*Security groups* under *Network settings* for the instance or *Security groups* under *Network interfaces* (*Add network interface*), but not both. For more information, see <>. +|*Security groups* under *Network settings* for the instance or *Security groups* under *Network interfaces* (*Add network interface*), but not both. For more information, see <>. |=== [NOTE] ==== -* If you deploy a node group using a launch template, specify zero or one *Instance type* under *Launch template contents* in a launch template. Alternatively, you can specify 0–20 instance types for *Instance types* on the *Set compute and scaling configuration* page in the console. Or, you can do so using other tools that use the Amazon EKS API. If you specify an instance type in a launch template, and use that launch template to deploy your node group, then you can't specify any instance types in the console or using other tools that use the Amazon EKS API. If you don't specify an instance type in a launch template, in the console, or using other tools that use the Amazon EKS API, the `t3.medium` instance type is used. If your node group is using the Spot capacity type, then we recommend specifying multiple instance types using the console. For more information, see <>. -* If any containers that you deploy to the node group use the Instance Metadata Service Version 2, make sure to set the *Metadata response hop limit* to `2` in your launch template. For more information, see link:AWSEC2/latest/UserGuide/ec2-instance-metadata.html[Instance metadata and user data,type="documentation"] in the _Amazon EC2 User Guide_. If you deploy a managed node group without using a custom launch template, this value is automatically set for the node group in the default launch template. - +* If you deploy a node group using a launch template, specify zero or one *Instance type* under *Launch template contents* in a launch template. Alternatively, you can specify 0–20 instance types for *Instance types* on the *Set compute and scaling configuration* page in the console. Or, you can do so using other tools that use the Amazon EKS API. If you specify an instance type in a launch template, and use that launch template to deploy your node group, then you can't specify any instance types in the console or using other tools that use the Amazon EKS API. If you don't specify an instance type in a launch template, in the console, or using other tools that use the Amazon EKS API, the `t3.medium` instance type is used. If your node group is using the Spot capacity type, then we recommend specifying multiple instance types using the console. For more information, see <>. +* If any containers that you deploy to the node group use the Instance Metadata Service Version 2, make sure to set the *Metadata response hop limit* to `2` in your launch template. For more information, see link:AWSEC2/latest/UserGuide/ec2-instance-metadata.html[Instance metadata and user data,type="documentation"] in the _Amazon EC2 User Guide_. +* Launch templates do not support the `InstanceRequirements` feature that allows flexible instance type selection. ==== -[[launch-template-tagging,launch-template-tagging.title]] +[#launch-template-tagging] == Tagging Amazon EC2 instances You can use the `TagSpecification` parameter of a launch template to specify which tags to apply to Amazon EC2 instances in your node group. The IAM entity calling the `CreateNodegroup` or `UpdateNodegroupVersion` APIs must have permissions for `ec2:RunInstances` and `ec2:CreateTags`, and the tags must be added to the launch template. -[[launch-template-security-groups,launch-template-security-groups.title]] +[#launch-template-security-groups] == Using custom security groups You can use a launch template to specify custom Amazon EC2 link:AWSEC2/latest/UserGuide/ec2-security-groups.html[security groups,type="documentation"] to apply to instances in your node group. This can be either in the instance level security groups parameter or as part of the network interface configuration parameters. However, you can't create a launch template that specifies both instance level and network interface security groups. Consider the following conditions that apply to using custom security groups with managed node groups: - - * When using the {aws-management-console}, Amazon EKS only allows launch templates with a single network interface specification. * By default, Amazon EKS applies the <> to the instances in your node group to facilitate communication between nodes and the control plane. If you specify custom security groups in the launch template using either option mentioned earlier, Amazon EKS doesn't add the cluster security group. So, you must ensure that the inbound and outbound rules of your security groups enable communication with the endpoint of your cluster. If your security group rules are incorrect, the worker nodes can't join the cluster. For more information about security group rules, see <>. * If you need SSH access to the instances in your node group, include a security group that allows that access. - -[[launch-template-user-data,launch-template-user-data.title]] +[#launch-template-user-data] == Amazon EC2 user data -The launch template includes a section for custom user data. You can specify configuration settings for your node group in this section without manually creating individual custom AMIs. For more information about the settings available for [.noloc]`Bottlerocket`, see https://github.com/bottlerocket-os/bottlerocket#using-user-data[Using user data] on [.noloc]`GitHub`. +The launch template includes a section for custom user data. You can specify configuration settings for your node group in this section without manually creating individual custom AMIs. For more information about the settings available for Bottlerocket, see https://github.com/bottlerocket-os/bottlerocket#using-user-data[Using user data] on GitHub. You can supply Amazon EC2 user data in your launch template using `cloud-init` when launching your instances. For more information, see the https://cloudinit.readthedocs.io/en/latest/index.html[cloud-init] documentation. Your user data can be used to perform common configuration operations. This includes the following operations: @@ -118,7 +107,7 @@ You can supply Amazon EC2 user data in your launch template using `cloud-init` w * https://cloudinit.readthedocs.io/en/latest/topics/examples.html#including-users-and-groups[Including users or groups] * https://cloudinit.readthedocs.io/en/latest/topics/examples.html#install-arbitrary-packages[Installing packages] -Amazon EC2 user data in launch templates that are used with managed node groups must be in the https://cloudinit.readthedocs.io/en/latest/topics/format.html#mime-multi-part-archive[MIME multi-part archive] format for Amazon Linux AMIs and TOML format for [.noloc]`Bottlerocket` AMIs. This is because your user data is merged with Amazon EKS user data required for nodes to join the cluster. Don't specify any commands in your user data that starts or modifies `kubelet`. This is performed as part of the user data merged by Amazon EKS. Certain `kubelet` parameters, such as setting labels on nodes, can be configured directly through the managed node groups API. +Amazon EC2 user data in launch templates that are used with managed node groups must be in the https://cloudinit.readthedocs.io/en/latest/topics/format.html#mime-multi-part-archive[MIME multi-part archive] format for Amazon Linux AMIs and TOML format for Bottlerocket AMIs. This is because your user data is merged with Amazon EKS user data required for nodes to join the cluster. Don't specify any commands in your user data that starts or modifies `kubelet`. This is performed as part of the user data merged by Amazon EKS. Certain `kubelet` parameters, such as setting labels on nodes, can be configured directly through the managed node groups API. [NOTE] ==== @@ -132,7 +121,7 @@ The following details provide more information about the user data section. *Amazon Linux 2 user data*:: -You can combine multiple user data blocks together into a single MIME multi-part file. For example, you can combine a cloud boothook that configures the [.noloc]`Docker` daemon with a user data shell script that installs a custom package. A MIME multi-part file consists of the following components: +You can combine multiple user data blocks together into a single MIME multi-part file. For example, you can combine a cloud boothook that configures the Docker daemon with a user data shell script that installs a custom package. A MIME multi-part file consists of the following components: + ** The content type and part boundary declaration – `Content-Type: multipart/mixed; boundary="==MYBOUNDARY=="` ** The MIME version declaration – `MIME-Version: 1.0` @@ -145,6 +134,7 @@ You can combine multiple user data blocks together into a single MIME multi-part + The following is an example of a MIME multi-part file that you can use to create your own. + + [source,none,subs="verbatim,attributes"] ---- @@ -194,11 +184,64 @@ kind: NodeConfig spec: [...] --BOUNDARY-- ---- + -In AL2, the metadata from these parameters was discovered from the Amazon EKS `DescribeCluster` API call. With AL2023, this behavior has changed since the additional API call risks throttling during large node scale ups. This change doesn't affect you if you're using managed node groups without a launch template or if you're using [.noloc]`Karpenter`. For more information on `certificateAuthority` and service `cidr`, see ` link:eks/latest/APIReference/API_DescribeCluster.html[DescribeCluster,type="documentation"]` in the _Amazon EKS API Reference_. +In AL2, the metadata from these parameters was discovered from the Amazon EKS `DescribeCluster` API call. With AL2023, this behavior has changed since the additional API call risks throttling during large node scale ups. This change doesn't affect you if you're using managed node groups without a launch template or if you're using Karpenter. For more information on `certificateAuthority` and service `cidr`, see link:eks/latest/APIReference/API_DescribeCluster.html[`DescribeCluster`,type="documentation"] in the _Amazon EKS API Reference_. ++ +Here's a complete example of AL2023 user data that combines a shell script for customizing the node (like installing packages or pre-caching container images) with the required `nodeadm` configuration. This example shows common customizations including: +* Installing additional system packages +* Pre-caching container images to improve Pod startup time +* Setting up HTTP proxy configuration +* Configuring `kubelet` flags for node labeling + ++ +[source,bash,subs="verbatim,attributes"] +---- +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="BOUNDARY" + +--BOUNDARY +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash +set -o errexit +set -o pipefail +set -o nounset + +# Install additional packages +yum install -y htop jq iptables-services + +# Pre-cache commonly used container images +nohup docker pull public.ecr.aws/eks-distro/kubernetes/pause:3.2 & +# Configure HTTP proxy if needed +cat > /etc/profile.d/http-proxy.sh << 'EOF' +export HTTP_PROXY="http://proxy.example.com:3128" +export HTTPS_PROXY="http://proxy.example.com:3128" +export NO_PROXY="localhost,127.0.0.1,169.254.169.254,.internal" +EOF + +--BOUNDARY +Content-Type: application/node.eks.aws -*[.noloc]`Bottlerocket` user data*:: -[.noloc]`Bottlerocket` structures user data in the TOML format. You can provide user data to be merged with the user data provided by Amazon EKS. For example, you can provide additional `kubelet` settings. +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: my-cluster + apiServerEndpoint: https://example.com + certificateAuthority: Y2VydGlmaWNhdGVBdXRob3JpdHk= + cidr: 10.100.0.0/16 + kubelet: + config: + clusterDNS: + - 10.100.0.10 + flags: + - --node-labels=app=my-app,environment=production + +--BOUNDARY-- +---- + +*Bottlerocket user data*:: +Bottlerocket structures user data in the TOML format. You can provide user data to be merged with the user data provided by Amazon EKS. For example, you can provide additional `kubelet` settings. + [source,none,subs="verbatim,attributes"] ---- @@ -221,8 +264,10 @@ Amazon EKS doesn't support all valid TOML. The following is a list of known unsu ** Bracketed headers with quoted keys: `[foo."bar.baz"]` -*[.noloc]`Windows` user data*:: -Windows user data uses [.noloc]`PowerShell` commands. When creating a managed node group, your custom user data combines with Amazon EKS managed user data. Your [.noloc]`PowerShell` commands come first, followed by the managed user data commands, all within one `` tag. +*Windows user data*:: +Windows user data uses PowerShell commands. When creating a managed node group, your custom user data combines with Amazon EKS managed user data. Your PowerShell commands come first, followed by the managed user data commands, all within one `` tag. ++ +IMPORTANT: When creating Windows node groups, Amazon EKS updates the `aws-auth` `ConfigMap` to allow Linux-based nodes to join the cluster. The service doesn't automatically configure permissions for Windows AMIs. If you're using Windows nodes, you'll need to manage access either via the access entry API or by updating the `aws-auth` `ConfigMap` directly. For more information, see <>. + NOTE: When no AMI ID is specified in the launch template, don't use the Windows Amazon EKS Bootstrap script in user data to configure Amazon EKS. + @@ -236,14 +281,14 @@ Write-Host "Running custom user data script" ---- -[[launch-template-custom-ami,launch-template-custom-ami.title]] +[#launch-template-custom-ami] == Specifying an AMI If you have either of the following requirements, then specify an AMI ID in the `ImageId` field of your launch template. Select the requirement you have for additional information. -[[mng-specify-eks-ami,mng-specify-eks-ami.title]] -.Provide user data to pass arguments to the `bootstrap.sh` file included with an Amazon EKS optimized [.noloc]`Linux`/[.noloc]`Bottlerocket` AMI +[#mng-specify-eks-ami] +.Provide user data to pass arguments to the `bootstrap.sh` file included with an Amazon EKS optimized Linux/Bottlerocket AMI [%collapsible] ==== @@ -251,11 +296,11 @@ Bootstrapping is a term used to describe adding commands that can be run when an -*[.noloc]`eksctl` without specifying a launch template*:: -Create a file named [.replaceable]`my-nodegroup.yaml` with the following contents. Replace every [.replaceable]`example value` with your own values. The `--apiserver-endpoint`, `--b64-cluster-ca`, and `--dns-cluster-ip` arguments are optional. However, defining them allows the `bootstrap.sh` script to avoid making a `describeCluster` call. This is useful in private cluster setups or clusters where you're scaling in and out nodes frequently. For more information on the `bootstrap.sh` script, see the https://github.com/awslabs/amazon-eks-ami/blob/main/templates/al2/runtime/bootstrap.sh[bootstrap.sh] file on [.noloc]`GitHub`. +*eksctl without specifying a launch template*:: +Create a file named [.replaceable]`my-nodegroup.yaml` with the following contents. Replace every [.replaceable]`example value` with your own values. The `--apiserver-endpoint`, `--b64-cluster-ca`, and `--dns-cluster-ip` arguments are optional. However, defining them allows the `bootstrap.sh` script to avoid making a `describeCluster` call. This is useful in private cluster setups or clusters where you're scaling in and out nodes frequently. For more information on the `bootstrap.sh` script, see the https://github.com/awslabs/amazon-eks-ami/blob/main/templates/al2/runtime/bootstrap.sh[bootstrap.sh] file on GitHub. + ** The only required argument is the cluster name ([.replaceable]`my-cluster`). -** To retrieve an optimized AMI ID for `ami-[.replaceable]``1234567890abcdef0```, you can use the tables in the following sections: +** To retrieve an optimized AMI ID for `ami-[.replaceable]``1234567890abcdef0```, see the following sections: + *** <> *** <> @@ -318,7 +363,7 @@ eksctl create nodegroup --config-file=my-nodegroup.yaml *User data in a launch template*:: -Specify the following information in the user data section of your launch template. Replace every [.replaceable]`example value` with your own values. The `--apiserver-endpoint`, `--b64-cluster-ca`, and `--dns-cluster-ip` arguments are optional. However, defining them allows the `bootstrap.sh` script to avoid making a `describeCluster` call. This is useful in private cluster setups or clusters where you're scaling in and out nodes frequently. For more information on the `bootstrap.sh` script, see the https://github.com/awslabs/amazon-eks-ami/blob/main/templates/al2/runtime/bootstrap.sh[bootstrap.sh] file on [.noloc]`GitHub`. +Specify the following information in the user data section of your launch template. Replace every [.replaceable]`example value` with your own values. The `--apiserver-endpoint`, `--b64-cluster-ca`, and `--dns-cluster-ip` arguments are optional. However, defining them allows the `bootstrap.sh` script to avoid making a `describeCluster` call. This is useful in private cluster setups or clusters where you're scaling in and out nodes frequently. For more information on the `bootstrap.sh` script, see the https://github.com/awslabs/amazon-eks-ami/blob/main/templates/al2/runtime/bootstrap.sh[bootstrap.sh] file on GitHub. + ** The only required argument is the cluster name ([.replaceable]`my-cluster`). ** To retrieve the [.replaceable]`certificate-authority` for your cluster, run the following command. @@ -363,18 +408,18 @@ set -ex ==== -[[mng-specify-eks-ami-windows,mng-specify-eks-ami-windows.title]] -.Provide user data to pass arguments to the `Start-EKSBootstrap.ps1` file included with an Amazon EKS optimized [.noloc]`Windows` AMI +[#mng-specify-eks-ami-windows] +.Provide user data to pass arguments to the `Start-EKSBootstrap.ps1` file included with an Amazon EKS optimized Windows AMI [%collapsible] ==== Bootstrapping is a term used to describe adding commands that can be run when an instance starts. You can pass arguments to the `Start-EKSBootstrap.ps1` script by using `eksctl` without specifying a launch template. Or you can do so by specifying the information in the user data section of a launch template. -If you want to specify a custom [.noloc]`Windows` AMI ID, keep in mind the following considerations: +If you want to specify a custom Windows AMI ID, keep in mind the following considerations: -* You must use a launch template and give the required bootstrap commands in the user data section. To retrieve your desired [.noloc]`Windows` ID, you can use the table in <>. +* You must use a launch template and give the required bootstrap commands in the user data section. To retrieve your desired Windows ID, you can use the table in <>. * There are several limits and conditions. For example, you must add `eks:kube-proxy-windows` to your {aws} IAM Authenticator configuration map. For more information, see <>. Specify the following information in the user data section of your launch template. Replace every [.replaceable]`example value` with your own values. The `-APIServerEndpoint`, `-Base64ClusterCA`, and `-DNSClusterIP` arguments are optional. However, defining them allows the `Start-EKSBootstrap.ps1` script to avoid making a `describeCluster` call. @@ -402,7 +447,7 @@ aws eks describe-cluster --query "cluster.kubernetesNetworkConfig.serviceIpv4Cid ---- * For additional arguments, see <>. + -NOTE: If you're using custom service CIDR, then you need to specify it using the `-ServiceCIDR` parameter. Otherwise, the DNS resolution for [.noloc]`Pods` in the cluster will fail. +NOTE: If you're using custom service CIDR, then you need to specify it using the `-ServiceCIDR` parameter. Otherwise, the DNS resolution for Pods in the cluster will fail. [source,xml,subs="verbatim,attributes"] @@ -417,12 +462,15 @@ NOTE: If you're using custom service CIDR, then you need to specify it using the ---- ==== -[[mng-specify-custom-ami,mng-specify-custom-ami.title]] +[#mng-specify-custom-ami] .Run a custom AMI due to specific security, compliance, or internal policy requirements [%collapsible] ==== -For more information, see link:AWSEC2/latest/UserGuide/AMIs.html[Amazon Machine Images (AMI),type="documentation"] in the _Amazon EC2 User Guide_. The Amazon EKS AMI build specification contains resources and configuration scripts for building a custom Amazon EKS AMI based on Amazon Linux. For more information, see https://github.com/awslabs/amazon-eks-ami/[Amazon EKS AMI Build Specification] on [.noloc]`GitHub`. To build custom AMIs installed with other operating systems, see https://github.com/aws-samples/amazon-eks-custom-amis[Amazon EKS Sample Custom AMIs] on [.noloc]`GitHub`. +For more information, see link:AWSEC2/latest/UserGuide/AMIs.html[Amazon Machine Images (AMI),type="documentation"] in the _Amazon EC2 User Guide_. The Amazon EKS AMI build specification contains resources and configuration scripts for building a custom Amazon EKS AMI based on Amazon Linux. For more information, see https://github.com/awslabs/amazon-eks-ami/[Amazon EKS AMI Build Specification] on GitHub. To build custom AMIs installed with other operating systems, see https://github.com/aws-samples/amazon-eks-custom-amis[Amazon EKS Sample Custom AMIs] on GitHub. + +You cannot use dynamic parameter references for AMI IDs in Launch Templates used with managed node groups. + ==== [IMPORTANT] @@ -432,13 +480,11 @@ When specifying an AMI, Amazon EKS doesn't merge any user data. Rather, you're r ==== -[[mng-ami-id-conditions,mng-ami-id-conditions.title]] +[#mng-ami-id-conditions] == Limits and conditions when specifying an AMI ID The following are the limits and conditions involved with specifying an AMI ID with managed node groups: - - * You must create a new node group to switch between specifying an AMI ID in a launch template and not specifying an AMI ID. * You aren't notified in the console when a newer AMI version is available. To update your node group to a newer AMI version, you need to create a new version of your launch template with an updated AMI ID. Then, you need to update the node group with the new launch template version. * The following fields can't be set in the API if you specify an AMI ID: @@ -446,8 +492,8 @@ The following are the limits and conditions involved with specifying an AMI ID w ** `amiType` ** `releaseVersion` ** `version` -* Any `taints` set in the API are applied asynchronously if you specify an AMI ID. To apply taints prior to a node joining the cluster, you must pass the taints to `kubelet` in your user data using the `--register-with-taints` command line flag. For more information, see https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/[kubelet] in the [.noloc]`Kubernetes` documentation. -* When specifying a custom AMI ID for [.noloc]`Windows` managed node groups, add `eks:kube-proxy-windows` to your {aws} IAM Authenticator configuration map. This is required for DNS to function properly. +* Any `taints` set in the API are applied asynchronously if you specify an AMI ID. To apply taints prior to a node joining the cluster, you must pass the taints to `kubelet` in your user data using the `--register-with-taints` command line flag. For more information, see https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/[kubelet] in the Kubernetes documentation. +* When specifying a custom AMI ID for Windows managed node groups, add `eks:kube-proxy-windows` to your {aws} IAM Authenticator configuration map. This is required for DNS to function properly. + .. Open the {aws} IAM Authenticator configuration map for editing. + @@ -455,10 +501,11 @@ The following are the limits and conditions involved with specifying an AMI ID w ---- kubectl edit -n kube-system cm aws-auth ---- -.. Add this entry to the `groups` list under each `rolearn` associated with [.noloc]`Windows` nodes. Your configuration map should look similar to https://s3.us-west-2.amazonaws.com/amazon-eks/cloudformation/2020-10-29/aws-auth-cm-windows.yaml[aws-auth-cm-windows.yaml]. +.. Add this entry to the `groups` list under each `rolearn` associated with Windows nodes. Your configuration map should look similar to https://s3.us-west-2.amazonaws.com/amazon-eks/cloudformation/2020-10-29/aws-auth-cm-windows.yaml[aws-auth-cm-windows.yaml]. + [source,yaml,subs="verbatim,attributes"] ---- - eks:kube-proxy-windows ---- .. Save the file and exit your text editor. +* For any AMI that uses a custom launch template, the default `HttpPutResponseHopLimit` for managed node groups is set to `2`. diff --git a/latest/ug/nodes/launch-windows-workers.adoc b/latest/ug/nodes/launch-windows-workers.adoc index 24b334685..881da214f 100644 --- a/latest/ug/nodes/launch-windows-workers.adoc +++ b/latest/ug/nodes/launch-windows-workers.adoc @@ -1,17 +1,16 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[launch-windows-workers,launch-windows-workers.title]] -= Create self-managed [.noloc]`Microsoft Windows` nodes +[#launch-windows-workers] += Create self-managed Microsoft Windows nodes :info_titleabbrev: Windows -include::../attributes.txt[] - [abstract] -- -This topic describes how to launch Auto Scaling groups of [.noloc]`Windows` nodes that register with your Amazon EKS cluster. +This topic describes how to launch Auto Scaling groups of Windows nodes that register with your Amazon EKS cluster. -- -This topic describes how to launch Auto Scaling groups of [.noloc]`Windows` nodes that register with your Amazon EKS cluster. After the nodes join the cluster, you can deploy [.noloc]`Kubernetes` applications to them. +This topic describes how to launch Auto Scaling groups of Windows nodes that register with your Amazon EKS cluster. After the nodes join the cluster, you can deploy Kubernetes applications to them. [IMPORTANT] ==== @@ -21,9 +20,9 @@ This topic describes how to launch Auto Scaling groups of [.noloc]`Windows` node ==== -Enable [.noloc]`Windows` support for your cluster. We recommend that you review important considerations before you launch a [.noloc]`Windows` node group. For more information, see <>. +Enable Windows support for your cluster. We recommend that you review important considerations before you launch a Windows node group. For more information, see <>. -You can launch self-managed [.noloc]`Windows` nodes with either of the following: +You can launch self-managed Windows nodes with either of the following: * <> * <> @@ -46,10 +45,10 @@ For instructions on how to install or upgrade `eksctl`, see https://eksctl.io/in This procedure only works for clusters that were created with `eksctl`. ==== -. (Optional) If the *AmazonEKS_CNI_Policy* managed IAM policy (if you have an `IPv4` cluster) or the [.replaceable]`AmazonEKS_CNI_IPv6_Policy` (that you <> if you have an `IPv6` cluster) is attached to your <>, we recommend assigning it to an IAM role that you associate to the [.noloc]`Kubernetes` `aws-node` service account instead. For more information, see <>. -. This procedure assumes that you have an existing cluster. If you don't already have an Amazon EKS cluster and an Amazon Linux node group to add a [.noloc]`Windows` node group to, we recommend that you follow <>. This guide provides a complete walkthrough for how to create an Amazon EKS cluster with Amazon Linux nodes. +. (Optional) If the *AmazonEKS_CNI_Policy* managed IAM policy (if you have an `IPv4` cluster) or the [.replaceable]`AmazonEKS_CNI_IPv6_Policy` (that you <> if you have an `IPv6` cluster) is attached to your <>, we recommend assigning it to an IAM role that you associate to the Kubernetes `aws-node` service account instead. For more information, see <>. +. This procedure assumes that you have an existing cluster. If you don't already have an Amazon EKS cluster and an Amazon Linux node group to add a Windows node group to, we recommend that you follow <>. This guide provides a complete walkthrough for how to create an Amazon EKS cluster with Amazon Linux nodes. + -Create your node group with the following command. Replace [.replaceable]`region-code` with the {aws} Region that your cluster is in. Replace [.replaceable]`my-cluster` with your cluster name. The name can contain only alphanumeric characters (case-sensitive) and hyphens. It must start with an alphanumeric character and can't be longer than 100 characters. The name must be unique within the {aws} Region and {aws} account that you're creating the cluster in. Replace [.replaceable]`ng-windows` with a name for your node group. The node group name can't be longer than 63 characters. It must start with letter or digit, but can also include hyphens and underscores for the remaining characters. For [.noloc]`Kubernetes` version `1.24` or later, you can replace [.replaceable]`2019` with `2022` to use [.noloc]`Windows` Server 2022. Replace the rest of the [.replaceable]`example values` with your own values. +Create your node group with the following command. Replace [.replaceable]`region-code` with the {aws} Region that your cluster is in. Replace [.replaceable]`my-cluster` with your cluster name. The name can contain only alphanumeric characters (case-sensitive) and hyphens. It must start with an alphanumeric character and can't be longer than 100 characters. The name must be unique within the {aws} Region and {aws} account that you're creating the cluster in. Replace [.replaceable]`ng-windows` with a name for your node group. The node group name can't be longer than 63 characters. It must start with letter or digit, but can also include hyphens and underscores for the remaining characters. You can replace [.replaceable]`2019` with `2022` to use Windows Server 2022. Replace the rest of the example values with your own values. + IMPORTANT: To deploy a node group to {aws} Outposts, {aws} Wavelength, or {aws} Local Zone subnets, don't pass the {aws} Outposts, Wavelength, or Local Zone subnets when you create the cluster. Create the node group with a config file, specifying the {aws} Outposts, Wavelength, or Local Zone subnets. For more information, see https://eksctl.io/usage/nodegroups/#creating-a-nodegroup-from-a-config-file[Create a nodegroup from a config file] and https://eksctl.io/usage/schema/[Config file schema] in the `eksctl` documentation. + @@ -84,11 +83,11 @@ An example output is as follows. Several lines are output while the nodes are cr ---- [✔] created 1 nodegroup(s) in cluster "my-cluster" ---- -. (Optional) Deploy a <> to test your cluster and [.noloc]`Windows` nodes. -. We recommend blocking [.noloc]`Pod` access to IMDS if the following conditions are true: +. (Optional) Deploy a <> to test your cluster and Windows nodes. +. We recommend blocking Pod access to IMDS if the following conditions are true: + -** You plan to assign IAM roles to all of your [.noloc]`Kubernetes` service accounts so that [.noloc]`Pods` only have the minimum permissions that they need. -** No [.noloc]`Pods` in the cluster require access to the Amazon EC2 instance metadata service (IMDS) for other reasons, such as retrieving the current {aws} Region. +** You plan to assign IAM roles to all of your Kubernetes service accounts so that Pods only have the minimum permissions that they need. +** No Pods in the cluster require access to the Amazon EC2 instance metadata service (IMDS) for other reasons, such as retrieving the current {aws} Region. + For more information, see https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node[Restrict access to the instance profile assigned to the worker node]. @@ -97,7 +96,7 @@ For more information, see https://aws.github.io/aws-eks-best-practices/security/ *Prerequisites* -** An existing Amazon EKS cluster and a [.noloc]`Linux` node group. If you don't have these resources, we recommend that you create them using one of our guides in <>. These guides describe how to create an Amazon EKS cluster with [.noloc]`Linux` nodes. +** An existing Amazon EKS cluster and a Linux node group. If you don't have these resources, we recommend that you create them using one of our guides in <>. These guides describe how to create an Amazon EKS cluster with Linux nodes. ** An existing VPC and security group that meet the requirements for an Amazon EKS cluster. For more information, see <> and <>. The guides in <> create a VPC that meets the requirements. Alternatively, you can also follow <> to create one manually. ** An existing Amazon EKS cluster that uses a VPC and security group that meets the requirements of an Amazon EKS cluster. For more information, see <>. If you have subnets in the {aws} Region where you have {aws} Outposts, {aws} Wavelength, or {aws} Local Zones enabled, those subnets must not have been passed in when you created the cluster. @@ -106,7 +105,7 @@ For more information, see https://aws.github.io/aws-eks-best-practices/security/ . Wait for your cluster status to show as `ACTIVE`. If you launch your nodes before the cluster is active, the nodes fail to register with the cluster and you need to relaunch them. . Open the link:cloudformation/[{aws} CloudFormation console,type="console"] . Choose *Create stack*. -. For *Specify template*, select *Amazon S3 URL*. +. For *Specify template*, select *Amazon S3 URL*. . Copy the following URL and paste it into *Amazon S3 URL*. + [source,none,subs="verbatim,attributes"] @@ -116,48 +115,48 @@ https://s3.us-west-2.amazonaws.com/amazon-eks/cloudformation/2023-02-09/amazon-e . Select *Next* twice. . On the *Quick create stack* page, enter the following parameters accordingly: + -** *Stack name*: Choose a stack name for your {aws} CloudFormation stack. For example, you can call it [.replaceable]`my-cluster`-nodes````. +** *Stack name*: Choose a stack name for your {aws} CloudFormation stack. For example, you can call it `my-cluster-nodes`. ** *ClusterName*: Enter the name that you used when you created your Amazon EKS cluster. + [IMPORTANT] ==== This name must exactly match the name that you used in <>. Otherwise, your nodes can't join the cluster. ==== -** *ClusterControlPlaneSecurityGroup*: Choose the security group from the {aws} CloudFormation output that you generated when you created your <>. +** *ClusterControlPlaneSecurityGroup*: Choose the security group from the {aws} CloudFormation output that you generated when you created your <>. The following steps show one method to retrieve the applicable group. + .. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. .. Choose the name of the cluster. .. Choose the *Networking* tab. -.. Use the *Additional security groups* value as a reference when selecting from the *ClusterControlPlaneSecurityGroup* dropdown list. +.. Use the *Additional security groups* value as a reference when selecting from the *ClusterControlPlaneSecurityGroup* dropdown list. ** *NodeGroupName*: Enter a name for your node group. This name can be used later to identify the Auto Scaling node group that's created for your nodes. The node group name can't be longer than 63 characters. It must start with letter or digit, but can also include hyphens and underscores for the remaining characters. ** *NodeAutoScalingGroupMinSize*: Enter the minimum number of nodes that your node Auto Scaling group can scale in to. ** *NodeAutoScalingGroupDesiredCapacity*: Enter the desired number of nodes to scale to when your stack is created. ** *NodeAutoScalingGroupMaxSize*: Enter the maximum number of nodes that your node Auto Scaling group can scale out to. ** *NodeInstanceType*: Choose an instance type for your nodes. For more information, see <>. + -NOTE: The supported instance types for the latest version of the https://github.com/aws/amazon-vpc-cni-k8s[Amazon VPC CNI plugin for Kubernetes] are listed in https://github.com/aws/amazon-vpc-cni-k8s/blob/master/pkg/vpc/vpc_ip_resource_limit.go[vpc_ip_resource_limit.go] on [.noloc]`GitHub`. You might need to update your CNI version to use the latest supported instance types. For more information, see <>. -** *NodeImageIdSSMParam*: Pre-populated with the Amazon EC2 Systems Manager parameter of the current recommended Amazon EKS optimized [.noloc]`Windows` Core AMI ID. To use the full version of [.noloc]`Windows`, replace [.replaceable]`Core` with `Full`. -** *NodeImageId*: (Optional) If you're using your own custom AMI (instead of an Amazon EKS optimized AMI), enter a node AMI ID for your {aws} Region. If you specify a value for this field, it overrides any values in the *NodeImageIdSSMParam* field. +NOTE: The supported instance types for the latest version of the https://github.com/aws/amazon-vpc-cni-k8s[Amazon VPC CNI plugin for Kubernetes] are listed in https://github.com/aws/amazon-vpc-cni-k8s/blob/master/pkg/vpc/vpc_ip_resource_limit.go[vpc_ip_resource_limit.go] on GitHub. You might need to update your CNI version to use the latest supported instance types. For more information, see <>. +** *NodeImageIdSSMParam*: Pre-populated with the Amazon EC2 Systems Manager parameter of the current recommended Amazon EKS optimized Windows Core AMI ID. To use the full version of Windows, replace [.replaceable]`Core` with `Full`. +** *NodeImageId*: (Optional) If you're using your own custom AMI (instead of an Amazon EKS optimized AMI), enter a node AMI ID for your {aws} Region. If you specify a value for this field, it overrides any values in the *NodeImageIdSSMParam* field. ** *NodeVolumeSize*: Specify a root volume size for your nodes, in GiB. ** *KeyName*: Enter the name of an Amazon EC2 SSH key pair that you can use to connect using SSH into your nodes with after they launch. If you don't already have an Amazon EC2 key pair, you can create one in the {aws-management-console}. For more information, see link:AWSEC2/latest/WindowsGuide/ec2-key-pairs.html[Amazon EC2 key pairs,type="documentation"] in the _Amazon EC2 User Guide_. + NOTE: If you don't provide a key pair here, the {aws} CloudFormation stack fails to be created. ** *BootstrapArguments*: Specify any optional arguments to pass to the node bootstrap script, such as extra `kubelet` arguments using `-KubeletExtraArgs`. -** *DisableIMDSv1*: By default, each node supports the Instance Metadata Service Version 1 (IMDSv1) and IMDSv2. You can disable IMDSv1. To prevent future nodes and [.noloc]`Pods` in the node group from using MDSv1, set *DisableIMDSv1* to *true*. For more information about IMDS, see link:AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html[Configuring the instance metadata service,type="documentation"]. -** *VpcId*: Select the ID for the <> that you created. -** *NodeSecurityGroups*: Select the security group that was created for your [.noloc]`Linux` node group when you created your <>. If your [.noloc]`Linux` nodes have more than one security group attached to them, specify all of them. This for, for example, if the [.noloc]`Linux` node group was created with `eksctl`. -** *Subnets*: Choose the subnets that you created. If you created your VPC using the steps in <>, then specify only the private subnets within the VPC for your nodes to launch into. +** *DisableIMDSv1*: By default, each node supports the Instance Metadata Service Version 1 (IMDSv1) and IMDSv2. You can disable IMDSv1. To prevent future nodes and Pods in the node group from using MDSv1, set *DisableIMDSv1* to *true*. For more information about IMDS, see link:AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html[Configuring the instance metadata service,type="documentation"]. +** *VpcId*: Select the ID for the <> that you created. +** *NodeSecurityGroups*: Select the security group that was created for your Linux node group when you created your <>. If your Linux nodes have more than one security group attached to them, specify all of them. This for, for example, if the Linux node group was created with `eksctl`. +** *Subnets*: Choose the subnets that you created. If you created your VPC using the steps in <>, then specify only the private subnets within the VPC for your nodes to launch into. + [IMPORTANT] ==== -*** If any of the subnets are public subnets, then they must have the automatic public IP address assignment setting enabled. If the setting isn't enabled for the public subnet, then any nodes that you deploy to that public subnet won't be assigned a public IP address and won't be able to communicate with the cluster or other {aws} services. If the subnet was deployed before March 26, 2020 using either of the <>, or by using `eksctl`, then automatic public IP address assignment is disabled for public subnets. For information about how to enable public IP address assignment for a subnet, see link:vpc/latest/userguide/vpc-ip-addressing.html#subnet-public-ip[Modifying the public IPv4 addressing attribute for your subnet,type="documentation"]. If the node is deployed to a private subnet, then it's able to communicate with the cluster and other {aws} services through a NAT gateway. +*** If any of the subnets are public subnets, then they must have the automatic public IP address assignment setting enabled. If the setting isn't enabled for the public subnet, then any nodes that you deploy to that public subnet won't be assigned a public IP address and won't be able to communicate with the cluster or other {aws} services. If the subnet was deployed before March 26, 2020 using either of the <>, or by using `eksctl`, then automatic public IP address assignment is disabled for public subnets. For information about how to enable public IP address assignment for a subnet, see link:vpc/latest/userguide/vpc-ip-addressing.html#subnet-public-ip[Modifying the public IPv4 addressing attribute for your subnet,type="documentation"]. If the node is deployed to a private subnet, then it's able to communicate with the cluster and other {aws} services through a NAT gateway. *** If the subnets don't have internet access, then make sure that you're aware of the considerations and extra steps in <>. *** If you select {aws} Outposts, Wavelength, or Local Zone subnets, then the subnets must not have been passed in when you created the cluster. ==== . Acknowledge that the stack might create IAM resources, and then choose *Create stack*. . When your stack has finished creating, select it in the console and choose *Outputs*. -. Record the *NodeInstanceRole* for the node group that was created. You need this when you configure your Amazon EKS [.noloc]`Windows` nodes. +. Record the *NodeInstanceRole* for the node group that was created. You need this when you configure your Amazon EKS Windows nodes. *Step 2: Enable nodes to join your cluster* @@ -204,7 +203,7 @@ data: ---- curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/cloudformation/2020-10-29/aws-auth-cm-windows.yaml ---- -.. In the `aws-auth-cm-windows.yaml` file, set the `rolearn` values to the applicable *NodeInstanceRole* values that you recorded in the previous procedures. You can do this with a text editor, or by replacing the [.replaceable]`example values` and running the following command: +.. In the `aws-auth-cm-windows.yaml` file, set the `rolearn` values to the applicable *NodeInstanceRole* values that you recorded in the previous procedures. You can do this with a text editor, or by replacing the example values and running the following command: + [source,bash,subs="verbatim,attributes"] ---- @@ -215,7 +214,7 @@ sed -i.bak -e 's||my-node-lin [IMPORTANT] ==== *** Don't modify any other lines in this file. -*** Don't use the same IAM role for both [.noloc]`Windows` and [.noloc]`Linux` nodes. +*** Don't use the same IAM role for both Windows and Linux nodes. ==== .. Apply the configuration. This command might take a few minutes to finish. + @@ -238,12 +237,12 @@ If nodes fail to join the cluster, then see <> in the Troubles *Step 3: Additional actions* -. (Optional) Deploy a <> to test your cluster and [.noloc]`Windows` nodes. -. (Optional) If the *AmazonEKS_CNI_Policy* managed IAM policy (if you have an `IPv4` cluster) or the [.replaceable]`AmazonEKS_CNI_IPv6_Policy` (that you <> if you have an `IPv6` cluster) is attached to your <>, we recommend assigning it to an IAM role that you associate to the [.noloc]`Kubernetes` `aws-node` service account instead. For more information, see <>. -. We recommend blocking [.noloc]`Pod` access to IMDS if the following conditions are true: +. (Optional) Deploy a <> to test your cluster and Windows nodes. +. (Optional) If the *AmazonEKS_CNI_Policy* managed IAM policy (if you have an `IPv4` cluster) or the [.replaceable]`AmazonEKS_CNI_IPv6_Policy` (that you <> if you have an `IPv6` cluster) is attached to your <>, we recommend assigning it to an IAM role that you associate to the Kubernetes `aws-node` service account instead. For more information, see <>. +. We recommend blocking Pod access to IMDS if the following conditions are true: + -** You plan to assign IAM roles to all of your [.noloc]`Kubernetes` service accounts so that [.noloc]`Pods` only have the minimum permissions that they need. -** No [.noloc]`Pods` in the cluster require access to the Amazon EC2 instance metadata service (IMDS) for other reasons, such as retrieving the current {aws} Region. +** You plan to assign IAM roles to all of your Kubernetes service accounts so that Pods only have the minimum permissions that they need. +** No Pods in the cluster require access to the Amazon EC2 instance metadata service (IMDS) for other reasons, such as retrieving the current {aws} Region. + -For more information, see https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node[Restrict access to the instance profile assigned to the worker node]. +For more information, see https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node[Restrict access to the instance profile assigned to the worker node]. \ No newline at end of file diff --git a/latest/ug/nodes/launch-workers.adoc b/latest/ug/nodes/launch-workers.adoc index 2156be5c7..3fd6b3aa4 100644 --- a/latest/ug/nodes/launch-workers.adoc +++ b/latest/ug/nodes/launch-workers.adoc @@ -1,20 +1,20 @@ -//!!NODE_ROOT
include::../attributes.txt[] + [.topic] -[[launch-workers,launch-workers.title]] +[#launch-workers] = Create self-managed Amazon Linux nodes :info_titleabbrev: Amazon Linux [abstract] -- -This topic describes how you can launch Auto Scaling groups of [.noloc]`Linux` nodes that register with your Amazon EKS cluster. +This topic describes how you can launch Auto Scaling groups of Linux nodes that register with your Amazon EKS cluster. -- -This topic describes how you can launch Auto Scaling groups of [.noloc]`Linux` nodes that register with your Amazon EKS cluster. After the nodes join the cluster, you can deploy [.noloc]`Kubernetes` applications to them. You can also launch self-managed Amazon Linux nodes with `eksctl` or the {aws-management-console}. If you need to launch nodes on {aws} Outposts, see <>. +This topic describes how you can launch Auto Scaling groups of Linux nodes that register with your Amazon EKS cluster. After the nodes join the cluster, you can deploy Kubernetes applications to them. You can also launch self-managed Amazon Linux nodes with `eksctl` or the {aws-management-console}. If you need to launch nodes on {aws} Outposts, see <>. * An existing Amazon EKS cluster. To deploy one, see <>. If you have subnets in the {aws} Region where you have {aws} Outposts, {aws} Wavelength, or {aws} Local Zones enabled, those subnets must not have been passed in when you created your cluster. * An existing IAM role for the nodes to use. To create one, see <>. If this role doesn't have either of the policies for the VPC CNI, the separate role that follows is required for the VPC CNI pods. -* (Optional, but recommended) The [.noloc]`Amazon VPC CNI plugin for Kubernetes` add-on configured with its own IAM role that has the necessary IAM policy attached to it. For more information, see <>. +* (Optional, but recommended) The Amazon VPC CNI plugin for Kubernetes add-on configured with its own IAM role that has the necessary IAM policy attached to it. For more information, see <>. * Familiarity with the considerations listed in <>. Depending on the instance type you choose, there may be additional prerequisites for your cluster and VPC. You can launch self-managed Linux nodes using either of the following: @@ -28,8 +28,8 @@ You can launch self-managed Linux nodes using either of the following: . Install version `{eksctl-min-version}` or later of the `eksctl` command line tool installed on your device or {aws} CloudShell. To install or update `eksctl`, see https://eksctl.io/installation[Installation] in the `eksctl` documentation. -. (Optional) If the *AmazonEKS_CNI_Policy* managed IAM policy is attached to your <>, we recommend assigning it to an IAM role that you associate to the [.noloc]`Kubernetes` `aws-node` service account instead. For more information, see <>. -. The following command creates a node group in an existing cluster. Replace [.replaceable]`al-nodes` with a name for your node group. The node group name can't be longer than 63 characters. It must start with letter or digit, but can also include hyphens and underscores for the remaining characters. Replace [.replaceable]`my-cluster` with the name of your cluster. The name can contain only alphanumeric characters (case-sensitive) and hyphens. It must start with an alphanumeric character and can't be longer than 100 characters. The name must be unique within the {aws} Region and {aws} account that you're creating the cluster in. Replace the remaining [.replaceable]`example value` with your own values. The nodes are created with the same [.noloc]`Kubernetes` version as the control plane, by default. +. (Optional) If the *AmazonEKS_CNI_Policy* managed IAM policy is attached to your <>, we recommend assigning it to an IAM role that you associate to the Kubernetes `aws-node` service account instead. For more information, see <>. +. The following command creates a node group in an existing cluster. Replace [.replaceable]`al-nodes` with a name for your node group. The node group name can't be longer than 63 characters. It must start with letter or digit, but can also include hyphens and underscores for the remaining characters. Replace [.replaceable]`my-cluster` with the name of your cluster. The name can contain only alphanumeric characters (case-sensitive) and hyphens. It must start with an alphanumeric character and can't be longer than 100 characters. The name must be unique within the {aws} Region and {aws} account that you're creating the cluster in. Replace the remaining [.replaceable]`example value` with your own values. The nodes are created with the same Kubernetes version as the control plane, by default. + Before choosing a value for `--node-type`, review <>. + @@ -59,10 +59,9 @@ eksctl create nodegroup \ --ssh-public-key my-key ---- To deploy a node group that: -**** can assign a significantly higher number of IP addresses to [.noloc]`Pods` than the default configuration, see <>. -**** can assign `IPv4` addresses to [.noloc]`Pods` from a different [.noloc]`CIDR` block than that of the instance, see <>. -**** can assign `IPv6` addresses to [.noloc]`Pods` and services, see <>. -**** use the `containerd` runtime, you must deploy the node group using a `config` file. For more information, see <>. +**** can assign a significantly higher number of IP addresses to Pods than the default configuration, see <>. +**** can assign `IPv4` addresses to Pods from a different CIDR block than that of the instance, see <>. +**** can assign `IPv6` addresses to Pods and services, see <>. **** don't have outbound internet access, see <>. + For a complete list of all available options and defaults, enter the following command. @@ -80,11 +79,11 @@ An example output is as follows. Several lines are output while the nodes are cr ---- [✔] created 1 nodegroup(s) in cluster "my-cluster" ---- -. (Optional) Deploy a <> to test your cluster and [.noloc]`Linux` nodes. -. We recommend blocking [.noloc]`Pod` access to IMDS if the following conditions are true: +. (Optional) Deploy a <> to test your cluster and Linux nodes. +. We recommend blocking Pod access to IMDS if the following conditions are true: + -*** You plan to assign IAM roles to all of your [.noloc]`Kubernetes` service accounts so that [.noloc]`Pods` only have the minimum permissions that they need. -*** No [.noloc]`Pods` in the cluster require access to the Amazon EC2 instance metadata service (IMDS) for other reasons, such as retrieving the current {aws} Region. +*** You plan to assign IAM roles to all of your Kubernetes service accounts so that Pods only have the minimum permissions that they need. +*** No Pods in the cluster require access to the Amazon EC2 instance metadata service (IMDS) for other reasons, such as retrieving the current {aws} Region. + For more information, see https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node[Restrict access to the instance profile assigned to the worker node]. @@ -101,59 +100,58 @@ curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/cloudformation/2022-12-23/ ---- . Wait for your cluster status to show as `ACTIVE`. If you launch your nodes before the cluster is active, the nodes fail to register with the cluster and you will have to relaunch them. . Open the link:cloudformation/[{aws} CloudFormation console,type="console"]. -. Choose *Create stack* and then select *With new resources (standard)*. -. For *Specify template*, select *Upload a template file* and then select *Choose file*. +. Choose *Create stack* and then select *With new resources (standard)*. +. For *Specify template*, select *Upload a template file* and then select *Choose file*. . Select the `amazon-eks-nodegroup.yaml` file that you downloaded. . Select *Next*. -. On the *Specify stack details* page, enter the following parameters accordingly, and then choose *Next*: +. On the *Specify stack details* page, enter the following parameters accordingly, and then choose *Next*: + ** *Stack name*: Choose a stack name for your {aws} CloudFormation stack. For example, you can call it [.replaceable]`my-cluster-nodes`. The name can contain only alphanumeric characters (case-sensitive) and hyphens. It must start with an alphanumeric character and can't be longer than 100 characters. The name must be unique within the {aws} Region and {aws} account that you're creating the cluster in. ** *ClusterName*: Enter the name that you used when you created your Amazon EKS cluster. This name must equal the cluster name or your nodes can't join the cluster. -** *ClusterControlPlaneSecurityGroup*: Choose the *SecurityGroups* value from the {aws} CloudFormation output that you generated when you created your <>. +** *ClusterControlPlaneSecurityGroup*: Choose the *SecurityGroups* value from the {aws} CloudFormation output that you generated when you created your <>. + The following steps show one operation to retrieve the applicable group. + .. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. .. Choose the name of the cluster. .. Choose the *Networking* tab. -.. Use the *Additional security groups* value as a reference when selecting from the *ClusterControlPlaneSecurityGroup* dropdown list. +.. Use the *Additional security groups* value as a reference when selecting from the *ClusterControlPlaneSecurityGroup* dropdown list. ** *NodeGroupName*: Enter a name for your node group. This name can be used later to identify the Auto Scaling node group that's created for your nodes. The node group name can't be longer than 63 characters. It must start with letter or digit, but can also include hyphens and underscores for the remaining characters. ** *NodeAutoScalingGroupMinSize*: Enter the minimum number of nodes that your node Auto Scaling group can scale in to. ** *NodeAutoScalingGroupDesiredCapacity*: Enter the desired number of nodes to scale to when your stack is created. ** *NodeAutoScalingGroupMaxSize*: Enter the maximum number of nodes that your node Auto Scaling group can scale out to. ** *NodeInstanceType*: Choose an instance type for your nodes. For more information, see <>. -** *NodeImageIdSSMParam*: Pre-populated with the Amazon EC2 Systems Manager parameter of a recent Amazon EKS optimized AMI for a variable [.noloc]`Kubernetes` version. To use a different [.noloc]`Kubernetes` minor version supported with Amazon EKS, replace [.replaceable]`1.XX` with a different <>. We recommend specifying the same [.noloc]`Kubernetes` version as your cluster. +** *NodeImageIdSSMParam*: Pre-populated with the Amazon EC2 Systems Manager parameter of a recent Amazon EKS optimized AMI for a variable Kubernetes version. To use a different Kubernetes minor version supported with Amazon EKS, replace [.replaceable]`1.XX` with a different link:eks/latest/userguide/kubernetes-versions.html[supported version,type="documentation"]. We recommend specifying the same Kubernetes version as your cluster. + You can also replace [.replaceable]`amazon-linux-2` with a different AMI type. For more information, see <>. + NOTE: The Amazon EKS node AMIs are based on Amazon Linux. You can track security or privacy events for Amazon Linux 2 at the https://alas.aws.amazon.com/alas2.html[Amazon Linux Security Center] or subscribe to the associated https://alas.aws.amazon.com/AL2/alas.rss[RSS feed]. Security and privacy events include an overview of the issue, what packages are affected, and how to update your instances to correct the issue. -** *NodeImageId*: (Optional) If you're using your own custom AMI (instead of an Amazon EKS optimized AMI), enter a node AMI ID for your {aws} Region. If you specify a value here, it overrides any values in the *NodeImageIdSSMParam* field. +** *NodeImageId*: (Optional) If you're using your own custom AMI (instead of an Amazon EKS optimized AMI), enter a node AMI ID for your {aws} Region. If you specify a value here, it overrides any values in the *NodeImageIdSSMParam* field. ** *NodeVolumeSize*: Specify a root volume size for your nodes, in GiB. ** *NodeVolumeType*: Specify a root volume type for your nodes. ** *KeyName*: Enter the name of an Amazon EC2 SSH key pair that you can use to connect using SSH into your nodes with after they launch. If you don't already have an Amazon EC2 key pair, you can create one in the {aws-management-console}. For more information, see link:AWSEC2/latest/UserGuide/ec2-key-pairs.html[Amazon EC2 key pairs,type="documentation"] in the _Amazon EC2 User Guide_. + NOTE: If you don't provide a key pair here, the {aws} CloudFormation stack creation fails. -** *BootstrapArguments*: Specify any optional arguments to pass to the node bootstrap script, such as extra `kubelet` arguments. For more information, view the https://github.com/awslabs/amazon-eks-ami/blob/main/templates/al2/runtime/bootstrap.sh[bootstrap script usage information] on [.noloc]`GitHub`. +** *BootstrapArguments*: Specify any optional arguments to pass to the node bootstrap script, such as extra `kubelet` arguments. For more information, view the https://github.com/awslabs/amazon-eks-ami/blob/main/templates/al2/runtime/bootstrap.sh[bootstrap script usage information] on GitHub. + To deploy a node group that: + -*** can assign a significantly higher number of IP addresses to [.noloc]`Pods` than the default configuration, see <>. -*** can assign `IPv4` addresses to [.noloc]`Pods` from a different [.noloc]`CIDR` block than that of the instance, see <>. -*** can assign `IPv6` addresses to [.noloc]`Pods` and services, see <>. -*** use the `containerd` runtime, you must deploy the node group using a `config` file. For more information, see <>. +*** can assign a significantly higher number of IP addresses to Pods than the default configuration, see <>. +*** can assign `IPv4` addresses to Pods from a different CIDR block than that of the instance, see <>. +*** can assign `IPv6` addresses to Pods and services, see <>. *** don't have outbound internet access, see <>. -** *DisableIMDSv1*: By default, each node supports the Instance Metadata Service Version 1 (IMDSv1) and IMDSv2. You can disable IMDSv1. To prevent future nodes and [.noloc]`Pods` in the node group from using MDSv1, set *DisableIMDSv1* to *true*. For more information about IMDS, see link:AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html[Configuring the instance metadata service,type="documentation"]. For more information about restricting access to it on your nodes, see https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node[Restrict access to the instance profile assigned to the worker node]. -** *VpcId*: Enter the ID for the <> that you created. -** *Subnets*: Choose the subnets that you created for your VPC. If you created your VPC using the steps that are described in <>, specify only the private subnets within the VPC for your nodes to launch into. You can see which subnets are private by opening each subnet link from the *Networking* tab of your cluster. +** *DisableIMDSv1*: By default, each node supports the Instance Metadata Service Version 1 (IMDSv1) and IMDSv2. You can disable IMDSv1. To prevent future nodes and Pods in the node group from using MDSv1, set *DisableIMDSv1* to *true*. For more information about IMDS, see link:AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html[Configuring the instance metadata service,type="documentation"]. For more information about restricting access to it on your nodes, see https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node[Restrict access to the instance profile assigned to the worker node]. +** *VpcId*: Enter the ID for the <> that you created. +** *Subnets*: Choose the subnets that you created for your VPC. If you created your VPC using the steps that are described in <>, specify only the private subnets within the VPC for your nodes to launch into. You can see which subnets are private by opening each subnet link from the *Networking* tab of your cluster. + [IMPORTANT] ==== -*** If any of the subnets are public subnets, then they must have the automatic public IP address assignment setting enabled. If the setting isn't enabled for the public subnet, then any nodes that you deploy to that public subnet won't be assigned a public IP address and won't be able to communicate with the cluster or other {aws} services. If the subnet was deployed before March 26, 2020 using either of the <>, or by using `eksctl`, then automatic public IP address assignment is disabled for public subnets. For information about how to enable public IP address assignment for a subnet, see link:vpc/latest/userguide/vpc-ip-addressing.html#subnet-public-ip[Modifying the public IPv4 addressing attribute for your subnet,type="documentation"]. If the node is deployed to a private subnet, then it's able to communicate with the cluster and other {aws} services through a NAT gateway. +*** If any of the subnets are public subnets, then they must have the automatic public IP address assignment setting enabled. If the setting isn't enabled for the public subnet, then any nodes that you deploy to that public subnet won't be assigned a public IP address and won't be able to communicate with the cluster or other {aws} services. If the subnet was deployed before March 26, 2020 using either of the <>, or by using `eksctl`, then automatic public IP address assignment is disabled for public subnets. For information about how to enable public IP address assignment for a subnet, see link:vpc/latest/userguide/vpc-ip-addressing.html#subnet-public-ip[Modifying the public IPv4 addressing attribute for your subnet,type="documentation"]. If the node is deployed to a private subnet, then it's able to communicate with the cluster and other {aws} services through a NAT gateway. *** If the subnets don't have internet access, make sure that you're aware of the considerations and extra steps in <>. *** If you select {aws} Outposts, Wavelength, or Local Zone subnets, the subnets must not have been passed in when you created the cluster. ==== -. Select your desired choices on the *Configure stack options* page, and then choose *Next*. -. Select the check box to the left of *I acknowledge that {aws} CloudFormation might create IAM resources.*, and then choose *Create stack*. +. Select your desired choices on the *Configure stack options* page, and then choose *Next*. +. Select the check box to the left of *I acknowledge that {aws} CloudFormation might create IAM resources.*, and then choose *Create stack*. . When your stack has finished creating, select it in the console and choose *Outputs*. . Record the *NodeInstanceRole* for the node group that was created. You need this when you configure your Amazon EKS nodes. @@ -225,7 +223,7 @@ Enter `Ctrl`+``C`` to return to a shell prompt. NOTE: If you receive any authorization or resource type errors, see <> in the troubleshooting topic. + If nodes fail to join the cluster, then see <> in the Troubleshooting chapter. -. (GPU nodes only) If you chose a GPU instance type and the Amazon EKS optimized accelerated AMI, you must apply the https://github.com/NVIDIA/k8s-device-plugin[NVIDIA device plugin for Kubernetes] as a [.noloc]`DaemonSet` on your cluster. Replace [.replaceable]`vX.X.X` with your desired https://github.com/NVIDIA/k8s-device-plugin/releases[NVIDIA/k8s-device-plugin] version before running the following command. +. (GPU nodes only) If you chose a GPU instance type and the Amazon EKS optimized accelerated AMI, you must apply the https://github.com/NVIDIA/k8s-device-plugin[NVIDIA device plugin for Kubernetes] as a DaemonSet on your cluster. Replace [.replaceable]`vX.X.X` with your desired https://github.com/NVIDIA/k8s-device-plugin/releases[NVIDIA/k8s-device-plugin] version before running the following command. + [source,bash,subs="verbatim,attributes"] ---- @@ -234,12 +232,12 @@ kubectl apply -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/vX.X *Step 3: Additional actions* -. (Optional) Deploy a <> to test your cluster and [.noloc]`Linux` nodes. -. (Optional) If the *AmazonEKS_CNI_Policy* managed IAM policy (if you have an `IPv4` cluster) or the [.replaceable]`AmazonEKS_CNI_IPv6_Policy` (that you <> if you have an `IPv6` cluster) is attached to your <>, we recommend assigning it to an IAM role that you associate to the [.noloc]`Kubernetes` `aws-node` service account instead. For more information, see <>. -. We recommend blocking [.noloc]`Pod` access to IMDS if the following conditions are true: +. (Optional) Deploy a <> to test your cluster and Linux nodes. +. (Optional) If the *AmazonEKS_CNI_Policy* managed IAM policy (if you have an `IPv4` cluster) or the [.replaceable]`AmazonEKS_CNI_IPv6_Policy` (that you <> if you have an `IPv6` cluster) is attached to your <>, we recommend assigning it to an IAM role that you associate to the Kubernetes `aws-node` service account instead. For more information, see <>. +. We recommend blocking Pod access to IMDS if the following conditions are true: + -** You plan to assign IAM roles to all of your [.noloc]`Kubernetes` service accounts so that [.noloc]`Pods` only have the minimum permissions that they need. -** No [.noloc]`Pods` in the cluster require access to the Amazon EC2 instance metadata service (IMDS) for other reasons, such as retrieving the current {aws} Region. +** You plan to assign IAM roles to all of your Kubernetes service accounts so that Pods only have the minimum permissions that they need. +** No Pods in the cluster require access to the Amazon EC2 instance metadata service (IMDS) for other reasons, such as retrieving the current {aws} Region. + -For more information, see https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node[Restrict access to the instance profile assigned to the worker node]. +For more information, see https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node[Restrict access to the instance profile assigned to the worker node]. \ No newline at end of file diff --git a/latest/ug/nodes/learn-status-conditions.adoc b/latest/ug/nodes/learn-status-conditions.adoc index 0cc8fa08f..49e227306 100644 --- a/latest/ug/nodes/learn-status-conditions.adoc +++ b/latest/ug/nodes/learn-status-conditions.adoc @@ -1,11 +1,10 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[learn-status-conditions,learn-status-conditions.title]] +[#learn-status-conditions] = View the health status of your nodes :info_titleabbrev: View node health -include::../attributes.txt[] - [abstract] -- This topic explains the tools and methods available for monitoring node health status in Amazon EKS clusters. @@ -13,9 +12,9 @@ This topic explains the tools and methods available for monitoring node health s This topic explains the tools and methods available for monitoring node health status in Amazon EKS clusters. The information covers node conditions, events, and detection cases that help you identify and diagnose node-level issues. Use the commands and patterns described here to inspect node health resources, interpret status conditions, and analyze node events for operational troubleshooting. -You can get some node health information with [.noloc]`Kubernetes` commands for all nodes. And if you use the node monitoring agent through Amazon EKS Auto Mode or the Amazon EKS managed add-on, you will get a wider variety of node signals to help troubleshoot. Descriptions of detected health issues by the node monitoring agent are also made available in the observability dashboard. For more information, see <>. +You can get some node health information with Kubernetes commands for all nodes. And if you use the node monitoring agent through Amazon EKS Auto Mode or the Amazon EKS managed add-on, you will get a wider variety of node signals to help troubleshoot. Descriptions of detected health issues by the node monitoring agent are also made available in the observability dashboard. For more information, see <>. -[[status-node-conditions,status-node-conditions.title]] +[#status-node-conditions] == Node conditions Node conditions represent terminal issues requiring remediation actions like instance replacement or reboot. @@ -59,7 +58,7 @@ kubectl describe node [.replaceable]`node-name` type: NetworkingReady ---- -[[status-node-events,status-node-events.title]] +[#status-node-events] == Node events Node events indicate temporary issues or sub-optimal configurations. @@ -111,7 +110,7 @@ LAST SEEN TYPE REASON OBJECT MESSAGE 5m Normal NodeReady Node/node-1 Node became ready ---- -[[status-node-troubleshooting,status-node-troubleshooting.title]] +[#status-node-troubleshooting] == Common troubleshooting commands [source,bash,subs="verbatim,attributes,quotes"] @@ -124,4 +123,4 @@ kubectl get nodes -w # Get node metrics kubectl top node ----- +---- \ No newline at end of file diff --git a/latest/ug/nodes/managed-node-groups.adoc b/latest/ug/nodes/managed-node-groups.adoc index b48631f88..c933d94db 100644 --- a/latest/ug/nodes/managed-node-groups.adoc +++ b/latest/ug/nodes/managed-node-groups.adoc @@ -1,15 +1,9 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[managed-node-groups,managed-node-groups.title]] +[#managed-node-groups] = Simplify node lifecycle with managed node groups -:info_doctype: section -:info_title: Simplify node lifecycle with managed node groups :info_titleabbrev: Managed node groups -:keywords: managed node group, MNG -:info_abstract: Amazon EKS managed node groups automate the provisioning and lifecycle management of \ - nodes (Amazon EC2 instances) for Amazon EKS Kubernetes clusters. - -include::../attributes.txt[] include::create-managed-node-group.adoc[leveloffset=+1] @@ -23,18 +17,18 @@ include::delete-managed-node-group.adoc[leveloffset=+1] [abstract] -- -Amazon EKS managed node groups automate the provisioning and lifecycle management of nodes (Amazon EC2 instances) for Amazon EKS [.noloc]`Kubernetes` clusters. +Amazon EKS managed node groups automate the provisioning and lifecycle management of nodes (Amazon EC2 instances) for Amazon EKS Kubernetes clusters. -- -Amazon EKS managed node groups automate the provisioning and lifecycle management of nodes (Amazon EC2 instances) for Amazon EKS [.noloc]`Kubernetes` clusters. +Amazon EKS managed node groups automate the provisioning and lifecycle management of nodes (Amazon EC2 instances) for Amazon EKS Kubernetes clusters. -With Amazon EKS managed node groups, you don't need to separately provision or register the Amazon EC2 instances that provide compute capacity to run your [.noloc]`Kubernetes` applications. You can create, automatically update, or terminate nodes for your cluster with a single operation. Node updates and terminations automatically drain nodes to ensure that your applications stay available. +With Amazon EKS managed node groups, you don't need to separately provision or register the Amazon EC2 instances that provide compute capacity to run your Kubernetes applications. You can create, automatically update, or terminate nodes for your cluster with a single operation. Node updates and terminations automatically drain nodes to ensure that your applications stay available. Every managed node is provisioned as part of an Amazon EC2 Auto Scaling group that's managed for you by Amazon EKS. Every resource including the instances and Auto Scaling groups runs within your {aws} account. Each node group runs across multiple Availability Zones that you define. Managed node groups can also optionally leverage node auto repair, which continuously monitors the health of nodes. It automatically reacts to detected problems and replaces nodes when possible. This helps overall availability of the cluster with minimal manual intervention. For more information, see <>. -You can add a managed node group to new or existing clusters using the Amazon EKS console, `eksctl`, {aws} CLI, {aws} API, or infrastructure as code tools including {aws} CloudFormation. Nodes launched as part of a managed node group are automatically tagged for auto-discovery by the [.noloc]`Kubernetes` https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md[Cluster Autoscaler]. You can use the node group to apply [.noloc]`Kubernetes` labels to nodes and update them at any time. +You can add a managed node group to new or existing clusters using the Amazon EKS console, `eksctl`, {aws} CLI, {aws} API, or infrastructure as code tools including {aws} CloudFormation. Nodes launched as part of a managed node group are automatically tagged for auto-discovery by the Kubernetes https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md[Cluster Autoscaler]. You can use the node group to apply Kubernetes labels to nodes and update them at any time. There are no additional costs to use Amazon EKS managed node groups, you only pay for the {aws} resources you provision. These include Amazon EC2 instances, Amazon EBS volumes, Amazon EKS cluster hours, and any other {aws} infrastructure. There are no minimum fees and no upfront commitments. @@ -42,19 +36,19 @@ To get started with a new Amazon EKS cluster and managed node group, see <>. -[[managed-node-group-concepts,managed-node-group-concepts.title]] +[#managed-node-group-concepts] == Managed node groups concepts * Amazon EKS managed node groups create and manage Amazon EC2 instances for you. * Every managed node is provisioned as part of an Amazon EC2 Auto Scaling group that's managed for you by Amazon EKS. Moreover, every resource including Amazon EC2 instances and Auto Scaling groups run within your {aws} account. * The Auto Scaling group of a managed node group spans every subnet that you specify when you create the group. -* Amazon EKS tags managed node group resources so that they are configured to use the [.noloc]`Kubernetes` https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md[Cluster Autoscaler]. +* Amazon EKS tags managed node group resources so that they are configured to use the Kubernetes https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md[Cluster Autoscaler]. + -IMPORTANT: If you are running a stateful application across multiple Availability Zones that is backed by Amazon EBS volumes and using the [.noloc]`Kubernetes` https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md[Cluster Autoscaler], you should configure multiple node groups, each scoped to a single Availability Zone. In addition, you should enable the `--balance-similar-node-groups` feature. +IMPORTANT: If you are running a stateful application across multiple Availability Zones that is backed by Amazon EBS volumes and using the Kubernetes https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md[Cluster Autoscaler], you should configure multiple node groups, each scoped to a single Availability Zone. In addition, you should enable the `--balance-similar-node-groups` feature. * You can use a custom launch template for a greater level of flexibility and customization when deploying managed nodes. For example, you can specify extra `kubelet` arguments and use a custom AMI. For more information, see <>. If you don't use a custom launch template when first creating a managed node group, there is an auto-generated launch template. Don't manually modify this auto-generated template or errors occur. * Amazon EKS follows the shared responsibility model for CVEs and security patches on managed node groups. When managed nodes run an Amazon EKS optimized AMI, Amazon EKS is responsible for building patched versions of the AMI when bugs or issues are reported. We can publish a fix. However, you're responsible for deploying these patched AMI versions to your managed node groups. When managed nodes run a custom AMI, you're responsible for building patched versions of the AMI when bugs or issues are reported and then deploying the AMI. For more information, see <>. -* Amazon EKS managed node groups can be launched in both public and private subnets. If you launch a managed node group in a public subnet on or after April 22, 2020, the subnet must have `MapPublicIpOnLaunch` set to true for the instances to successfully join a cluster. If the public subnet was created using `eksctl` or the <> on or after March 26, 2020, then this setting is already set to true. If the public subnets were created before March 26, 2020, you must change the setting manually. For more information, see link:vpc/latest/userguide/vpc-ip-addressing.html#subnet-public-ip[Modifying the public IPv4 addressing attribute for your subnet,type="documentation"]. -* When deploying a managed node group in private subnets, you must ensure that it can access Amazon ECR for pulling container images. You can do this by connecting a NAT gateway to the route table of the subnet or by adding the following link:AmazonECR/latest/userguide/vpc-endpoints.html#ecr-setting-up-vpc-create[{aws} PrivateLink VPC endpoints,type="documentation"]: +* Amazon EKS managed node groups can be launched in both public and private subnets. If you launch a managed node group in a public subnet on or after April 22, 2020, the subnet must have `MapPublicIpOnLaunch` set to true for the instances to successfully join a cluster. If the public subnet was created using `eksctl` or the <> on or after March 26, 2020, then this setting is already set to true. If the public subnets were created before March 26, 2020, you must change the setting manually. For more information, see link:vpc/latest/userguide/vpc-ip-addressing.html#subnet-public-ip[Modifying the public IPv4 addressing attribute for your subnet,type="documentation"]. +* When deploying a managed node group in private subnets, you must ensure that it can access Amazon ECR for pulling container images. You can do this by connecting a NAT gateway to the route table of the subnet or by adding the following link:AmazonECR/latest/userguide/vpc-endpoints.html#ecr-setting-up-vpc-create[{aws} PrivateLink VPC endpoints,type="documentation"]: + ** Amazon ECR API endpoint interface – `com.amazonaws.[.replaceable]``region-code``.ecr.api` ** Amazon ECR Docker registry API endpoint interface – `com.amazonaws.[.replaceable]``region-code``.ecr.dkr` @@ -65,21 +59,21 @@ For other commonly-used services and endpoints, see <>. * Managed node groups can't be deployed on <> or in link:wavelength/[{aws} Wavelength,type="documentation"]. Managed node groups can be created on link:about-aws/global-infrastructure/localzones/[{aws} Local Zones,type="marketing"]. For more information, see <>. * You can create multiple managed node groups within a single cluster. For example, you can create one node group with the standard Amazon EKS optimized Amazon Linux AMI for some workloads and another with the GPU variant for workloads that require GPU support. * If your managed node group encounters an link:AWSEC2/latest/UserGuide/monitoring-system-instance-status-check.html[Amazon EC2 instance status check,type="documentation"] failure, Amazon EKS returns an error code to help you to diagnose the issue. For more information, see <>. -* Amazon EKS adds [.noloc]`Kubernetes` labels to managed node group instances. These Amazon EKS provided labels are prefixed with `eks.amazonaws.com`. -* Amazon EKS automatically drains nodes using the [.noloc]`Kubernetes` API during terminations or updates. -* Pod disruption budgets aren't respected when terminating a node with `AZRebalance` or reducing the desired node count. These actions try to evict [.noloc]`Pods` on the node. But if it takes more than 15 minutes, the node is terminated regardless of whether all [.noloc]`Pods` on the node are terminated. To extend the period until the node is terminated, add a lifecycle hook to the Auto Scaling group. For more information, see link:autoscaling/ec2/userguide/adding-lifecycle-hooks.html[Add lifecycle hooks,type="documentation"] in the _Amazon EC2 Auto Scaling User Guide_. +* Amazon EKS adds Kubernetes labels to managed node group instances. These Amazon EKS provided labels are prefixed with `eks.amazonaws.com`. +* Amazon EKS automatically drains nodes using the Kubernetes API during terminations or updates. +* Pod disruption budgets aren't respected when terminating a node with `AZRebalance` or reducing the desired node count. These actions try to evict Pods on the node. But if it takes more than 15 minutes, the node is terminated regardless of whether all Pods on the node are terminated. To extend the period until the node is terminated, add a lifecycle hook to the Auto Scaling group. For more information, see link:autoscaling/ec2/userguide/adding-lifecycle-hooks.html[Add lifecycle hooks,type="documentation"] in the _Amazon EC2 Auto Scaling User Guide_. * In order to run the drain process correctly after receiving a Spot interruption notification or a capacity rebalance notification, `CapacityRebalance` must be set to `true`. -* Updating managed node groups respects the [.noloc]`Pod` disruption budgets that you set for your [.noloc]`Pods`. For more information, see <>. +* Updating managed node groups respects the Pod disruption budgets that you set for your Pods. For more information, see <>. * There are no additional costs to use Amazon EKS managed node groups. You only pay for the {aws} resources that you provision. -* If you want to encrypt Amazon EBS volumes for your nodes, you can deploy the nodes using a launch template. To deploy managed nodes with encrypted Amazon EBS volumes without using a launch template, encrypt all new Amazon EBS volumes created in your account. For more information, see link:AWSEC2/latest/UserGuide/EBSEncryption.html#encryption-by-default[Encryption by default,type="documentation"] in the _Amazon EC2 User Guide_. +* If you want to encrypt Amazon EBS volumes for your nodes, you can deploy the nodes using a launch template. To deploy managed nodes with encrypted Amazon EBS volumes without using a launch template, encrypt all new Amazon EBS volumes created in your account. For more information, see link:AWSEC2/latest/UserGuide/EBSEncryption.html#encryption-by-default[Encryption by default,type="documentation"] in the _Amazon EC2 User Guide_. -[[managed-node-group-capacity-types,managed-node-group-capacity-types.title]] +[#managed-node-group-capacity-types] == Managed node group capacity types -When creating a managed node group, you can choose either the On-Demand or Spot capacity type. Amazon EKS deploys a managed node group with an Amazon EC2 Auto Scaling group that either contains only On-Demand or only Amazon EC2 Spot Instances. You can schedule [.noloc]`Pods` for fault tolerant applications to Spot managed node groups, and fault intolerant applications to On-Demand node groups within a single [.noloc]`Kubernetes` cluster. By default, a managed node group deploys On-Demand Amazon EC2 instances. +When creating a managed node group, you can choose either the On-Demand or Spot capacity type. Amazon EKS deploys a managed node group with an Amazon EC2 Auto Scaling group that either contains only On-Demand or only Amazon EC2 Spot Instances. You can schedule Pods for fault tolerant applications to Spot managed node groups, and fault intolerant applications to On-Demand node groups within a single Kubernetes cluster. By default, a managed node group deploys On-Demand Amazon EC2 instances. -[[managed-node-group-capacity-types-on-demand,managed-node-group-capacity-types-on-demand.title]] +[#managed-node-group-capacity-types-on-demand] === On-Demand With On-Demand Instances, you pay for compute capacity by the second, with no long-term commitments. @@ -89,24 +83,21 @@ With On-Demand Instances, you pay for compute capacity by the second, with no lo By default, if you don't specify a *Capacity Type*, the managed node group is provisioned with On-Demand Instances. A managed node group configures an Amazon EC2 Auto Scaling group on your behalf with the following settings applied: -* The allocation strategy to provision On-Demand capacity is set to `prioritized`. Managed node groups use the order of instance types passed in the API to determine which instance type to use first when fulfilling On-Demand capacity. For example, you might specify three instance types in the following order: `c5.large`, `c4.large`, and `c3.large`. When your On-Demand Instances are launched, the managed node group fulfills On-Demand capacity by starting with `c5.large`, then `c4.large`, and then `c3.large`. For more information, see link:autoscaling/ec2/userguide/asg-purchase-options.html#asg-allocation-strategies[Amazon EC2 Auto Scaling group,type="documentation"] in the _Amazon EC2 Auto Scaling User Guide_. -* Amazon EKS adds the following [.noloc]`Kubernetes` label to all nodes in your managed node group that specifies the capacity type: `eks.amazonaws.com/capacityType: ON_DEMAND`. You can use this label to schedule stateful or fault intolerant applications on On-Demand nodes. +* The allocation strategy to provision On-Demand capacity is set to `prioritized`. Managed node groups use the order of instance types passed in the API to determine which instance type to use first when fulfilling On-Demand capacity. For example, you might specify three instance types in the following order: `c5.large`, `c4.large`, and `c3.large`. When your On-Demand Instances are launched, the managed node group fulfills On-Demand capacity by starting with `c5.large`, then `c4.large`, and then `c3.large`. For more information, see link:autoscaling/ec2/userguide/asg-purchase-options.html#asg-allocation-strategies[Amazon EC2 Auto Scaling group,type="documentation"] in the _Amazon EC2 Auto Scaling User Guide_. +* Amazon EKS adds the following Kubernetes label to all nodes in your managed node group that specifies the capacity type: `eks.amazonaws.com/capacityType: ON_DEMAND`. You can use this label to schedule stateful or fault intolerant applications on On-Demand nodes. -[[managed-node-group-capacity-types-spot,managed-node-group-capacity-types-spot.title]] +[#managed-node-group-capacity-types-spot] === Spot Amazon EC2 Spot Instances are spare Amazon EC2 capacity that offers steep discounts off of On-Demand prices. Amazon EC2 Spot Instances can be interrupted with a two-minute interruption notice when EC2 needs the capacity back. For more information, see link:AWSEC2/latest/UserGuide/using-spot-instances.html[Spot Instances,type="documentation"] in the _Amazon EC2 User Guide_. You can configure a managed node group with Amazon EC2 Spot Instances to optimize costs for the compute nodes running in your Amazon EKS cluster. - - - To use Spot Instances inside a managed node group, create a managed node group by setting the capacity type as `spot`. A managed node group configures an Amazon EC2 Auto Scaling group on your behalf with the following Spot best practices applied: * To ensure that your Spot nodes are provisioned in the optimal Spot capacity pools, the allocation strategy is set to one of the following: + -** `price-capacity-optimized` (PCO) – When creating new node groups in a cluster with [.noloc]`Kubernetes` version `1.28` or higher, the allocation strategy is set to `price-capacity-optimized`. However, the allocation strategy won't be changed for node groups already created with `capacity-optimized` before Amazon EKS managed node groups started to support PCO. -** `capacity-optimized` (CO) – When creating new node groups in a cluster with [.noloc]`Kubernetes` version `1.27` or lower, the allocation strategy is set to `capacity-optimized`. +** `price-capacity-optimized` (PCO) – When creating new node groups in a cluster with Kubernetes version `1.28` or higher, the allocation strategy is set to `price-capacity-optimized`. However, the allocation strategy won't be changed for node groups already created with `capacity-optimized` before Amazon EKS managed node groups started to support PCO. +** `capacity-optimized` (CO) – When creating new node groups in a cluster with Kubernetes version `1.27` or lower, the allocation strategy is set to `capacity-optimized`. + To increase the number of Spot capacity pools available for allocating capacity from, configure a managed node group to use multiple instance types. @@ -114,10 +105,8 @@ To increase the number of Spot capacity pools available for allocating capacity + ** When a Spot node receives a rebalance recommendation, Amazon EKS automatically attempts to launch a new replacement Spot node. ** If a Spot two-minute interruption notice arrives before the replacement Spot node is in a `Ready` state, Amazon EKS starts draining the Spot node that received the rebalance recommendation. Amazon EKS drains the node on a best-effort basis. As a result, there's no guarantee that Amazon EKS will wait for the replacement node to join the cluster before draining the existing node. -** When a replacement Spot node is bootstrapped and in the `Ready` state on [.noloc]`Kubernetes`, Amazon EKS cordons and drains the Spot node that received the rebalance recommendation. Cordoning the Spot node ensures that the service controller doesn't send any new requests to this Spot node. It also removes it from its list of healthy, active Spot nodes. Draining the Spot node ensures that running [.noloc]`Pods` are evicted gracefully. -* Amazon EKS adds the following [.noloc]`Kubernetes` label to all nodes in your managed node group that specifies the capacity type: `eks.amazonaws.com/capacityType: SPOT`. You can use this label to schedule fault tolerant applications on Spot nodes. - - +** When a replacement Spot node is bootstrapped and in the `Ready` state on Kubernetes, Amazon EKS cordons and drains the Spot node that received the rebalance recommendation. Cordoning the Spot node ensures that the service controller doesn't send any new requests to this Spot node. It also removes it from its list of healthy, active Spot nodes. Draining the Spot node ensures that running Pods are evicted gracefully. +* Amazon EKS adds the following Kubernetes label to all nodes in your managed node group that specifies the capacity type: `eks.amazonaws.com/capacityType: SPOT`. You can use this label to schedule fault tolerant applications on Spot nodes. When deciding whether to deploy a node group with On-Demand or Spot capacity, you should consider the following conditions: @@ -127,4 +116,4 @@ When deciding whether to deploy a node group with On-Demand or Spot capacity, yo + ** Within a managed node group, if you're using the https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md[Cluster Autoscaler], we recommend using a flexible set of instance types with the same amount of vCPU and memory resources. This is to ensure that the nodes in your cluster scale as expected. For example, if you need four vCPUs and eight GiB memory, use `c3.xlarge`, `c4.xlarge`, `c5.xlarge`, `c5d.xlarge`, `c5a.xlarge`, `c5n.xlarge`, or other similar instance types. ** To enhance application availability, we recommend deploying multiple Spot managed node groups. For this, each group should use a flexible set of instance types that have the same vCPU and memory resources. For example, if you need 4 vCPUs and 8 GiB memory, we recommend that you create one managed node group with `c3.xlarge`, `c4.xlarge`, `c5.xlarge`, `c5d.xlarge`, `c5a.xlarge`, `c5n.xlarge`, or other similar instance types, and a second managed node group with `m3.xlarge`, `m4.xlarge`, `m5.xlarge`, `m5d.xlarge`, `m5a.xlarge`, `m5n.xlarge` or other similar instance types. -** When deploying your node group with the Spot capacity type that's using a custom launch template, use the API to pass multiple instance types. Don't pass a single instance type through the launch template. For more information about deploying a node group using a launch template, see <>. +** When deploying your node group with the Spot capacity type that's using a custom launch template, use the API to pass multiple instance types. Don't pass a single instance type through the launch template. For more information about deploying a node group using a launch template, see <>. \ No newline at end of file diff --git a/latest/ug/nodes/managed-node-update-behavior.adoc b/latest/ug/nodes/managed-node-update-behavior.adoc index 8a9cff1cf..01179d4fa 100644 --- a/latest/ug/nodes/managed-node-update-behavior.adoc +++ b/latest/ug/nodes/managed-node-update-behavior.adoc @@ -1,7 +1,7 @@ -//!!NODE_ROOT
include::../attributes.txt[] + [.topic] -[[managed-node-update-behavior,managed-node-update-behavior.title]] +[#managed-node-update-behavior] = Understand each phase of node updates :info_titleabbrev: Update behavior details @@ -12,93 +12,105 @@ The Amazon EKS managed worker node upgrade strategy has four different phases. The Amazon EKS managed worker node upgrade strategy has four different phases described in the following sections. -[[managed-node-update-set-up,managed-node-update-set-up.title]] +[#managed-node-update-set-up] == Setup phase The setup phase has these steps: -. It creates a new Amazon EC2 launch template version for the Auto Scaling group that's associated with your node group. The new launch template version uses the target AMI or a custom launch template version for the update. -. It updates the Auto Scaling group to use the latest launch template version. -. It determines the maximum quantity of nodes to upgrade in parallel using the `updateConfig` property for the node group. The maximum unavailable has a quota of 100 nodes. The default value is one node. For more information, see the link:eks/latest/APIReference/API_UpdateNodegroupConfig.html#API_UpdateNodegroupConfig_RequestSyntax[updateConfig,type="documentation"] property in the _Amazon EKS API Reference_. +. It creates a new Amazon EC2 launch template version for the Auto Scaling Group that's associated with your node group. The new launch template version uses the target AMI or a custom launch template version for the update. +. It updates the Auto Scaling Group to use the latest launch template version. +. It determines the maximum quantity of nodes to upgrade in parallel using the `updateConfig` property for the node group. The maximum unavailable has a quota of 100 nodes. The default value is one node. For more information, see the link:eks/latest/APIReference/API_UpdateNodegroupConfig.html#API_UpdateNodegroupConfig_RequestSyntax[updateConfig,type="documentation"] property in the _Amazon EKS API Reference_. -[[managed-node-update-scale-up,managed-node-update-scale-up.title]] +[#managed-node-update-scale-up] == Scale up phase -When upgrading the nodes in a managed node group, the upgraded nodes are launched in the same Availability Zone as those that are being upgraded. To guarantee this placement, we use Amazon EC2's Availability Zone Rebalancing. For more information, see link:autoscaling/ec2/userguide/auto-scaling-benefits.html#AutoScalingBehavior.InstanceUsage[Availability Zone Rebalancing,type="documentation"] in the _Amazon EC2 Auto Scaling User Guide_. To meet this requirement, it's possible that we'd launch up to two instances per Availability Zone in your managed node group. +When upgrading the nodes in a managed node group, the upgraded nodes are launched in the same Availability Zone as those that are being upgraded. To guarantee this placement, we use Amazon EC2's Availability Zone Rebalancing. For more information, see link:autoscaling/ec2/userguide/auto-scaling-benefits.html#AutoScalingBehavior.InstanceUsage[Availability Zone Rebalancing,type="documentation"] in the _Amazon EC2 Auto Scaling User Guide_. To meet this requirement, it's possible that we'd launch up to two instances per Availability Zone in your managed node group. The scale up phase has these steps: . It increments the Auto Scaling Group's maximum size and desired size by the larger of either: + -** Up to twice the number of Availability Zones that the Auto Scaling group is deployed in. +** Up to twice the number of Availability Zones that the Auto Scaling Group is deployed in. ** The maximum unavailable of upgrade. + For example, if your node group has five Availability Zones and `maxUnavailable` as one, the upgrade process can launch a maximum of 10 nodes. However when `maxUnavailable` is 20 (or anything higher than 10), the process would launch 20 new nodes. -. After scaling the Auto Scaling group, it checks if the nodes using the latest configuration are present in the node group. This step succeeds only when it meets these criteria: +. After scaling the Auto Scaling Group, it checks if the nodes using the latest configuration are present in the node group. This step succeeds only when it meets these criteria: + ** At least one new node is launched in every Availability Zone where the node exists. ** Every new node should be in `Ready` state. ** New nodes should have Amazon EKS applied labels. + These are the Amazon EKS applied labels on the worker nodes in a regular node group: -+ -*** `eks.amazonaws.com/nodegroup-image=[.replaceable]``$amiName``` -*** `eks.amazonaws.com/nodegroup=[.replaceable]``$nodeGroupName``` + +*** `eks.amazonaws.com/nodegroup-image=$amiName` +*** `eks.amazonaws.com/nodegroup=$nodeGroupName` + These are the Amazon EKS applied labels on the worker nodes in a custom launch template or AMI node group: + + -*** `eks.amazonaws.com/nodegroup-image=[.replaceable]``$amiName``` -*** `eks.amazonaws.com/nodegroup=[.replaceable]``$nodeGroupName``` -*** `eks.amazonaws.com/sourceLaunchTemplateId=[.replaceable]``$launchTemplateId``` -*** `eks.amazonaws.com/sourceLaunchTemplateVersion=[.replaceable]``$launchTemplateVersion``` -. It marks nodes as unschedulable to avoid scheduling new [.noloc]`Pods`. It also labels nodes with `node.kubernetes.io/exclude-from-external-load-balancers=true` to remove the nodes from load balancers before terminating the nodes. +*** `eks.amazonaws.com/nodegroup-image=$amiName` +*** `eks.amazonaws.com/nodegroup=$nodeGroupName` +*** `eks.amazonaws.com/sourceLaunchTemplateId=$launchTemplateId` +*** `eks.amazonaws.com/sourceLaunchTemplateVersion=$launchTemplateVersion` +. It marks nodes as unschedulable to avoid scheduling new Pods. It also labels nodes with `node.kubernetes.io/exclude-from-external-load-balancers=true` to remove the old nodes from load balancers before terminating the nodes. The following are known reasons which lead to a `NodeCreationFailure` error in this phase: - - *Insufficient capacity in the Availability Zone*:: There is a possibility that the Availability Zone might not have capacity of requested instance types. It's recommended to configure multiple instance types while creating a managed node group. - *EC2 instance limits in your account*:: -You may need to increase the number of Amazon EC2 instances your account can run simultaneously using Service Quotas. For more information, see link:AWSEC2/latest/UserGuide/ec2-resource-limits.html[EC2 Service Quotas,type="documentation"] in the _Amazon Elastic Compute Cloud User Guide for [.noloc]`Linux` Instances_. - +You may need to increase the number of Amazon EC2 instances your account can run simultaneously using Service Quotas. For more information, see link:AWSEC2/latest/UserGuide/ec2-resource-limits.html[EC2 Service Quotas,type="documentation"] in the _Amazon Elastic Compute Cloud User Guide for Linux Instances_. *Custom user data*:: Custom user data can sometimes break the bootstrap process. This scenario can lead to the `kubelet` not starting on the node or nodes not getting expected Amazon EKS labels on them. For more information, see <>. - *Any changes which make a node unhealthy or not ready*:: Node disk pressure, memory pressure, and similar conditions can lead to a node not going to `Ready` state. +*Each node most bootstrap within 15 minutes*:: +If any node takes more than 15 minutes to bootstrap and join the cluster, it will cause the upgrade to time out. This is the total runtime for bootstrapping a new node measured from when a new node is required to when it joins the cluster. When upgrading a managed node group, the time counter starts as soon as the Auto Scaling Group size increases. -[[managed-node-update-upgrade,managed-node-update-upgrade.title]] + +[#managed-node-update-upgrade] == Upgrade phase -The upgrade phase has these steps: +The upgrade phase behaves in two different ways, depending on the _update strategy_. There are two update strategies: *default* and *minimal*. + +We recommend the default strategy in most scenarios. It creates new nodes before terminating the old ones, so that the available capacity is maintained during the upgrade phase. +The minimal strategy is useful in scenarios where you are constrained to resources or costs, for example with hardware accelerators such as GPUs. It terminating the old nodes before creating the new ones, so that total capacity never increases beyond your configured quantity. + +The _default_ update strategy has these steps: +. It increases the quantity of nodes (desired count) in the Auto Scaling Group, causing the node group to create additional nodes. . It randomly selects a node that needs to be upgraded, up to the maximum unavailable configured for the node group. -. It drains the [.noloc]`Pods` from the node. If the [.noloc]`Pods` don't leave the node within 15 minutes and there's no force flag, the upgrade phase fails with a `PodEvictionFailure` error. For this scenario, you can apply the force flag with the `update-nodegroup-version` request to delete the [.noloc]`Pods`. -. It cordons the node after every [.noloc]`Pod` is evicted and waits for 60 seconds. This is done so that the service controller doesn't send any new requests to this node and removes this node from its list of active nodes. +. It drains the Pods from the node. If the Pods don't leave the node within 15 minutes and there's no force flag, the upgrade phase fails with a `PodEvictionFailure` error. For this scenario, you can apply the force flag with the `update-nodegroup-version` request to delete the Pods. +. It cordons the node after every Pod is evicted and waits for 60 seconds. This is done so that the service controller doesn't send any new requests to this node and removes this node from its list of active nodes. . It sends a termination request to the Auto Scaling Group for the cordoned node. . It repeats the previous upgrade steps until there are no nodes in the node group that are deployed with the earlier version of the launch template. -The following are known reasons which lead to a `PodEvictionFailure` error in this phase: +The _minimal_ update strategy has these steps: +. It cordons all nodes of the node group in the beginning, so that the service controller doesn't send any new requests to these nodes. +. It randomly selects a node that needs to be upgraded, up to the maximum unavailable configured for the node group. +. It drains the Pods from the selected nodes. If the Pods don't leave the node within 15 minutes and there's no force flag, the upgrade phase fails with a `PodEvictionFailure` error. For this scenario, you can apply the force flag with the `update-nodegroup-version` request to delete the Pods. +. After every Pod is evicted and waits for 60 seconds, it sends a termination request to the Auto Scaling Group for the selected nodes. The Auto Scaling Group creates new nodes (same as the number of selected nodes) to replace the missing capacity. +. It repeats the previous upgrade steps until there are no nodes in the node group that are deployed with the earlier version of the launch template. +=== `PodEvictionFailure` errors during the upgrade phase -*Aggressive PDB*:: -Aggressive PDB is defined on the [.noloc]`Pod` or there are multiple PDBs pointing to the same [.noloc]`Pod`. +The following are known reasons which lead to a `PodEvictionFailure` error in this phase: +*Aggressive PDB*:: +Aggressive PDB is defined on the Pod or there are multiple PDBs pointing to the same Pod. *Deployment tolerating all the taints*:: -Once every [.noloc]`Pod` is evicted, it's expected for the node to be empty because the node is https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/[tainted] in the earlier steps. However, if the deployment tolerates every taint, then the node is more likely to be non-empty, leading to [.noloc]`Pod` eviction failure. +Once every Pod is evicted, it's expected for the node to be empty because the node is https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/[tainted] in the earlier steps. However, if the deployment tolerates every taint, then the node is more likely to be non-empty, leading to Pod eviction failure. -[[managed-node-update-scale-down,managed-node-update-scale-down.title]] +[#managed-node-update-scale-down] == Scale down phase The scale down phase decrements the Auto Scaling group maximum size and desired size by one to return to values before the update started. diff --git a/latest/ug/nodes/migrate-stack.adoc b/latest/ug/nodes/migrate-stack.adoc index ac74e543d..0b3f6768f 100644 --- a/latest/ug/nodes/migrate-stack.adoc +++ b/latest/ug/nodes/migrate-stack.adoc @@ -1,7 +1,7 @@ -//!!NODE_ROOT
include::../attributes.txt[] + [.topic] -[[migrate-stack,migrate-stack.title]] +[#migrate-stack] = Migrate applications to a new node group :info_titleabbrev: Migration @@ -50,17 +50,17 @@ An example output is as follows. CLUSTER NODEGROUP CREATED MIN SIZE MAX SIZE DESIRED CAPACITY INSTANCE TYPE IMAGE ID default standard-nodes 2019-05-01T22:26:58Z 1 4 3 t3.medium ami-05a71d034119ffc12 ---- -. Launch a new node group with `eksctl` with the following command. In the command, replace every [.replaceable]`example value` with your own values. The version number can't be later than the [.noloc]`Kubernetes` version for your control plane. Also, it can't be more than two minor versions earlier than the [.noloc]`Kubernetes` version for your control plane. We recommend that you use the same version as your control plane. +. Launch a new node group with `eksctl` with the following command. In the command, replace every [.replaceable]`example value` with your own values. The version number can't be later than the Kubernetes version for your control plane. Also, it can't be more than two minor versions earlier than the Kubernetes version for your control plane. We recommend that you use the same version as your control plane. + -We recommend blocking [.noloc]`Pod` access to IMDS if the following conditions are true: +We recommend blocking Pod access to IMDS if the following conditions are true: + -** You plan to assign IAM roles to all of your [.noloc]`Kubernetes` service accounts so that [.noloc]`Pods` only have the minimum permissions that they need. -** No [.noloc]`Pods` in the cluster require access to the Amazon EC2 instance metadata service (IMDS) for other reasons, such as retrieving the current {aws} Region. +** You plan to assign IAM roles to all of your Kubernetes service accounts so that Pods only have the minimum permissions that they need. +** No Pods in the cluster require access to the Amazon EC2 instance metadata service (IMDS) for other reasons, such as retrieving the current {aws} Region. + For more information, see https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node[Restrict access to the instance profile assigned to the worker node]. + -To block [.noloc]`Pod` access to IMDS, add the `--disable-pod-imds` option to the following command. +To block Pod access to IMDS, add the `--disable-pod-imds` option to the following command. + [NOTE] ==== @@ -72,7 +72,7 @@ For more available flags and their descriptions, see https://eksctl.io/. ---- eksctl create nodegroup \ --cluster my-cluster \ - --version 1.30 \ + --version {k8s-n} \ --name standard-nodes-new \ --node-type t3.medium \ --nodes 3 \ @@ -99,7 +99,7 @@ eksctl delete nodegroup --cluster my-cluster --name standard-nodes-old . Launch a new node group by following the steps that are outlined in <>. . When your stack has finished creating, select it in the console and choose *Outputs*. -. [[node-instance-role-step]]Record the *NodeInstanceRole* for the node group that was created. You need this to add the new Amazon EKS nodes to your cluster. +. [[node-instance-role-step]]Record the *NodeInstanceRole* for the node group that was created. You need this to add the new Amazon EKS nodes to your cluster. + NOTE: If you attached any additional IAM policies to your old node group IAM role, attach those same policies to your new node group IAM role to maintain that functionality on the new group. This applies to you if you added permissions for the https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler[Kubernetes Cluster Autoscaler], for example. . Update the security groups for both node groups so that they can communicate with each other. For more information, see <>. @@ -122,7 +122,7 @@ newSecGroup=$(aws cloudformation describe-stack-resources --stack-name $newNodes ---- .. Add ingress rules to each node security group so that they accept traffic from each other. + -The following {aws} CLI commands add inbound rules to each security group that allow all traffic on all protocols from the other security group. This configuration allows [.noloc]`Pods` in each node group to communicate with each other while you're migrating your workload to the new group. +The following {aws} CLI commands add inbound rules to each security group that allow all traffic on all protocols from the other security group. This configuration allows Pods in each node group to communicate with each other while you're migrating your workload to the new group. + [source,bash,subs="verbatim,attributes"] ---- @@ -138,7 +138,7 @@ aws ec2 authorize-security-group-ingress --group-id $newSecGroup \ kubectl edit configmap -n kube-system aws-auth ---- + -Add a new `mapRoles` entry for the new node group. If your cluster is in the {aws} GovCloud (US-East) or {aws} GovCloud (US-West) {aws} Regions, then replace `{arn-aws}` with `arn:aws-us-gov:`. +Add a new `mapRoles` entry for the new node group. + [source,yaml,subs="verbatim,attributes"] ---- @@ -157,7 +157,7 @@ data: - system:nodes ---- + -Replace the [.replaceable]`ARN of instance role (not instance profile)` snippet with the *NodeInstanceRole* value that you recorded in a <>. Then, save and close the file to apply the updated configmap. +Replace the [.replaceable]`ARN of instance role (not instance profile)` snippet with the *NodeInstanceRole* value that you recorded in a <>. Then, save and close the file to apply the updated configmap. . Watch the status of your nodes and wait for your new nodes to join your cluster and reach the `Ready` status. + [source,bash,subs="verbatim,attributes"] @@ -170,18 +170,18 @@ kubectl get nodes --watch ---- kubectl scale deployments/cluster-autoscaler --replicas=0 -n kube-system ---- -. Use the following command to taint each of the nodes that you want to remove with `NoSchedule`. This is so that new [.noloc]`Pods` aren't scheduled or rescheduled on the nodes that you're replacing. For more information, see https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/[Taints and Tolerations] in the [.noloc]`Kubernetes` documentation. +. Use the following command to taint each of the nodes that you want to remove with `NoSchedule`. This is so that new Pods aren't scheduled or rescheduled on the nodes that you're replacing. For more information, see https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/[Taints and Tolerations] in the Kubernetes documentation. + [source,bash,subs="verbatim,attributes"] ---- kubectl taint nodes node_name key=value:NoSchedule ---- + -If you're upgrading your nodes to a new [.noloc]`Kubernetes` version, you can identify and taint all of the nodes of a particular [.noloc]`Kubernetes` version (in this case, `1.28`) with the following code snippet. The version number can't be later than the [.noloc]`Kubernetes` version of your control plane. It also can't be more than two minor versions earlier than the [.noloc]`Kubernetes` version of your control plane. We recommend that you use the same version as your control plane. +If you're upgrading your nodes to a new Kubernetes version, you can identify and taint all of the nodes of a particular Kubernetes version (in this case, `{k8s-n-2}`) with the following code snippet. The version number can't be later than the Kubernetes version of your control plane. It also can't be more than two minor versions earlier than the Kubernetes version of your control plane. We recommend that you use the same version as your control plane. + [source,bash,subs="verbatim,attributes"] ---- -K8S_VERSION=1.28 +K8S_VERSION={k8s-n-2} nodes=$(kubectl get nodes -o jsonpath="{.items[?(@.status.nodeInfo.kubeletVersion==\"v$K8S_VERSION\")].metadata.name}") for node in ${nodes[@]} do @@ -196,7 +196,7 @@ done kubectl get deployments -l k8s-app=kube-dns -n kube-system ---- + -An example output is as follows. This cluster is using [.noloc]`CoreDNS` for DNS resolution, but your cluster can return `kube-dns` instead): +An example output is as follows. This cluster is using CoreDNS for DNS resolution, but your cluster can return `kube-dns` instead): + [source,bash,subs="verbatim,attributes"] ---- @@ -216,11 +216,11 @@ kubectl scale deployments/coredns --replicas=2 -n kube-system kubectl drain node_name --ignore-daemonsets --delete-local-data ---- + -If you're upgrading your nodes to a new [.noloc]`Kubernetes` version, identify and drain all of the nodes of a particular [.noloc]`Kubernetes` version (in this case, [.replaceable]`1.28`) with the following code snippet. +If you're upgrading your nodes to a new Kubernetes version, identify and drain all of the nodes of a particular Kubernetes version (in this case, [.replaceable]`{k8s-n-2}`) with the following code snippet. + [source,bash,subs="verbatim,attributes"] ---- -K8S_VERSION=1.28 +K8S_VERSION={k8s-n-2} nodes=$(kubectl get nodes -o jsonpath="{.items[?(@.status.nodeInfo.kubeletVersion==\"v$K8S_VERSION\")].metadata.name}") for node in ${nodes[@]} do @@ -253,7 +253,7 @@ aws ec2 revoke-security-group-ingress --group-id $newSecGroup \ .. Open the link:cloudformation/[{aws} CloudFormation console,type="console"]. .. Select your old node stack. .. Choose *Delete*. -.. In the *Delete stack* confirmation dialog box, choose *Delete stack*. +.. In the *Delete stack* confirmation dialog box, choose *Delete stack*. . Edit the `aws-auth` configmap to remove the old node instance role from RBAC. + [source,bash,subs="verbatim,attributes"] @@ -261,7 +261,7 @@ aws ec2 revoke-security-group-ingress --group-id $newSecGroup \ kubectl edit configmap -n kube-system aws-auth ---- + -Delete the `mapRoles` entry for the old node group. If your cluster is in the {aws} GovCloud (US-East) or {aws} GovCloud (US-West) {aws} Regions, then replace `{arn-aws}` with `arn:aws-us-gov:`. +Delete the `mapRoles` entry for the old node group. + [source,yaml,subs="verbatim,attributes"] ---- @@ -301,4 +301,4 @@ kubectl scale deployments/cluster-autoscaler --replicas=1 -n kube-system [source,bash,subs="verbatim,attributes"] ---- kubectl scale deployments/kube-dns --replicas=1 -n kube-system ----- +---- \ No newline at end of file diff --git a/latest/ug/nodes/monitoring-fargate-usage.adoc b/latest/ug/nodes/monitoring-fargate-usage.adoc index b57717404..f6dd00d62 100644 --- a/latest/ug/nodes/monitoring-fargate-usage.adoc +++ b/latest/ug/nodes/monitoring-fargate-usage.adoc @@ -1,7 +1,7 @@ -//!!NODE_ROOT
include::../attributes.txt[] + [.topic] -[[monitoring-fargate-usage,monitoring-fargate-usage.title]] +[#monitoring-fargate-usage] = Collect {aws} Fargate app and usage metrics :info_titleabbrev: Collect metrics @@ -10,21 +10,14 @@ include::../attributes.txt[] You can collect system metrics and CloudWatch usage metrics for {aws} Fargate. -- -[IMPORTANT] -==== - -{aws} Fargate with Amazon EKS isn't available in {aws} GovCloud (US-East) and {aws} GovCloud (US-West). - -==== - You can collect system metrics and CloudWatch usage metrics for {aws} Fargate. -[[fargate-application-metrics,fargate-application-metrics.title]] +[#fargate-application-metrics] == Application metrics -For applications running on Amazon EKS and {aws} Fargate, you can use the {aws} Distro for [.noloc]`OpenTelemetry` (ADOT). ADOT allows you to collect system metrics and send them to CloudWatch Container Insights dashboards. To get started with ADOT for applications running on Fargate, see https://aws-otel.github.io/docs/getting-started/container-insights[Using CloudWatch Container Insights with {aws} Distro for OpenTelemetry] in the ADOT documentation. +For applications running on Amazon EKS and {aws} Fargate, you can use the {aws} Distro for OpenTelemetry (ADOT). ADOT allows you to collect system metrics and send them to CloudWatch Container Insights dashboards. To get started with ADOT for applications running on Fargate, see https://aws-otel.github.io/docs/getting-started/container-insights[Using CloudWatch Container Insights with {aws} Distro for OpenTelemetry] in the ADOT documentation. -[[fargate-usage-metrics,fargate-usage-metrics.title]] +[#fargate-usage-metrics] == Usage metrics You can use CloudWatch usage metrics to provide visibility into your account's usage of resources. Use these metrics to visualize your current service usage on CloudWatch graphs and dashboards. @@ -33,7 +26,7 @@ You can use CloudWatch usage metrics to provide visibility into your account's u {aws} Fargate publishes the following metrics in the `{aws}/Usage` namespace. -[cols="1,1", options="header"] +[%header,cols="2"] |=== |Metric |Description @@ -45,7 +38,7 @@ You can use CloudWatch usage metrics to provide visibility into your account's u The following dimensions are used to refine the usage metrics that are published by {aws} Fargate. -[cols="1,1", options="header"] +[%header,cols="2"] |=== |Dimension |Description @@ -65,7 +58,7 @@ Currently, {aws} Fargate returns information on your Fargate On-Demand usage. Th [NOTE] ==== -Fargate On-Demand usage combines Amazon EKS [.noloc]`Pods` using Fargate, Amazon ECS tasks using the Fargate launch type and Amazon ECS tasks using the `FARGATE` capacity provider. +Fargate On-Demand usage combines Amazon EKS Pods using Fargate, Amazon ECS tasks using the Fargate launch type and Amazon ECS tasks using the `FARGATE` capacity provider. ==== @@ -73,7 +66,7 @@ Fargate On-Demand usage combines Amazon EKS [.noloc]`Pods` using Fargate, Amazon |The class of resource being tracked. Currently, {aws} Fargate doesn't use the class dimension. |=== -[[service-quota-alarm,service-quota-alarm.title]] +[#service-quota-alarm] === Creating a CloudWatch alarm to monitor Fargate resource usage metrics {aws} Fargate provides CloudWatch usage metrics that correspond to the {aws} service quotas for Fargate On-Demand resource usage. In the Service Quotas console, you can visualize your usage on a graph. You can also configure alarms that alert you when your usage approaches a service quota. For more information, see <>. @@ -82,8 +75,8 @@ Use the following steps to create a CloudWatch alarm based on the Fargate resour . Open the Service Quotas console at https://console.aws.amazon.com/servicequotas/. . In the left navigation pane, choose *{aws} services*. -. From the *{aws} services* list, search for and select *{aws} Fargate*. +. From the *{aws} services* list, search for and select *{aws} Fargate*. . In the *Service quotas* list, choose the Fargate usage quota you want to create an alarm for. . In the Amazon CloudWatch alarms section, choose *Create*. . For *Alarm threshold*, choose the percentage of your applied quota value that you want to set as the alarm value. -. For *Alarm name*, enter a name for the alarm and then choose *Create*. +. For *Alarm name*, enter a name for the alarm and then choose *Create*. \ No newline at end of file diff --git a/latest/ug/nodes/node-health.adoc b/latest/ug/nodes/node-health.adoc index d11d41ad9..194db406e 100644 --- a/latest/ug/nodes/node-health.adoc +++ b/latest/ug/nodes/node-health.adoc @@ -1,11 +1,10 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[node-health,node-health.title]] +[#node-health] = Enable node auto repair and investigate node health issues :info_titleabbrev: Node health -include::../attributes.txt[] - include::learn-status-conditions.adoc[leveloffset=+1] include::auto-get-logs.adoc[leveloffset=+1] @@ -17,16 +16,21 @@ You can use the node monitoring agent to show health issues and use node auto re Node health refers to the operational status and capability of a node to effectively run workloads. A healthy node maintains expected connectivity, has sufficient resources, and can successfully run Pods without disruption. For information on getting details about your nodes, see <> and <>. -To help with maintaining healthy nodes, Amazon EKS offers the node monitoring agent and node auto repair. +To help with maintaining healthy nodes, Amazon EKS offers the _node monitoring agent_ and _node auto repair_. + +[IMPORTANT] +==== +The _node monitoring agent_ and _node auto repair_ are only available on Linux. These features aren't available on Windows. +==== -[[node-monitoring-agent,node-monitoring-agent.title]] +[#node-monitoring-agent] == Node monitoring agent The node monitoring agent automatically reads node logs to detect certain health issues. It parses through node logs to detect failures and surfaces various status information about worker nodes. A dedicated `NodeCondition` is applied on the worker nodes for each category of issues detected, such as storage and networking issues. Descriptions of detected health issues are made available in the observability dashboard. For more information, see <>. The node monitoring agent is included as a capability for all Amazon EKS Auto Mode clusters. For other cluster types, you can add the monitoring agent as an Amazon EKS add-on. For more information, see <>. -[[node-auto-repair,node-auto-repair.title]] +[#node-auto-repair] == Node auto repair Node auto repair is an additional feature that continuously monitors the health of nodes, automatically reacting to detected problems and replacing nodes when possible. This helps overall availability of the cluster with minimal manual intervention. If a health check fails, the node is automatically cordoned so that no new Pods are scheduled on the node. @@ -43,12 +47,40 @@ Managed node groups will also automatically disable node repairs for safety reas You can enable node auto repair when creating or editing a managed node group. * When using the Amazon EKS console, activate the *Enable node auto repair* checkbox for the managed node group. For more information, see <>. - * When using the {aws} CLI, add the `--node-repair-config enabled=true` to the link:cli/latest/reference/eks/create-nodegroup.html[`eks create nodegroup`,type="documentation"] or link:cli/latest/reference/eks/update-nodegroup-config.html[`eks update-nodegroup-config`, type="documentation"] command. - * For an example `eksctl` `ClusterConfig` that uses a managed node group with node auto repair, see link:https://github.com/eksctl-io/eksctl/blob/main/examples/44-node-repair.yaml[44-node-repair.yaml] on GitHub. -[[node-health-issues,node-health-issues.title]] +Amazon EKS provides more granular control over the node auto repair behavior through the following: + +* `maxUnhealthyNodeThresholdCount` and `maxUnhealthyNodeThresholdPercentage` + +** These fields allow you to specify a count or percentage threshold of unhealthy nodes, above which node auto repair actions will stop. This provides more control over the "blast radius" of node auto repairs. + +** You can set either the absolute count or percentage, but not both. + +* `maxParallelNodesRepairedCount` and `maxParallelNodesRepairedPercentage` + +** These fields allow you to specify the maximum number of nodes that can be repaired concurrently or in parallel, expressed as either a count or percentage of all unhealthy nodes. This gives you finer-grained control over the pace of node replacements. +** As with the unhealthy node threshold, you can set either the absolute count or percentage, but not both. + +* `nodeRepairConfigOverrides` + +** This is a complex structure that allows you to set granular overrides for specific repair actions. These overrides control the repair action and the repair delay time before a node is considered eligible for repair. + +** The specific fields in this structure are: +*** `nodeMonitoringCondition`: The unhealthy condition reported by the node monitoring agent. +*** `nodeUnhealthyReason`: The reason why the node monitoring agent identified the node as unhealthy. +*** `minRepairWaitTimeMins`: The minimum time (in minutes) that the repair condition and unhealthy reason must persist before the node is eligible for repair. +*** `repairAction`: The action the repair system should take when the above conditions are met. + +** If you use this field, you must specify all the fields in the structure. You can also provide a list of these overrides. +** The `nodeMonitoringCondition` and `nodeUnhealthyReason` are manual text inputs that you set to indicate you want to deviate from the system's default behavior. +** The `minRepairWaitTimeMins` and `repairAction` fields allow you to specify deviations from the system's default behavior. + +// DO NOT EDIT THE FOLLOWING MANUALLY. +// table content is generated using scripts owned by the @eks-node-runtime team. + +[#node-health-issues] == Node health issues The following tables describe node health issues that can be detected by the node monitoring agent. There are two types of issues: @@ -57,331 +89,331 @@ The following tables describe node health issues that can be detected by the nod * Event – A temporary issue or sub-optimal node configuration. No auto repair action will take place. For more information, see <>. -[[node-health-kernel,node-health-kernel.title]] -=== Kernel node health issues +[#node-health-AcceleratedHardware] +=== AcceleratedHardware node health issues + +The monitoring condition is `AcceleratedHardwareReady` for issues in the following table that have a severity of “Condition”. -[cols="3", options="header"] +If auto repair is enabled, the repair actions that are listed start 10 minutes after the issue is detected. For more information on XID errors, see link:https://docs.nvidia.com/deploy/xid-errors/index.html#topic_5_1[Xid Errors] in the _NVIDIA GPU Deployment and Management Documentation_. For more information on the individual XID messages, see link:https://docs.nvidia.com/deploy/gpu-debug-guidelines/index.html#understanding-xid-messages[Understanding Xid Messages] in the _NVIDIA GPU Deployment and Management Documentation_. + +[%header,cols="3"] |=== |Name |Severity |Description -|ForkFailedOutOfPID +|DCGMDiagnosticFailure |Condition -|A fork or exec call has failed due to the system being out of process IDs or memory, which may be caused by zombie processes or physical memory exhaustion. +|A test case from the DCGM active diagnostics test suite failed. -|AppBlocked -|Event -|The task has been blocked for a long period of time from scheduling, usually caused by being blocked on input or output. +|DCGMError +|Condition +|Connection to the DCGM host process was lost or could not be established. -|AppCrash +|DCGMFieldError[Code] |Event -|An application on the node has crashed. +|DCGM detected GPU degredation through a field identifier. -|ApproachingKernelPidMax +|DCGMHealthCode[Code] |Event -|The number of processes is approaching the maximum number of PIDs that are available per the current kernel.pid_max setting, after which no more processes can be launched. +|A DCGM health check failed in a non-fatal manner. -|ApproachingMaxOpenFiles +|DCGMHealthCode[Code] +|Condition +|A DCGM health check failed in a fatal manner. + +|NeuronDMAError +|Condition +|A DMA engine encountered an unrecoverable error. + +|NeuronHBMUncorrectableError +|Condition +|An HBM encountered an uncorrectable error and produced incorrect results. + +|NeuronNCUncorrectableError +|Condition +|A Neuron Core uncorrectable memory error was detected. + +|NeuronSRAMUncorrectableError +|Condition +|An on-chip SRAM encountered a parity error and produced incorrect results. + +|NvidiaDeviceCountMismatch |Event -|The number of open files is approaching the maximum number of possible open files given the current kernel settings, after which opening new files will fail. +|The number of GPUs visible through NVML is inconsistent with the NVIDIA device count on the filesystem. -|ConntrackExceededKernel +|NvidiaDoubleBitError +|Condition +|A double bit error was produced by the GPU driver. + +|NvidiaNCCLError |Event -|Connection tracking exceeded the maximum for the kernel and new connections could not be established, which can result in packet loss. +|A segfault occurred in the NVIDIA Collective Communications library (`libnccl`). -|ExcessiveZombieProcesses +|NvidiaNVLinkError +|Condition +|NVLink errors were reported by the GPU driver. + +|NvidiaPCIeError |Event -|Processes which can't be fully reclaimed are accumulating in large numbers, which indicates application issues and may lead to reaching system process limits. +|PCIe replays were triggered to recover from transmission errors. -|KernelBug +|NvidiaPageRetirement |Event -|A kernel bug was detected and reported by the Linux kernel itself, though this may sometimes be caused by nodes with high CPU or memory usage leading to delayed event processing. +|The GPU driver has marked a memory page for retirement. This may occur if there is a single double bit error or two single bit errors are encountered at the same address. -|LargeEnvironment +|NvidiaPowerError |Event -|The number of environment variables for this process is larger than expected, potentially caused by many services with enableServiceLinks set to true, which may cause performance issues. +|Power utilization of GPUs breached the allowed thresholds. -|RapidCron +|NvidiaThermalError |Event -|A cron job is running faster than every five minutes on this node, which may impact performance if the job consumes significant resources. +|Thermal status of GPUs breached the allowed thresholds. -|SoftLockup +|NvidiaXID[Code]Error +|Condition +|A critical GPU error occurred. + +|NvidiaXID[Code]Warning |Event -|The CPU stalled for a given amount of time. +|A non-critical GPU error occurred. |=== -[[node-health-networking,node-health-networking.title]] -=== Networking node health issues +[#node-health-ContainerRuntime] +=== ContainerRuntime node health issues -[cols="3", options="header"] +The monitoring condition is `ContainerRuntimeReady` for issues in the following table that have a severity of “Condition”. + +[%header,cols="3"] |=== |Name |Severity |Description -|InterfaceNotRunning -|Condition -|This interface appears to not be running or there are network issues. +|ContainerRuntimeFailed +|Event +|The container runtime has failed to create a container, likely related to any reported issues if occurring repeatedly. -|InterfaceNotUp -|Condition -|This interface appears to not be up or there are network issues. +|DeprecatedContainerdConfiguration +|Event +|A container image using deprecated image manifest version 2, schema 1 was recently pulled onto the node through `containerd`. -|IPAMDNotReady -|Condition -|IPAMD fails to connect to the API server. +|KubeletFailed +|Event +|The kubelet entered a failed state. -|IPAMDNotRunning -|Condition -|The `aws-k8s-agent` process was not found to be running. +|LivenessProbeFailures +|Event +|A liveness probe failure was detected, potentially indicating application code issues or insufficient timeout values if occurring repeatedly. -|MissingLoopbackInterface +|PodStuckTerminating |Condition -|The loopback interface is missing from this instance, causing failure of services depending on local connectivity. +|A Pod is or was stuck terminating for an excessive amount of time, which can be caused by CRI errors preventing pod state progression. -|BandwidthInExceeded +|ReadinessProbeFailures |Event -|Packets have been queued or dropped because the inbound aggregate bandwidth exceeded the maximum for the instance. +|A readiness probe failure was detected, potentially indicating application code issues or insufficient timeout values if occurring repeatedly. -|BandwidthOutExceeded +|[Name]RepeatedRestart |Event -|Packets have been queued or dropped because the outbound aggregate bandwidth exceeded the maximum for the instance. +|A systemd unit is restarting frequently. -|ConntrackExceeded +|ServiceFailedToStart |Event -|Connection tracking exceeded the maximum for the instance and new connections could not be established, which can result in packet loss. +|A systemd unit failed to start. -|IPAMDNoIPs -|Event -|IPAM-D is out of IP addresses. +|=== -|IPAMDRepeatedlyRestart -|Event -|Multiple restarts in the IPAMD service have occurred. +[#node-health-Kernel] +=== Kernel node health issues -|KubeProxyNotReady -|Event -|Kube-proxy failed to watch or list resources. +The monitoring condition is `KernelReady` for issues in the following table that have a severity of “Condition”. -|LinkLocalExceeded -|Event -|Packets were dropped because the PPS of traffic to local proxy services exceeded the network interface maximum. +[%header,cols="3"] +|=== -|MissingDefaultRoutes -|Event -|There are missing default route rules. +|Name +|Severity +|Description -|MissingIPRules, MissingIPRoutes +|AppBlocked |Event -|There are missing route rules for the following Pod IPs from the route table. +|The task has been blocked for a long period of time from scheduling, usually caused by being blocked on input or output. -|NetworkSysctl +|AppCrash |Event -|This node's network sysctl settings are potentially incorrect. +|An application on the node has crashed. -|PortConflict +|ApproachingKernelPidMax |Event -|If a Pod uses hostPort, it can write iptables rules that override the host's already bound ports, potentially preventing API server access to `kubelet`. +|The number of processes is approaching the maximum number of PIDs that are available per the current `kernel.pid_max` setting, after which no more processes can be launched. -|PPSExceeded +|ApproachingMaxOpenFiles |Event -|Packets have been queued or dropped because the bidirectional PPS exceeded the maximum for the instance. +|The number of open files is approaching the maximum number of possible open files given the current kernel settings, after which opening new files will fail. -|UnexpectedRejectRule +|ConntrackExceededKernel |Event -|An unexpected `REJECT`` or `DROP` rule was found in the iptables, potentially blocking expected traffic. - -|=== - -[[node-health-neuron,node-health-neuron.title]] -=== Neuron node health issues - -[cols="3", options="header"] -|=== +|Connection tracking exceeded the maximum for the kernel and new connections could not be established, which can result in packet loss. -|Name -|Severity -|Description +|ExcessiveZombieProcesses +|Event +|Processes which can't be fully reclaimed are accumulating in large numbers, which indicates application issues and may lead to reaching system process limits. -|NeuronDMAError +|ForkFailedOutOfPIDs |Condition -|A DMA engine encountered an unrecoverable error. +|A fork or exec call has failed due to the system being out of process IDs or memory, which may be caused by zombie processes or physical memory exhaustion. -|NeuronHBMUncorrectableError -|Condition -|An HBM encountered an uncorrectable error and produced incorrect results. +|KernelBug +|Event +|A kernel bug was detected and reported by the Linux kernel itself, though this may sometimes be caused by nodes with high CPU or memory usage leading to delayed event processing. -|NeuronNCUncorrectableError -|Condition -|A Neuron Core uncorrectable memory error was detected. +|LargeEnvironment +|Event +|The number of environment variables for this process is larger than expected, potentially caused by many services with `enableServiceLinks` set to true, which may cause performance issues. -|NeuronSRAMUncorrectableError -|Condition -|An on-chip SRAM encountered a parity error and produced incorrect results. +|RapidCron +|Event +|A cron job is running faster than every five minutes on this node, which may impact performance if the job consumes significant resources. + +|SoftLockup +|Event +|The CPU stalled for a given amount of time. |=== -[[node-health-nvidia,node-health-nvidia.title]] -=== NVIDIA node health issues +[#node-health-Networking] +=== Networking node health issues -If auto repair is enabled, the repair actions that are listed start 10 minutes after the issue is detected. For more information on XID errors, see link:https://docs.nvidia.com/deploy/xid-errors/index.html#topic_5_1[Xid Errors] in the _NVIDIA GPU Deployment and Management Documentation_. For more information on the individual XID messages, see link:https://docs.nvidia.com/deploy/gpu-debug-guidelines/index.html#understanding-xid-messages[Understanding Xid Messages] in the _NVIDIA GPU Deployment and Management Documentation_. +The monitoring condition is `NetworkingReady` for issues in the following table that have a severity of “Condition”. -[cols="4", options="header"] +[%header,cols="3"] |=== |Name |Severity |Description -|Repair action - -|NvidiaDoubleBitError -|Condition -|A double bit error was produced by the GPU driver. -|Replace - -|NvidiaNVLinkError -|Condition -|NVLink errors were reported by the GPU driver. -|Replace - -|NvidiaXID13Error -|Condition -|There is a graphics engine exception. -|Reboot -|NvidiaXID31Error -|Condition -|There are suspected hardware problems. -|Reboot - -|NvidiaXID48Error -|Condition -|Double bit ECC errors are reported by the driver. -|Reboot - -|NvidiaXID63Error -|Condition -|There's a page retirement or row remap. -|Reboot +|BandwidthInExceeded +|Event +|Packets have been queued or dropped because the inbound aggregate bandwidth exceeded the maximum for the instance. -|NvidiaXID64Error -|Condition -|There are failures trying to retire a page or perform a node remap. -|Reboot +|BandwidthOutExceeded +|Event +|Packets have been queued or dropped because the outbound aggregate bandwidth exceeded the maximum for the instance. -|NvidiaXID74Error -|Condition -|There is a problem with a connection from the GPU to another GPU or NVSwitch over NVLink. This may indicate a hardware failure with the link itself or may indicate a problem with the device at the remote end of the link. -|Replace +|ConntrackExceeded +|Event +|Connection tracking exceeded the maximum for the instance and new connections could not be established, which can result in packet loss. -|NvidiaXID79Error -|Condition -|The GPU driver attempted to access the GPU over its PCI Express connection and found that the GPU is not accessible. -|Replace +|IPAMDInconsistentState +|Event +|The state of the IPAMD checkpoint on disk does not reflect the IPs in the container runtime. -|NvidiaXID94Error -|Condition -|There are ECC memory errors. -|Reboot +|IPAMDNoIPs +|Event +|IPAMD is out of IP addresses. -|NvidiaXID95Error +|IPAMDNotReady |Condition -|There are ECC memory errors. -|Reboot +|IPAMD fails to connect to the API server. -|NvidiaXID119Error +|IPAMDNotRunning |Condition -|The GSP timed out responding to RPC requests from other bits in the driver. -|Replace +|The Amazon VPC CNI process was not found to be running. -|NvidiaXID120Error -|Condition -|The GSP has responded in time, but with an error. -|Replace +|IPAMDRepeatedlyRestart +|Event +|Multiple restarts in the IPAMD service have occurred. -|NvidiaXID121Error +|InterfaceNotRunning |Condition -|C2C is chip interconnect. It enables sharing memory between CPUs, accelerators, and more. -|Replace +|This interface appears to not be running or there are network issues. -|NvidiaXID140Error +|InterfaceNotUp |Condition -|The GPU driver may have observed uncorrectable errors in GPU memory, in such a way as to interrupt the GPU driver's ability to mark the pages for dynamic page offlining or row remapping. -|Replace +|This interface appears to not be up or there are network issues. -|NvidiaPageRetirement +|KubeProxyNotReady |Event -|The GPU driver has marked a memory page for retirement. This may occur if there is a single double bit error or two single bit errors are encountered at the same address. -|None +|Kube-proxy failed to watch or list resources. -|NvidiaXID[Code]Warning +|LinkLocalExceeded |Event -|Any occurrences of XIDs other than the ones defined in this list result in this event. -|None - -|=== - -[[node-health-runtime,node-health-runtime.title]] -=== Runtime node health issues - -[cols="3", options="header"] -|=== +|Packets were dropped because the PPS of traffic to local proxy services exceeded the network interface maximum. -|Name -|Severity -|Description +|MACAddressPolicyMisconfigured +|Event +|The systemd-networkd link configuration has the incorrect `MACAddressPolicy` value. -|PodStuckTerminating -|Condition -|A Pod is or was stuck terminating for an excessive amount of time, which can be caused by CRI errors preventing pod state progression. +|MissingDefaultRoutes +|Event +|There are missing default route rules. -|%sRepeatedRestart +|MissingIPRoutes |Event -|Restarts of any systemd service on the node (formatted using the title-cased unit name). +|There are missing routes for Pod IPs. -|ContainerRuntimeFailed +|MissingIPRules |Event -|The container runtime has failed to create a container, likely related to any reported issues if occurring repeatedly. +|There are missing rules for Pod IPs. -|KubeletFailed +|MissingLoopbackInterface +|Condition +|The loopback interface is missing from this instance, causing failure of services depending on local connectivity. + +|NetworkSysctl |Event -|The kubelet entered a failed state. +|This node's network `sysctl` settings are potentially incorrect. -|LivenessProbeFailures +|PPSExceeded |Event -|A liveness probe failure was detected, potentially indicating application code issues or insufficient timeout values if occurring repeatedly. +|Packets have been queued or dropped because the bidirectional PPS exceeded the maximum for the instance. -|ReadinessProbeFailures +|PortConflict |Event -|A readiness probe failure was detected, potentially indicating application code issues or insufficient timeout values if occurring repeatedly. +|If a Pod uses hostPort, it can write `iptables` rules that override the host's already bound ports, potentially preventing API server access to `kubelet`. -|ServiceFailedToStart +|UnexpectedRejectRule |Event -|A systemd unit failed to start. +|An unexpected `REJECT` or `DROP` rule was found in the `iptables`, potentially blocking expected traffic. |=== -[[node-health-storage,node-health-storage.title]] +[#node-health-Storage] === Storage node health issues -[cols="3", options="header"] +The monitoring condition is `StorageReady` for issues in the following table that have a severity of “Condition”. + +[%header,cols="3"] |=== |Name |Severity |Description -|XFSSmallAverageClusterSize -|Condition -|The XFS Average Cluster size is small, indicating excessive free space fragmentation that can prevent file creation despite available inodes or free space. +|EBSInstanceIOPSExceeded +|Event +|Maximum IOPS for the instance was exceeded. + +|EBSInstanceThroughputExceeded +|Event +|Maximum Throughput for the instance was exceeded. + +|EBSVolumeIOPSExceeded +|Event +|Maximum IOPS to a particular EBS Volume was exceeded. + +|EBSVolumeThroughputExceeded +|Event +|Maximum Throughput to a particular Amazon EBS volume was exceeded. |EtcHostsMountFailed |Event -|Mounting of the kubelet generated `/etc/hosts` failed due to userdata remounting `/var/lib/kubelet/pods` during kubelet-container operation. +|Mounting of the kubelet generated `/etc/hosts` failed due to userdata remounting `/var/lib/kubelet/pods` during `kubelet-container` operation. |IODelays |Event @@ -389,6 +421,10 @@ If auto repair is enabled, the repair actions that are listed start 10 minutes a |KubeletDiskUsageSlow |Event -|Kubelet is reporting slow disk usage while trying to access the filesystem, potentially indicating insufficient disk input-output or filesystem issues. +|The `kubelet` is reporting slow disk usage while trying to access the filesystem. This potentially indicates insufficient disk input-output or filesystem issues. + +|XFSSmallAverageClusterSize +|Event +|The XFS Average Cluster size is small, indicating excessive free space fragmentation. This can prevent file creation despite available inodes or free space. |=== diff --git a/latest/ug/nodes/retrieve-ami-id-bottlerocket.adoc b/latest/ug/nodes/retrieve-ami-id-bottlerocket.adoc index d908e9041..e6a7b6802 100644 --- a/latest/ug/nodes/retrieve-ami-id-bottlerocket.adoc +++ b/latest/ug/nodes/retrieve-ami-id-bottlerocket.adoc @@ -1,11 +1,10 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[retrieve-ami-id-bottlerocket,retrieve-ami-id-bottlerocket.title]] -= Retrieve recommended [.noloc]`Bottlerocket` AMI IDs +[#retrieve-ami-id-bottlerocket] += Retrieve recommended Bottlerocket AMI IDs :info_titleabbrev: Get latest IDs -include::../attributes.txt[] - [abstract] -- You can retrieve the Amazon Machine Image (AMI) ID for Amazon EKS optimized AMIs by querying the {aws} Systems Manager Parameter Store API. @@ -13,9 +12,9 @@ You can retrieve the Amazon Machine Image (AMI) ID for Amazon EKS optimized AMIs When deploying nodes, you can specify an ID for a pre-built Amazon EKS optimized Amazon Machine Image (AMI). To retrieve an AMI ID that fits your desired configuration, query the {aws} Systems Manager Parameter Store API. Using this API eliminates the need to manually look up Amazon EKS optimized AMI IDs. For more information, see link:systems-manager/latest/APIReference/API_GetParameter.html[GetParameter,type="documentation"]. The link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] that you use must have the `ssm:GetParameter` IAM permission to retrieve the Amazon EKS optimized AMI metadata. -You can retrieve the image ID of the latest recommended Amazon EKS optimized [.noloc]`Bottlerocket` AMI with the following {aws} CLI command, which uses the sub-parameter `image_id`. Make the following modifications to the command as needed and then run the modified command: +You can retrieve the image ID of the latest recommended Amazon EKS optimized Bottlerocket AMI with the following {aws} CLI command, which uses the sub-parameter `image_id`. Make the following modifications to the command as needed and then run the modified command: -* Replace [.replaceable]`kubernetes-version` with a supported <>. +* Replace [.replaceable]`kubernetes-version` with a supported link:eks/latest/userguide/platform-versions.html[platform-version,type="documentation"]. * Replace [.replaceable]`-flavor` with one of the following options. + ** Remove [.replaceable]`-flavor` for variants without a GPU. @@ -47,4 +46,4 @@ An example output is as follows. [source,bash,subs="verbatim,attributes,quotes"] ---- ami-[.replaceable]`1234567890abcdef0` ----- +---- \ No newline at end of file diff --git a/latest/ug/nodes/retrieve-ami-id.adoc b/latest/ug/nodes/retrieve-ami-id.adoc index 197fa8989..405e28712 100644 --- a/latest/ug/nodes/retrieve-ami-id.adoc +++ b/latest/ug/nodes/retrieve-ami-id.adoc @@ -1,11 +1,10 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[retrieve-ami-id,retrieve-ami-id.title]] +[#retrieve-ami-id] = Retrieve recommended Amazon Linux AMI IDs :info_titleabbrev: Get latest IDs -include::../attributes.txt[] - [abstract] -- You can programmatically retrieve the Amazon Machine Image (AMI) ID for Amazon EKS optimized AMIs by querying the {aws} Systems Manager Parameter Store API. @@ -15,22 +14,23 @@ When deploying nodes, you can specify an ID for a pre-built Amazon EKS optimized You can retrieve the image ID of the latest recommended Amazon EKS optimized Amazon Linux AMI with the following command, which uses the sub-parameter `image_id`. Make the following modifications to the command as needed and then run the modified command: -* Replace [.replaceable]`kubernetes-version` with a supported <>. +* Replace `` with an link:eks/latest/userguide/kubernetes-versions.html[Amazon EKS supported version,type="documentation"]. * Replace [.replaceable]`ami-type` with one of the following options. For information about the types of Amazon EC2 instances, see link:AWSEC2/latest/UserGuide/instance-types.html[Amazon EC2 instance types,type="documentation"]. + ** Use [.replaceable]`amazon-linux-2023/x86_64/standard` for Amazon Linux 2023 (AL2023) `x86` based instances. -** Use [.replaceable]`amazon-linux-2023/arm64/standard` for AL2023 ARM instances. -** Use [.replaceable]`amazon-linux-2023/x86_64/nvidia` for the latest approved AL2023 NVIDIA instances. +** Use [.replaceable]`amazon-linux-2023/arm64/standard` for AL2023 ARM instances, such as link:ec2/graviton/[{aws} Graviton,type="marketing"] based instances. +** Use [.replaceable]`amazon-linux-2023/x86_64/nvidia` for the latest approved AL2023 NVIDIA `x86` based instances. +** Use [.replaceable]`amazon-linux-2023/arm64/nvidia` for the latest approved AL2023 NVIDIA `arm64` based instances. ** Use [.replaceable]`amazon-linux-2023/x86_64/neuron` for the latest AL2023 link:machine-learning/neuron/[{aws} Neuron,type="marketing"] instances. ** Use [.replaceable]`amazon-linux-2` for Amazon Linux 2 (AL2) `x86` based instances. ** Use [.replaceable]`amazon-linux-2-arm64` for AL2 ARM instances, such as link:ec2/graviton/[{aws} Graviton,type="marketing"] based instances. -** Use [.replaceable]`amazon-linux-2-gpu` for AL2 link:AWSEC2/latest/UserGuide/accelerated-computing-instances.html[hardware accelerated,type="documentation"] `x86` based instances for [.noloc]`NVIDIA` GPU, link:machine-learning/inferentia/[Inferentia,type="marketing"], and link:machine-learning/trainium/[Trainium,type="marketing"] based workloads. -* Replace [.replaceable]`region-code` with an link:general/latest/gr/eks.html[Amazon EKS supported {aws} Region,type="documentation"] for which you want the AMI ID. +** Use [.replaceable]`amazon-linux-2-gpu` for AL2 link:AWSEC2/latest/UserGuide/accelerated-computing-instances.html[hardware accelerated,type="documentation"] `x86` based instances for NVIDIA GPU, link:machine-learning/inferentia/[Inferentia,type="marketing"], and link:machine-learning/trainium/[Trainium,type="marketing"] based workloads. +* Replace `` with an link:general/latest/gr/eks.html[Amazon EKS supported {aws} Region,type="documentation"] for which you want the AMI ID. -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- -aws ssm get-parameter --name /aws/service/eks/optimized-ami/[.replaceable]`kubernetes-version`/[.replaceable]`ami-type`/recommended/image_id \ - --region [.replaceable]`region-code` --query "Parameter.Value" --output text +aws ssm get-parameter --name /aws/service/eks/optimized-ami///recommended/image_id \ + --region --query "Parameter.Value" --output text ---- Here's an example command after placeholder replacements have been made. diff --git a/latest/ug/nodes/retrieve-windows-ami-id.adoc b/latest/ug/nodes/retrieve-windows-ami-id.adoc index 861ca2632..424b77505 100644 --- a/latest/ug/nodes/retrieve-windows-ami-id.adoc +++ b/latest/ug/nodes/retrieve-windows-ami-id.adoc @@ -1,11 +1,10 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[retrieve-windows-ami-id,retrieve-windows-ami-id.title]] -= Retrieve recommended [.noloc]`Microsoft Windows` AMI IDs +[#retrieve-windows-ami-id] += Retrieve recommended Microsoft Windows AMI IDs :info_titleabbrev: Get latest IDs -include::../attributes.txt[] - [abstract] -- You can programmatically retrieve the Amazon Machine Image (AMI) ID for Amazon EKS optimized AMIs by querying the {aws} Systems Manager Parameter Store API. @@ -13,17 +12,17 @@ You can programmatically retrieve the Amazon Machine Image (AMI) ID for Amazon E When deploying nodes, you can specify an ID for a pre-built Amazon EKS optimized Amazon Machine Image (AMI). To retrieve an AMI ID that fits your desired configuration, query the {aws} Systems Manager Parameter Store API. Using this API eliminates the need to manually look up Amazon EKS optimized AMI IDs. For more information, see link:systems-manager/latest/APIReference/API_GetParameter.html[GetParameter,type="documentation"]. The link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] that you use must have the `ssm:GetParameter` IAM permission to retrieve the Amazon EKS optimized AMI metadata. -You can retrieve the image ID of the latest recommended Amazon EKS optimized [.noloc]`Windows` AMI with the following command, which uses the sub-parameter `image_id`. Make the following modifications to the command as needed and then run the modified command: +You can retrieve the image ID of the latest recommended Amazon EKS optimized Windows AMI with the following command, which uses the sub-parameter `image_id`. Make the following modifications to the command as needed and then run the modified command: * Replace [.replaceable]`release` with one of the following options. + -** Use [.replaceable]`2022` for [.noloc]`Windows` Server 2022, but only if you're using [.noloc]`Kubernetes` version `1.24` or later. -** Use [.replaceable]`2019` for [.noloc]`Windows` Server 2019. +** Use [.replaceable]`2022` for Windows Server 2022. +** Use [.replaceable]`2019` for Windows Server 2019. * Replace [.replaceable]`installation-option` with one of the following options. For more information, see https://learn.microsoft.com/en-us/windows-server/administration/server-core/what-is-server-core[What is the Server Core installation option in Windows Server]. + ** Use [.replaceable]`Core` for a minimal installation with a smaller attack surface. -** Use [.replaceable]`Full` to include the [.noloc]`Windows` desktop experience. -* Replace [.replaceable]`kubernetes-version` with a supported <>. +** Use [.replaceable]`Full` to include the Windows desktop experience. +* Replace [.replaceable]`kubernetes-version` with a supported link:eks/latest/userguide/platform-versions.html[platform-version,type="documentation"]. * Replace [.replaceable]`region-code` with an link:general/latest/gr/eks.html[Amazon EKS supported {aws} Region,type="documentation"] for which you want the AMI ID. [source,bash,subs="verbatim,attributes,quotes"] @@ -36,7 +35,7 @@ Here's an example command after placeholder replacements have been made. [source,bash,subs="verbatim,attributes,quotes"] ---- -aws ssm get-parameter --name /aws/service/ami-windows-latest/Windows_Server-[.replaceable]`2022`-English-[.replaceable]`Core`-EKS_Optimized-[.replaceable]`1.31`/image_id \ +aws ssm get-parameter --name /aws/service/ami-windows-latest/Windows_Server-[.replaceable]`2022`-English-[.replaceable]`Core`-EKS_Optimized-[.replaceable]`k8s-n-2`/image_id \ --region [.replaceable]`us-west-2` --query "Parameter.Value" --output text ---- @@ -45,4 +44,4 @@ An example output is as follows. [source,bash,subs="verbatim,attributes,quotes"] ---- ami-[.replaceable]`1234567890abcdef0` ----- +---- \ No newline at end of file diff --git a/latest/ug/nodes/self-managed-windows-server-2022.adoc b/latest/ug/nodes/self-managed-windows-server-2022.adoc index 4b339d382..ff20f858d 100644 --- a/latest/ug/nodes/self-managed-windows-server-2022.adoc +++ b/latest/ug/nodes/self-managed-windows-server-2022.adoc @@ -1,21 +1,21 @@ -//!!NODE_ROOT
include::../attributes.txt[] + [.topic] -[[self-managed-windows-server-2022,self-managed-windows-server-2022.title]] -= Create self-managed [.noloc]`Windows` Server 2022 nodes with `eksctl` -:info_titleabbrev: Create Windows Server 2022 nodes +[#self-managed-windows-server-2022] += Create self-managed Windows Server 2022 nodes with `eksctl` +:info_titleabbrev: Windows Server 2022 [abstract] -- -This topic includes a YAML file as reference for creating self-managed [.noloc]`Windows` Server 2022 nodes. +This topic includes a YAML file as reference for creating self-managed Windows Server 2022 nodes. -- -You can use the following `test-windows-2022.yaml` as reference for creating self-managed [.noloc]`Windows` Server 2022 nodes. Replace every [.replaceable]`example value` with your own values. +You can use the following `test-windows-2022.yaml` as reference for creating self-managed Windows Server 2022 nodes. Replace every [.replaceable]`example value` with your own values. [NOTE] ==== -You must use `eksctl` version https://github.com/weaveworks/eksctl/releases/tag/v0.116.0[0.116.0] or later to run self-managed [.noloc]`Windows` Server 2022 nodes. +You must use `eksctl` version https://github.com/weaveworks/eksctl/releases/tag/v0.116.0[0.116.0] or later to run self-managed Windows Server 2022 nodes. ==== @@ -27,7 +27,7 @@ kind: ClusterConfig metadata: name: windows-2022-cluster region: region-code - version: '1.31' + version: '{k8s-n}' nodeGroups: - name: windows-ng @@ -47,4 +47,4 @@ The node groups can then be created using the following command. [source,bash,subs="verbatim,attributes"] ---- eksctl create cluster -f test-windows-2022.yaml ----- +---- \ No newline at end of file diff --git a/latest/ug/nodes/update-managed-node-group.adoc b/latest/ug/nodes/update-managed-node-group.adoc index caf056281..8f146a8d2 100644 --- a/latest/ug/nodes/update-managed-node-group.adoc +++ b/latest/ug/nodes/update-managed-node-group.adoc @@ -1,40 +1,39 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[update-managed-node-group,update-managed-node-group.title]] +[#update-managed-node-group] = Update a managed node group for your cluster :info_titleabbrev: Update -include::../attributes.txt[] - [abstract] -- When you initiate a managed node group update, Amazon EKS automatically updates your nodes for you. -- -When you initiate a managed node group update, Amazon EKS automatically updates your nodes for you, completing the steps listed in <>. If you're using an Amazon EKS optimized AMI, Amazon EKS automatically applies the latest security patches and operating system updates to your nodes as part of the latest AMI release version. +When you initiate a managed node group update, Amazon EKS automatically updates your nodes for you, completing the steps listed in <>. If you're using an Amazon EKS optimized AMI, Amazon EKS automatically applies the latest security patches and operating system updates to your nodes as part of the latest AMI release version. There are several scenarios where it's useful to update your Amazon EKS managed node group's version or configuration: -* You have updated the [.noloc]`Kubernetes` version for your Amazon EKS cluster and want to update your nodes to use the same [.noloc]`Kubernetes` version. +* You have updated the Kubernetes version for your Amazon EKS cluster and want to update your nodes to use the same Kubernetes version. * A new AMI release version is available for your managed node group. For more information about AMI versions, see these sections: + ** <> ** <> ** <> * You want to adjust the minimum, maximum, or desired count of the instances in your managed node group. -* You want to add or remove [.noloc]`Kubernetes` labels from the instances in your managed node group. +* You want to add or remove Kubernetes labels from the instances in your managed node group. * You want to add or remove {aws} tags from your managed node group. * You need to deploy a new version of a launch template with configuration changes, such as an updated custom AMI. -* You have deployed version `1.9.0` or later of the Amazon VPC CNI add-on, enabled the add-on for prefix delegation, and want new {aws} [.noloc]`Nitro System` instances in a node group to support a significantly increased number of [.noloc]`Pods`. For more information, see <>. -* You have enabled IP prefix delegation for Windows nodes and want new {aws} Nitro System instances in a node group to support a significantly increased number of [.noloc]`Pods`. For more information, see <>. +* You have deployed version `1.9.0` or later of the Amazon VPC CNI add-on, enabled the add-on for prefix delegation, and want new {aws} Nitro System instances in a node group to support a significantly increased number of Pods. For more information, see <>. +* You have enabled IP prefix delegation for Windows nodes and want new {aws} Nitro System instances in a node group to support a significantly increased number of Pods. For more information, see <>. -If there's a newer AMI release version for your managed node group's [.noloc]`Kubernetes` version, you can update your node group's version to use the newer AMI version. Similarly, if your cluster is running a [.noloc]`Kubernetes` version that's newer than your node group, you can update the node group to use the latest AMI release version to match your cluster's [.noloc]`Kubernetes` version. +If there's a newer AMI release version for your managed node group's Kubernetes version, you can update your node group's version to use the newer AMI version. Similarly, if your cluster is running a Kubernetes version that's newer than your node group, you can update the node group to use the latest AMI release version to match your cluster's Kubernetes version. -When a node in a managed node group is terminated due to a scaling operation or update, the [.noloc]`Pods` in that node are drained first. For more information, see <>. +When a node in a managed node group is terminated due to a scaling operation or update, the Pods in that node are drained first. For more information, see <>. -[[mng-update,mng-update.title]] +[#mng-update] == Update a node group version You can update a node group version with either of the following: @@ -48,7 +47,7 @@ The version that you update to can't be greater than the control plane's version *Update a managed node group using `eksctl`* -Update a managed node group to the latest AMI release of the same [.noloc]`Kubernetes` version that's currently deployed on the nodes with the following command. Replace every [.replaceable]`example value` with your own values. +Update a managed node group to the latest AMI release of the same Kubernetes version that's currently deployed on the nodes with the following command. Replace every [.replaceable]`example value` with your own values. [source,bash,subs="verbatim,attributes"] ---- @@ -62,7 +61,7 @@ NOTE: If you're upgrading a node group that's deployed with a launch template to You can't directly upgrade a node group that's deployed without a launch template to a new launch template version. Instead, you must deploy a new node group using the launch template to update the node group to a new launch template version. -You can upgrade a node group to the same version as the control plane's [.noloc]`Kubernetes` version. For example, if you have a cluster running [.noloc]`Kubernetes` `1.29`, you can upgrade nodes currently running [.noloc]`Kubernetes` `1.28` to version `1.29` with the following command. +You can upgrade a node group to the same version as the control plane's Kubernetes version. For example, if you have a cluster running Kubernetes `{k8s-n}`, you can upgrade nodes currently running Kubernetes `{k8s-n-1}` to version `{k8s-n}` with the following command. [source,bash,subs="verbatim,attributes"] ---- @@ -70,7 +69,7 @@ eksctl upgrade nodegroup \ --name=node-group-name \ --cluster=my-cluster \ --region=region-code \ - --kubernetes-version=1.29 + --kubernetes-version={k8s-n} ---- == {aws-management-console} [[console_update_managed_nodegroup]] @@ -78,7 +77,7 @@ eksctl upgrade nodegroup \ . Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. . Choose the cluster that contains the node group to update. -. If at least one node group has an available update, a box appears at the top of the page notifying you of the available update. If you select the *Compute* tab, you'll see *Update now* in the *AMI release version* column in the *Node groups* table for the node group that has an available update. To update the node group, choose *Update now*. +. If at least one node group has an available update, a box appears at the top of the page notifying you of the available update. If you select the *Compute* tab, you'll see *Update now* in the *AMI release version* column in the *Node groups* table for the node group that has an available update. To update the node group, choose *Update now*. + You won't see a notification for node groups that were deployed with a custom AMI. If your nodes are deployed with a custom AMI, complete the following steps to deploy a new updated custom AMI. + @@ -89,15 +88,15 @@ You won't see a notification for node groups that were deployed with a custom AM . On the *Update node group version* dialog box, activate or deactivate the following options: + ** *Update node group version* – This option is unavailable if you deployed a custom AMI or your Amazon EKS optimized AMI is currently on the latest version for your cluster. -** *Change launch template version* – This option is unavailable if the node group is deployed without a custom launch template. You can only update the launch template version for a node group that has been deployed with a custom launch template. Select the *Launch template version* that you want to update the node group to. If your node group is configured with a custom AMI, then the version that you select must also specify an AMI. When you upgrade to a newer version of your launch template, every node is recycled to match the new configuration of the launch template version specified. +** *Change launch template version* – This option is unavailable if the node group is deployed without a custom launch template. You can only update the launch template version for a node group that has been deployed with a custom launch template. Select the *Launch template version* that you want to update the node group to. If your node group is configured with a custom AMI, then the version that you select must also specify an AMI. When you upgrade to a newer version of your launch template, every node is recycled to match the new configuration of the launch template version specified. . For *Update strategy*, select one of the following options: + -** *Rolling update* – This option respects the [.noloc]`Pod` disruption budgets for your cluster. Updates fail if there's a [.noloc]`Pod` disruption budget issue that causes Amazon EKS to be unable to gracefully drain the [.noloc]`Pods` that are running on this node group. -** *Force update* – This option doesn't respect [.noloc]`Pod` disruption budgets. Updates occur regardless of [.noloc]`Pod` disruption budget issues by forcing node restarts to occur. +** *Rolling update* – This option respects the Pod disruption budgets for your cluster. Updates fail if there's a Pod disruption budget issue that causes Amazon EKS to be unable to gracefully drain the Pods that are running on this node group. +** *Force update* – This option doesn't respect Pod disruption budgets. Updates occur regardless of Pod disruption budget issues by forcing node restarts to occur. . Choose *Update*. -[[mng-edit,mng-edit.title]] +[#mng-edit] == Edit a node group configuration You can modify some of the configurations of a managed node group. @@ -113,16 +112,16 @@ You can modify some of the configurations of a managed node group. *** *Desired size* – Specify the current number of nodes that the managed node group should maintain. *** *Minimum size* – Specify the minimum number of nodes that the managed node group can scale in to. *** *Maximum size* – Specify the maximum number of nodes that the managed node group can scale out to. For the maximum number of nodes supported in a node group, see <>. -.. (Optional) Add or remove *[.noloc]`Kubernetes` labels* to the nodes in your node group. The labels shown here are only the labels that you have applied with Amazon EKS. Other labels may exist on your nodes that aren't shown here. -.. (Optional) Add or remove *[.noloc]`Kubernetes` taints* to the nodes in your node group. Added taints can have the effect of either `*NoSchedule*`, `*NoExecute*`, or `*PreferNoSchedule*`. For more information, see <>. +.. (Optional) Add or remove *Kubernetes labels* to the nodes in your node group. The labels shown here are only the labels that you have applied with Amazon EKS. Other labels may exist on your nodes that aren't shown here. +.. (Optional) Add or remove *Kubernetes taints* to the nodes in your node group. Added taints can have the effect of either `*NoSchedule*`, `*NoExecute*`, or `*PreferNoSchedule*`. For more information, see <>. .. (Optional) Add or remove *Tags* from your node group resource. These tags are only applied to the Amazon EKS node group. They don't propagate to other resources, such as subnets or Amazon EC2 instances in the node group. -.. (Optional) Edit the *Node Group update configuration*. Select either *Number* or *Percentage*. +.. (Optional) Edit the *Node Group update configuration*. Select either *Number* or *Percentage*. + *** *Number* – Select and specify the number of nodes in your node group that can be updated in parallel. These nodes will be unavailable during update. *** *Percentage* – Select and specify the percentage of nodes in your node group that can be updated in parallel. These nodes will be unavailable during update. This is useful if you have many nodes in your node group. .. When you're finished editing, choose *Save changes*. -IMPORTANT: When updating the node group configuration, modifying the link:eks/latest/APIReference/API_NodegroupScalingConfig.html[`NodegroupScalingConfig`,type="documentation"] does not respect [.noloc]`Pod` disruption budgets (PDBs). +IMPORTANT: When updating the node group configuration, modifying the link:eks/latest/APIReference/API_NodegroupScalingConfig.html[`NodegroupScalingConfig`,type="documentation"] does not respect Pod disruption budgets (PDBs). Unlike the <> process (which drains nodes and respects PDBs during the upgrade phase), updating the scaling configuration causes nodes to be terminated immediately through an Auto Scaling Group (ASG) scale-down call. This happens without considering PDBs, regardless of the target size you're scaling down to. -That means when you reduce the `desiredSize` of an Amazon EKS managed node group, [.noloc]`Pods` are evicted as soon as the nodes are terminated, without honoring any PDBs. +That means when you reduce the `desiredSize` of an Amazon EKS managed node group, Pods are evicted as soon as the nodes are terminated, without honoring any PDBs. \ No newline at end of file diff --git a/latest/ug/nodes/update-stack.adoc b/latest/ug/nodes/update-stack.adoc index 122b92c79..e990da962 100644 --- a/latest/ug/nodes/update-stack.adoc +++ b/latest/ug/nodes/update-stack.adoc @@ -1,16 +1,16 @@ -//!!NODE_ROOT
include::../attributes.txt[] + [.topic] -[[update-stack,update-stack.title]] +[#update-stack] = Update an {aws} CloudFormation node stack -:info_titleabbrev: {aws} CloudFormation stack +:info_titleabbrev: CloudFormation stack [abstract] -- This topic describes how you can update an existing {aws} CloudFormation self-managed node stack with a new AMI. -- -This topic describes how you can update an existing {aws} CloudFormation self-managed node stack with a new AMI. You can use this procedure to update your nodes to a new version of [.noloc]`Kubernetes` following a cluster update. Otherwise, you can update to the latest Amazon EKS optimized AMI for an existing [.noloc]`Kubernetes` version. +This topic describes how you can update an existing {aws} CloudFormation self-managed node stack with a new AMI. You can use this procedure to update your nodes to a new version of Kubernetes following a cluster update. Otherwise, you can update to the latest Amazon EKS optimized AMI for an existing Kubernetes version. [IMPORTANT] ==== @@ -34,7 +34,7 @@ This method isn't supported for node groups that were created with `eksctl`. If kubectl get deployments -l k8s-app=kube-dns -n kube-system ---- + -An example output is as follows. This cluster is using [.noloc]`CoreDNS` for DNS resolution, but your cluster might return `kube-dns` instead. Your output might look different depending on the version of `kubectl` that you're using. +An example output is as follows. This cluster is using CoreDNS for DNS resolution, but your cluster might return `kube-dns` instead. Your output might look different depending on the version of `kubectl` that you're using. + [source,bash,subs="verbatim,attributes"] ---- @@ -47,7 +47,7 @@ coredns 1 1 1 1 31m ---- kubectl scale deployments/coredns --replicas=2 -n kube-system ---- -. (Optional) If you're using the [.noloc]`Kubernetes` https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md[Cluster Autoscaler], scale the deployment down to zero (0) replicas to avoid conflicting scaling actions. +. (Optional) If you're using the Kubernetes https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md[Cluster Autoscaler], scale the deployment down to zero (0) replicas to avoid conflicting scaling actions. + [source,bash,subs="verbatim,attributes"] ---- @@ -57,41 +57,41 @@ kubectl scale deployments/cluster-autoscaler --replicas=0 -n kube-system + .. Open the Amazon EC2 console at https://console.aws.amazon.com/ec2/. .. In the left navigation pane, choose *Launch Configurations*, and note the instance type for your existing node launch configuration. -.. In the left navigation pane, choose *Auto Scaling Groups*, and note the *Desired* instance count for your existing node Auto Scaling group. +.. In the left navigation pane, choose *Auto Scaling Groups*, and note the *Desired* instance count for your existing node Auto Scaling group. . Open the link:cloudformation/[{aws} CloudFormation console,type="console"]. . Select your node group stack, and then choose *Update*. -. Select *Replace current template* and select *Amazon S3 URL*. -. For *Amazon S3 URL*, paste the following URL into the text area to ensure that you're using the latest version of the node {aws} CloudFormation template. Then, choose *Next*: +. Select *Replace current template* and select *Amazon S3 URL*. +. For *Amazon S3 URL*, paste the following URL into the text area to ensure that you're using the latest version of the node {aws} CloudFormation template. Then, choose *Next*: + [source,none,subs="verbatim,attributes"] ---- https://s3.us-west-2.amazonaws.com/amazon-eks/cloudformation/2022-12-23/amazon-eks-nodegroup.yaml ---- -. On the *Specify stack details* page, fill out the following parameters, and choose *Next*: +. On the *Specify stack details* page, fill out the following parameters, and choose *Next*: + -** *NodeAutoScalingGroupDesiredCapacity* – Enter the desired instance count that you recorded in a <>. Or, enter your new desired number of nodes to scale to when your stack is updated. +** *NodeAutoScalingGroupDesiredCapacity* – Enter the desired instance count that you recorded in a <>. Or, enter your new desired number of nodes to scale to when your stack is updated. ** *NodeAutoScalingGroupMaxSize* – Enter the maximum number of nodes to which your node Auto Scaling group can scale out. This value must be at least one node more than your desired capacity. This is so that you can perform a rolling update of your nodes without reducing your node count during the update. -** *NodeInstanceType* – Choose the instance type your recorded in a <>. Alternatively, choose a different instance type for your nodes. Before choosing a different instance type, review <>. Each Amazon EC2 instance type supports a maximum number of elastic network interfaces (network interface) and each network interface supports a maximum number of IP addresses. Because each worker node and [.noloc]`Pod` ,is assigned its own IP address, it's important to choose an instance type that will support the maximum number of [.noloc]`Pods` that you want to run on each Amazon EC2 node. For a list of the number of network interfaces and IP addresses supported by instance types, see link:AWSEC2/latest/UserGuide/using-eni.html#AvailableIpPerENI[IP addresses per network interface per instance type,type="documentation"]. For example, the `m5.large` instance type supports a maximum of 30 IP addresses for the worker node and [.noloc]`Pods`. +** *NodeInstanceType* – Choose the instance type your recorded in a <>. Alternatively, choose a different instance type for your nodes. Before choosing a different instance type, review <>. Each Amazon EC2 instance type supports a maximum number of elastic network interfaces (network interface) and each network interface supports a maximum number of IP addresses. Because each worker node and Pod ,is assigned its own IP address, it's important to choose an instance type that will support the maximum number of Pods that you want to run on each Amazon EC2 node. For a list of the number of network interfaces and IP addresses supported by instance types, see link:AWSEC2/latest/UserGuide/using-eni.html#AvailableIpPerENI[IP addresses per network interface per instance type,type="documentation"]. For example, the `m5.large` instance type supports a maximum of 30 IP addresses for the worker node and Pods. + -NOTE: The supported instance types for the latest version of the https://github.com/aws/amazon-vpc-cni-k8s[Amazon VPC CNI plugin for Kubernetes] are shown in https://github.com/aws/amazon-vpc-cni-k8s/blob/master/pkg/vpc/vpc_ip_resource_limit.go[vpc_ip_resource_limit.go] on [.noloc]`GitHub`. You might need to update your [.noloc]`Amazon VPC CNI plugin for Kubernetes` version to use the latest supported instance types. For more information, see <>. +NOTE: The supported instance types for the latest version of the https://github.com/aws/amazon-vpc-cni-k8s[Amazon VPC CNI plugin for Kubernetes] are shown in https://github.com/aws/amazon-vpc-cni-k8s/blob/master/pkg/vpc/vpc_ip_resource_limit.go[vpc_ip_resource_limit.go] on GitHub. You might need to update your Amazon VPC CNI plugin for Kubernetes version to use the latest supported instance types. For more information, see <>. + IMPORTANT: Some instance types might not be available in all {aws} Regions. -** *NodeImageIdSSMParam* – The Amazon EC2 Systems Manager parameter of the AMI ID that you want to update to. The following value uses the latest Amazon EKS optimized AMI for [.noloc]`Kubernetes` version `1.30`. +** *NodeImageIdSSMParam* – The Amazon EC2 Systems Manager parameter of the AMI ID that you want to update to. The following value uses the latest Amazon EKS optimized AMI for Kubernetes version `{k8s-n}`. + [source,none,subs="verbatim,attributes"] ---- -/aws/service/eks/optimized-ami/1.30/amazon-linux-2/recommended/image_id +/aws/service/eks/optimized-ami/{k8s-n}/amazon-linux-2/recommended/image_id ---- + -You can replace [.replaceable]`1.30` with a <> that's the same. Or, it should be up to one version earlier than the [.noloc]`Kubernetes` version running on your control plane. We recommend that you keep your nodes at the same version as your control plane. You can also replace [.replaceable]`amazon-linux-2` with a different AMI type. For more information, see <>. +You can replace [.replaceable]`{k8s-n}` with a link:eks/latest/userguide/platform-versions.html[platform-version,type="documentation"] that's the same. Or, it should be up to one version earlier than the Kubernetes version running on your control plane. We recommend that you keep your nodes at the same version as your control plane. You can also replace [.replaceable]`amazon-linux-2` with a different AMI type. For more information, see <>. + -NOTE: Using the Amazon EC2 Systems Manager parameter enables you to update your nodes in the future without having to look up and specify an AMI ID. If your {aws} CloudFormation stack is using this value, any stack update always launches the latest recommended Amazon EKS optimized AMI for your specified [.noloc]`Kubernetes` version. This is even the case even if you don't change any values in the template. +NOTE: Using the Amazon EC2 Systems Manager parameter enables you to update your nodes in the future without having to look up and specify an AMI ID. If your {aws} CloudFormation stack is using this value, any stack update always launches the latest recommended Amazon EKS optimized AMI for your specified Kubernetes version. This is even the case even if you don't change any values in the template. ** *NodeImageId* – To use your own custom AMI, enter the ID for the AMI to use. + -IMPORTANT: This value overrides any value specified for *NodeImageIdSSMParam*. If you want to use the *NodeImageIdSSMParam* value, ensure that the value for *NodeImageId* is blank. -** *DisableIMDSv1* – By default, each node supports the Instance Metadata Service Version 1 (IMDSv1) and IMDSv2. However, you can disable IMDSv1. Select *true* if you don't want any nodes or any [.noloc]`Pods` scheduled in the node group to use IMDSv1. For more information about IMDS, see link:AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html[Configuring the instance metadata service,type="documentation"]. If you've implemented IAM roles for service accounts, assign necessary permissions directly to all [.noloc]`Pods` that require access to {aws} services. This way, no [.noloc]`Pods` in your cluster require access to IMDS for other reasons, such as retrieving the current {aws} Region. Then, you can also disable access to IMDSv2 for [.noloc]`Pods` that don't use host networking. For more information, see https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node[Restrict access to the instance profile assigned to the worker node]. -. (Optional) On the *Options* page, tag your stack resources. Choose *Next*. -. On the *Review* page, review your information, acknowledge that the stack might create IAM resources, and then choose *Update stack*. +IMPORTANT: This value overrides any value specified for *NodeImageIdSSMParam*. If you want to use the *NodeImageIdSSMParam* value, ensure that the value for *NodeImageId* is blank. +** *DisableIMDSv1* – By default, each node supports the Instance Metadata Service Version 1 (IMDSv1) and IMDSv2. However, you can disable IMDSv1. Select *true* if you don't want any nodes or any Pods scheduled in the node group to use IMDSv1. For more information about IMDS, see link:AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html[Configuring the instance metadata service,type="documentation"]. If you've implemented IAM roles for service accounts, assign necessary permissions directly to all Pods that require access to {aws} services. This way, no Pods in your cluster require access to IMDS for other reasons, such as retrieving the current {aws} Region. Then, you can also disable access to IMDSv2 for Pods that don't use host networking. For more information, see https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node[Restrict access to the instance profile assigned to the worker node]. +. (Optional) On the *Options* page, tag your stack resources. Choose *Next*. +. On the *Review* page, review your information, acknowledge that the stack might create IAM resources, and then choose *Update stack*. + NOTE: The update of each node in the cluster takes several minutes. Wait for the update of all nodes to complete before performing the next steps. . If your cluster's DNS provider is `kube-dns`, scale in the `kube-dns` deployment to one replica. @@ -100,10 +100,10 @@ NOTE: The update of each node in the cluster takes several minutes. Wait for the ---- kubectl scale deployments/kube-dns --replicas=1 -n kube-system ---- -. (Optional) If you are using the [.noloc]`Kubernetes` https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md[Cluster Autoscaler], scale the deployment back to your desired amount of replicas. +. (Optional) If you are using the Kubernetes https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md[Cluster Autoscaler], scale the deployment back to your desired amount of replicas. + [source,bash,subs="verbatim,attributes"] ---- kubectl scale deployments/cluster-autoscaler --replicas=1 -n kube-system ---- -. (Optional) Verify that you're using the latest version of the https://github.com/aws/amazon-vpc-cni-k8s[Amazon VPC CNI plugin for Kubernetes]. You might need to update your [.noloc]`Amazon VPC CNI plugin for Kubernetes` version to use the latest supported instance types. For more information, see <>. +. (Optional) Verify that you're using the latest version of the https://github.com/aws/amazon-vpc-cni-k8s[Amazon VPC CNI plugin for Kubernetes]. You might need to update your Amazon VPC CNI plugin for Kubernetes version to use the latest supported instance types. For more information, see <>. \ No newline at end of file diff --git a/latest/ug/nodes/update-workers.adoc b/latest/ug/nodes/update-workers.adoc index 2c7bff146..59532c86b 100644 --- a/latest/ug/nodes/update-workers.adoc +++ b/latest/ug/nodes/update-workers.adoc @@ -1,33 +1,32 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[update-workers,update-workers.title]] +[#update-workers] = Update self-managed nodes for your cluster :info_titleabbrev: Update methods -include::../attributes.txt[] - [abstract] -- When a new Amazon EKS optimized AMI is released, consider replacing the nodes in your self-managed node group with the new AMI. -- -When a new Amazon EKS optimized AMI is released, consider replacing the nodes in your self-managed node group with the new AMI. Likewise, if you have updated the [.noloc]`Kubernetes` version for your Amazon EKS cluster, update the nodes to use nodes with the same [.noloc]`Kubernetes` version. +When a new Amazon EKS optimized AMI is released, consider replacing the nodes in your self-managed node group with the new AMI. Likewise, if you have updated the Kubernetes version for your Amazon EKS cluster, update the nodes to use nodes with the same Kubernetes version. [IMPORTANT] ==== -This topic covers node updates for self-managed nodes. If you are using <>, see <>. +This topic covers node updates for self-managed nodes. If you are using <>, see <>. ==== There are two basic ways to update self-managed node groups in your clusters to use a new AMI: *<>*:: -Create a new node group and migrate your [.noloc]`Pods` to that group. Migrating to a new node group is more graceful than simply updating the AMI ID in an existing {aws} CloudFormation stack. This is because the migration process https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/[taints] the old node group as `NoSchedule` and drains the nodes after a new stack is ready to accept the existing [.noloc]`Pod` workload. +Create a new node group and migrate your Pods to that group. Migrating to a new node group is more graceful than simply updating the AMI ID in an existing {aws} CloudFormation stack. This is because the migration process https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/[taints] the old node group as `NoSchedule` and drains the nodes after a new stack is ready to accept the existing Pod workload. *<>*:: Update the {aws} CloudFormation stack for an existing node group to use the new AMI. This method isn't supported for node groups that were created with `eksctl`. include::migrate-stack.adoc[leveloffset=+1] -include::update-stack.adoc[leveloffset=+1] +include::update-stack.adoc[leveloffset=+1] \ No newline at end of file diff --git a/latest/ug/nodes/worker.adoc b/latest/ug/nodes/worker.adoc index 747613458..df2e29237 100644 --- a/latest/ug/nodes/worker.adoc +++ b/latest/ug/nodes/worker.adoc @@ -1,39 +1,50 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[worker,worker.title]] +[#worker] = Maintain nodes yourself with self-managed nodes -:info_doctype: section -:info_title: Maintain nodes yourself with self-managed nodes :info_titleabbrev: Self-managed nodes -:keywords: self-managed, node -:info_abstract: A cluster contains one or more Amazon EC2 nodes that Pods are scheduled on. - -include::../attributes.txt[] - -include::launch-workers.adoc[leveloffset=+1] - -include::launch-node-bottlerocket.adoc[leveloffset=+1] - -include::launch-windows-workers.adoc[leveloffset=+1] - -include::launch-node-ubuntu.adoc[leveloffset=+1] - -include::update-workers.adoc[leveloffset=+1] [abstract] -- -A cluster contains one or more Amazon EC2 nodes that [.noloc]`Pods` are scheduled on. +A cluster contains one or more Amazon EC2 nodes that Pods are scheduled on. -- -A cluster contains one or more Amazon EC2 nodes that [.noloc]`Pods` are scheduled on. Amazon EKS nodes run in your {aws} account and connect to the control plane of your cluster through the cluster API server endpoint. You're billed for them based on Amazon EC2 prices. For more information, see link:ec2/pricing/[Amazon EC2 pricing,type="marketing"]. +A cluster contains one or more Amazon EC2 nodes that Pods are scheduled on. Amazon EKS nodes run in your {aws} account and connect to the control plane of your cluster through the cluster API server endpoint. You're billed for them based on Amazon EC2 prices. For more information, see link:ec2/pricing/[Amazon EC2 pricing,type="marketing"]. A cluster can contain several node groups. Each node group contains one or more nodes that are deployed in an link:autoscaling/ec2/userguide/AutoScalingGroup.html[Amazon EC2 Auto Scaling group,type="documentation"]. The instance type of the nodes within the group can vary, such as when using link:AWSEC2/latest/UserGuide/ec2-fleet-attribute-based-instance-type-selection.html[attribute-based instance type selection,type="documentation"] with https://karpenter.sh/[Karpenter]. All instances in a node group must use the <>. -Amazon EKS provides specialized Amazon Machine Images (AMIs) that are called Amazon EKS optimized AMIs. The AMIs are configured to work with Amazon EKS. Their components include `containerd`, `kubelet`, and the {aws} IAM Authenticator. The AMIs also contain a specialized https://github.com/awslabs/amazon-eks-ami/blob/main/templates/al2/runtime/bootstrap.sh[bootstrap script] that allows it to discover and connect to your cluster's control plane automatically. +Amazon EKS provides specialized Amazon Machine Images (AMIs) that are called Amazon EKS optimized AMIs. The AMIs are configured to work with Amazon EKS. Their components include `containerd`, `kubelet`, and the {aws} IAM Authenticator. The AMIs use https://awslabs.github.io/amazon-eks-ami/nodeadm[nodeadm], a tool that automatically connects nodes to your cluster's control plane and configures `kubelet` and `containerd`. If you restrict access to the public endpoint of your cluster using CIDR blocks, we recommend that you also enable private endpoint access. This is so that nodes can communicate with the cluster. Without the private endpoint enabled, the CIDR blocks that you specify for public access must include the egress sources from your VPC. For more information, see <>. -To add self-managed nodes to your Amazon EKS cluster, see the topics that follow. If you launch self-managed nodes manually, add the following tag to each node. For more information, see link:AWSEC2/latest/UserGuide/Using_Tags.html#adding-or-deleting-tags[Adding and deleting tags on an individual resource,type="documentation"]. If you follow the steps in the guides that follow, the required tag is automatically added to nodes for you. +To add self-managed nodes to your Amazon EKS cluster, see the topics that follow. If you launch self-managed nodes manually, add the following tag to each node while making sure that `` matches your cluster. For more information, see link:AWSEC2/latest/UserGuide/Using_Tags.html#adding-or-deleting-tags[Adding and deleting tags on an individual resource,type="documentation"]. If you follow the steps in the guides that follow, the required tag is automatically added to nodes for you. + +[%header,cols="2"] +|=== +|Key|Value + +|`kubernetes.io/cluster/`|`owned` +|=== + +[IMPORTANT] +==== +Tags in Amazon EC2 Instance Metadata Service (IMDS) are not compatible with EKS nodes. When Instance Metadata Tags are enabled, the use of forward slashes ('/') in tag values is prevented. This limitation can cause instance launch failures, particularly when using node management tools like Karpenter or Cluster Autoscaler, as these services rely on tags containing forward slashes for proper functionality. +==== + + +For more information about nodes from a general Kubernetes perspective, see https://kubernetes.io/docs/concepts/architecture/nodes/[Nodes] in the Kubernetes documentation. [.topiclist] -[[Topic List]] \ No newline at end of file +[[Topic List]] + +include::launch-workers.adoc[leveloffset=+1] + +include::launch-node-bottlerocket.adoc[leveloffset=+1] + +include::launch-windows-workers.adoc[leveloffset=+1] + +include::launch-node-ubuntu.adoc[leveloffset=+1] + +include::update-workers.adoc[leveloffset=+1] + diff --git a/latest/ug/observability/cloudwatch.adoc b/latest/ug/observability/cloudwatch.adoc index f378cbaa9..990d2cdef 100644 --- a/latest/ug/observability/cloudwatch.adoc +++ b/latest/ug/observability/cloudwatch.adoc @@ -1,12 +1,9 @@ -//!!NODE_ROOT
+include::../attributes.txt[] [.topic] -[[cloudwatch,cloudwatch.title]] +[#cloudwatch] = Monitor cluster data with Amazon CloudWatch :info_titleabbrev: Amazon CloudWatch -:keywords: CloudWatch, observability, operator, add-on - -include::../attributes.txt[] [abstract] -- @@ -15,137 +12,256 @@ With Amazon CloudWatch, you can view metrics, real-time logs, and trace data. Amazon CloudWatch is a monitoring service that collects metrics and logs from your cloud resources. CloudWatch provides some basic Amazon EKS metrics for free when using a new cluster that is version `1.28` and above. However, when using the CloudWatch Observability Operator as an Amazon EKS add-on, you can gain enhanced observability features. -[[cloudwatch-basic-metrics,cloudwatch-basic-metrics.title]] +[#cloudwatch-basic-metrics] == Basic metrics in Amazon CloudWatch -For new clusters that are [.noloc]`Kubernetes` version `1.28` and above, you get CloudWatch vended metrics for free in the `AWS/EKS` namespace. Basic metrics are also available for existing clusters that have a platform version that is the same or later compared to the following table. +For clusters that are Kubernetes version `1.28` and above, you get CloudWatch vended metrics for free in the `AWS/EKS` namespace. The following table gives a list of the basic metrics that are available for the supported versions. Every metric listed has a frequency of one minute. + +// Match format of SQS and SNS documentation: +// * https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-available-cloudwatch-metrics.html +// * https://docs.aws.amazon.com/sns/latest/dg/sns-monitoring-using-cloudwatch.html -[cols="1,1", options="header"] +[%header,cols="2"] |=== -|Kubernetes version -|Platform version -|`1.31` -|`eks.12` +|Metric name +|Description -|`1.30` -|`eks.20` +|`scheduler_schedule_attempts_total` +|The number of total attempts by the scheduler to schedule Pods in the cluster for a given period. This metric helps monitor the scheduler's workload and can indicate scheduling pressure or potential issues with Pod placement. -|`1.29` -|`eks.23` +*Units:* Count -|`1.28` -|`eks.29` +*Valid statistics:* Sum -|=== +|`scheduler_schedule_attempts_SCHEDULED` +|The number of successful attempts by the scheduler to schedule Pods to nodes in the cluster for a given period. -The following table gives a list of the basic metrics that are available for the supported versions. Every metric listed has a frequency of one minute. +*Units:* Count -[cols="1,1,1,1,1,1", options="header"] -|=== +*Valid statistics:* Sum -|Metric name -|Description -|Unit -|Metric dimension -|Metric type -|Source [.noloc]`Kubernetes` metric - -|`APIServerRequests` -|The number of times requests were made to the API server. -|Count -|Cluster Name -|Traffic -|`kube-apiserver :: apiserver_request_total` - -|`APIServerRequestsHTTP4XX` -|The number of API Server requests that had an HTTP 4XX error response (client-side error). -|Count -|Cluster Name -|Error -|`kube-apiserver :: apiserver_request_total` - -|`APIServerRequestsHTTP429` -|The number of API Server requests that had an HTTP 429 error response (too many requests). -|Count -|Cluster Name -|Error -|`kube-apiserver :: apiserver_request_total` - -|`APIServerRequestsHTTP5XX` -|The number of API Server requests that had an HTTP 5XX error response (server-side error). -|Count -|Cluster Name -|Error -|`kube-apiserver :: apiserver_request_total` - -|`APIServerRequestLatency` -|The average amount of seconds taken by `APIServer` to respond to requests. -|Seconds -|Cluster Name, Verb -|Latency -|`kube-apiserver :: apiserver_request_duration_seconds` - -|`APIServerCurrentInflightRequests` -|The number of requests that are being actively served. -|Count -|Cluster Name, Request Kind {mutating, readOnly} -|Saturation -|`kube-apiserver :: apiserver_current_inflight_requests` - -|`APIServerStorageSize` -|The size of the storage database. -|Bytes -|Cluster Name -|Saturation -|`kube-apiserver :: apiserver_storage_size_bytes` - -|`SchedulerAttempts` -|The number of attempts to schedule Pods. -|Count -|Cluster Name, Result {unschedulable, error, scheduled} -|Latency -|`kube-scheduler :: scheduler_schedule_attempts_total` - -|`PendingPods` -|The number of Pods that are pending to be scheduled. -|Count -|Cluster Name, Queue {activeQ unschedulable, backoff, gated} -|Latency -|`kube-scheduler :: scheduler_pending_pods` - -|`APIServerWebhookRequests` -|The number of admission webhook requests made. -|Count -|Cluster Name, Admission Type (validating, admit) -|Traffic -|`kube-apiserver :: apiserver_admission_webhook_request_total` - -|`APIServerWebhookRejections` -|The number of admission webhook rejections. -|Count -|Cluster Name, Admission Type (validating, admit) -|Error -|`kube-apiserver :: apiserver_admission_webhook_rejection_count` - -|`APIServerWebhookLatencyP99` -|The 99th percentile latency of external, third-party admission webhooks. -|Seconds -|Cluster Name, Admission Type (validating, admit) -|Latency -|`kube-apiserver :: apiserver_admission_webhook_admission_duration_seconds` +|`scheduler_schedule_attempts_UNSCHEDULABLE` +|The number of attempts to schedule Pods that were unschedulable for a given period due to valid constraints, such as insufficient CPU or memory on a node. + +*Units:* Count + +*Valid statistics:* Sum + +|`scheduler_schedule_attempts_ERROR` +|The number of attempts to schedule Pods that failed for a given period due to an internal problem with the scheduler itself, such as API Server connectivity issues. + +*Units:* Count + +*Valid statistics:* Sum + +|`scheduler_pending_pods` +|The number of total pending Pods to be scheduled by the scheduler in the cluster for a given period. + +*Units:* Count + +*Valid statistics:* Sum + +|`scheduler_pending_pods_ACTIVEQ` +|The number of pending Pods in activeQ, that are waiting to be scheduled in the cluster for a given period. + +*Units:* Count + +*Valid statistics:* Sum + +|`scheduler_pending_pods_UNSCHEDULABLE` +|The number of pending Pods that the scheduler attempted to schedule and failed, and are kept in an unschedulable state for retry. + +*Units:* Count + +*Valid statistics:* Sum + +|`scheduler_pending_pods_BACKOFF` +|The number of pending Pods in `backoffQ` in a backoff state that are waiting for their backoff period to expire. + +*Units:* Count + +*Valid statistics:* Sum + +|`scheduler_pending_pods_GATED` +|The number of pending Pods that are currently waiting in a gated state as they cannot be scheduled until they meet required conditions. + +*Units:* Count + +*Valid statistics:* Sum + +|`apiserver_request_total` +|The number of HTTP requests made across all the API servers in the cluster. + +*Units:* Count + +*Valid statistics:* Sum + +|`apiserver_request_total_4XX` +|The number of HTTP requests made to all the API servers in the cluster that resulted in `4XX` (client error) status codes. + +*Units:* Count + +*Valid statistics:* Sum + +|`apiserver_request_total_429` +|The number of HTTP requests made to all the API servers in the cluster that resulted in `429` status code, which occurs when clients exceed the rate limiting thresholds. + +*Units:* Count + +*Valid statistics:* Sum + +|`apiserver_request_total_5XX` +|The number of HTTP requests made to all the API servers in the cluster that resulted in `5XX` (server error) status codes. + +*Units:* Count + +*Valid statistics:* Sum + +|`apiserver_request_total_LIST_PODS` +|The number of `LIST` Pods requests made to all the API servers in the cluster. + +*Units:* Count + +*Valid statistics:* Sum + +|`apiserver_request_duration_seconds_PUT_P99` +|The 99th percentile of latency for `PUT` requests calculated from all requests across all API servers in the cluster. Represents the response time below which 99% of all `PUT` requests are completed. + +*Units:* Seconds + +*Valid statistics:* Average + +|`apiserver_request_duration_seconds_PATCH_P99` +|The 99th percentile of latency for `PATCH` requests calculated from all requests across all API servers in the cluster. Represents the response time below which 99% of all `PATCH` requests are completed. + +*Units:* Seconds + +*Valid statistics:* Average + +|`apiserver_request_duration_seconds_POST_P99` +|The 99th percentile of latency for `POST` requests calculated from all requests across all API servers in the cluster. Represents the response time below which 99% of all `POST` requests are completed. + +*Units:* Seconds + +*Valid statistics:* Average + +|`apiserver_request_duration_seconds_GET_P99` +|The 99th percentile of latency for `GET` requests calculated from all requests across all API servers in the cluster. Represents the response time below which 99% of all `GET` requests are completed. + +*Units:* Seconds + +*Valid statistics:* Average + +|`apiserver_request_duration_seconds_LIST_P99` +|The 99th percentile of latency for `LIST` requests calculated from all requests across all API servers in the cluster. Represents the response time below which 99% of all `LIST` requests are completed. + +*Units:* Seconds + +*Valid statistics:* Average + +|`apiserver_request_duration_seconds_DELETE_P99` +|The 99th percentile of latency for `DELETE` requests calculated from all requests across all API servers in the cluster. Represents the response time below which 99% of all `DELETE` requests are completed. + +*Units:* Seconds + +*Valid statistics:* Average + +|`apiserver_current_inflight_requests_MUTATING` +|The number of mutating requests (`POST`, `PUT`, `DELETE`, `PATCH`) currently being processed across all API servers in the cluster. This metric represents requests that are in-flight and haven't completed processing yet. + +*Units:* Count + +*Valid statistics:* Sum + +|`apiserver_current_inflight_requests_READONLY` +|The number of read-only requests (`GET`, `LIST`) currently being processed across all API servers in the cluster. This metric represents requests that are in-flight and haven't completed processing yet. + +*Units:* Count + +*Valid statistics:* Sum + +|`apiserver_admission_webhook_request_total` +|The number of admission webhook requests made across all API servers in the cluster. + +*Units:* Count + +*Valid statistics:* Sum + +|`apiserver_admission_webhook_request_total_ADMIT` +|The number of mutating admission webhook requests made across all API servers in the cluster. + +*Units:* Count + +*Valid statistics:* Sum + +|`apiserver_admission_webhook_request_total_VALIDATING` +|The number of validating admission webhook requests made across all API servers in the cluster. + +*Units:* Count + +*Valid statistics:* Sum + +|`apiserver_admission_webhook_rejection_count` +|The number of admission webhook requests made across all API servers in the cluster that were rejected. + +*Units:* Count + +*Valid statistics:* Sum + +|`apiserver_admission_webhook_rejection_count_ADMIT` +|The number of mutating admission webhook requests made across all API servers in the cluster that were rejected. + +*Units:* Count + +*Valid statistics:* Sum + +|`apiserver_admission_webhook_rejection_count_VALIDATING` +|The number of validating admission webhook requests made across all API servers in the cluster that were rejected. + +*Units:* Count + +*Valid statistics:* Sum + +|`apiserver_admission_webhook_admission_duration_seconds` +|The 99th percentile of latency for third-party admission webhook requests calculated from all requests across all API servers in the cluster. Represents the response time below which 99% of all third-party admission webhook requests are completed. + +*Units:* Seconds + +*Valid statistics:* Average + +|`apiserver_admission_webhook_admission_duration_seconds_ADMIT_P99` +|The 99th percentile of latency for third-party mutating admission webhook requests calculated from all requests across all API servers in the cluster. Represents the response time below which 99% of all third-party mutating admission webhook requests are completed. + +*Units:* Seconds + +*Valid statistics:* Average + +|`apiserver_admission_webhook_admission_duration_seconds_VALIDATING_P99` +|The 99th percentile of latency for third-party validating admission webhook requests calculated from all requests across all API servers in the cluster. Represents the response time below which 99% of all third-party validating admission webhook requests are completed. + +*Units:* Seconds + +*Valid statistics:* Average + +|`apiserver_storage_size_bytes` +|The physical size in bytes of the etcd storage database file used by the API servers in the cluster. This metric represents the actual disk space allocated for the storage. + +*Units:* Bytes + +*Valid statistics:* Maximum |=== -[[cloudwatch-operator,cloudwatch-operator.title]] -== [.noloc]`Amazon CloudWatch Observability Operator` +[#cloudwatch-operator] +== Amazon CloudWatch Observability Operator -[.noloc]`Amazon CloudWatch Observability` collects real-time logs, metrics, and trace data. It sends them to link:AmazonCloudWatch/latest/monitoring/WhatIsCloudWatch.html[Amazon CloudWatch,type="documentation"] and link:xray/latest/devguide/aws-xray.html[{aws} X-Ray,type="documentation"]. You can install this add-on to enable both CloudWatch Application Signals and CloudWatch [.noloc]`Container Insights` with enhanced observability for Amazon EKS. This helps you monitor the health and performance of your infrastructure and containerized applications. The Amazon CloudWatch Observability Operator is designed to install and configure the necessary components. +Amazon CloudWatch Observability collects real-time logs, metrics, and trace data. It sends them to link:AmazonCloudWatch/latest/monitoring/WhatIsCloudWatch.html[Amazon CloudWatch,type="documentation"] and link:xray/latest/devguide/aws-xray.html[{aws} X-Ray,type="documentation"]. You can install this add-on to enable both CloudWatch Application Signals and CloudWatch Container Insights with enhanced observability for Amazon EKS. This helps you monitor the health and performance of your infrastructure and containerized applications. The Amazon CloudWatch Observability Operator is designed to install and configure the necessary components. -Amazon EKS supports the CloudWatch Observability Operator as an <>. The add-on allows [.noloc]`Container Insights` on both [.noloc]`Linux` and [.noloc]`Windows` worker nodes in the cluster. To enable [.noloc]`Container Insights` on [.noloc]`Windows`, the Amazon EKS add-on version must be `1.5.0` or higher. Currently, CloudWatch Application Signals isn't supported on Amazon EKS [.noloc]`Windows`. +Amazon EKS supports the CloudWatch Observability Operator as an <>. The add-on allows Container Insights on both Linux and Windows worker nodes in the cluster. To enable Container Insights on Windows, the Amazon EKS add-on version must be `1.5.0` or higher. Currently, CloudWatch Application Signals isn't supported on Amazon EKS Windows. The topics below describe how to get started using CloudWatch Observability Operator for your Amazon EKS cluster. * For instructions on installing this add-on, see link:AmazonCloudWatch/latest/monitoring/install-CloudWatch-Observability-EKS-addon.html[Install the CloudWatch agent with the Amazon CloudWatch Observability EKS add-on or the Helm chart,type="documentation"] in the _Amazon CloudWatch User Guide_. * For more information about CloudWatch Application Signals, see link:AmazonCloudWatch/latest/monitoring/CloudWatch-Application-Monitoring-Sections.html[Application Signals,type="documentation"] in the _Amazon CloudWatch User Guide_. -* For more information about [.noloc]`Container Insights`, see link:AmazonCloudWatch/latest/monitoring/ContainerInsights.html[Using Container Insights,type="documentation"] in the _Amazon CloudWatch User Guide_. +* For more information about Container Insights, see link:AmazonCloudWatch/latest/monitoring/ContainerInsights.html[Using Container Insights,type="documentation"] in the _Amazon CloudWatch User Guide_. \ No newline at end of file diff --git a/latest/ug/observability/cluster-dashboard-orgs.adoc b/latest/ug/observability/cluster-dashboard-orgs.adoc new file mode 100644 index 000000000..ba629fed4 --- /dev/null +++ b/latest/ug/observability/cluster-dashboard-orgs.adoc @@ -0,0 +1,138 @@ +include::../attributes.txt[] + +[.topic] +[#cluster-dashboard-orgs] += Configure EKS Dashboard integration with {aws} Organizations +:info_titleabbrev: Configure organizations + +This section provides step-by-step instructions for configuring the EKS Dashboard's integration with {aws} Organizations. You'll learn how to enable and disable trusted access between services, as well as how to register and deregister delegated administrator accounts. Each configuration task can be performed using either the {aws} console or the {aws} CLI. + +== Enable trusted access + +Trusted access authorizes the EKS Dashboard to securely access cluster information across all accounts in your organization. + +=== Using the {aws} console + +. Log in to the management account of your {aws} Organization. +. Navigate to the EKS console in the us-east-1 region. +. In the left sidebar, select Dashboard Settings. +. Click *Enable trusted access*. + +[NOTE] +==== +When you enable trusted access through the EKS console, the system automatically creates the `AWSServiceRoleForAmazonEKSDashboard` service-linked role. This automatic creation does not occur if you enable trusted access using the {aws} CLI or {aws} Organizations console. +==== + +[#dashboard-enable-cli] +=== Using the {aws} CLI + +. Log in to the management account of your {aws} Organization. +. Run the following commands: ++ +[source,bash] +---- +aws iam create-service-linked-role --aws-service-name dashboard.eks.amazonaws.com +aws organizations enable-aws-service-access --service-principal eks.amazonaws.com +---- + +== Disable trusted access + +Disabling trusted access revokes the EKS Dashboard's permission to access cluster information across your organization's accounts. + +=== Using the {aws} console + +. Log in to the management account of your {aws} Organization. +. Navigate to the EKS Console in the us-east-1 region. +. In the left sidebar, select Dashboard Settings. +. Click *Disable trusted access*. + +=== Using the {aws} CLI + +. Log in to the management account of your {aws} Organization. +. Run the following command: ++ +[source,bash] +---- +aws organizations disable-aws-service-access --service-principal eks.amazonaws.com +---- + +== Enable a delegated administrator account + +A delegated administrator is a member account that's granted permission to access the EKS Dashboard. + +=== Using the {aws} console + +. Log in to the management account of your {aws} Organization. +. Navigate to the EKS console in the us-east-1 region. +. In the left sidebar, select Dashboard Settings. +. Click *Register delegated administrator*. +. Enter the Account ID of the {aws} Account you want to choose as delegated administrator. +. Confirm the registration. + +=== Using the {aws} CLI + +. Log in to the management account of your {aws} Organization. +. Run the following command, replacing `123456789012` with your account ID: ++ +[source,bash] +---- +aws organizations register-delegated-administrator --account-id 123456789012 --service-principal eks.amazonaws.com +---- + +== Disable a delegated administrator account + +Disabling a delegated administrator removes the account's permission to access the EKS Dashboard. + +=== Using the {aws} console + +. Log in to the management account of your {aws} Organization. +. Navigate to the EKS console in the us-east-1 region. +. In the left sidebar, select Dashboard Settings. +. Locate the delegated administrator in the list. +. Click *Deregister* next to the account you want to remove as delegated administrator. + +=== Using the {aws} CLI + +. Log in to the management account of your {aws} Organization. +. Run the following command, replacing `123456789012` with the account ID of the delegated administrator: ++ +[source,bash] +---- +aws organizations deregister-delegated-administrator --account-id 123456789012 --service-principal eks.amazonaws.com +---- + +[#dashboard-iam-policy] +== Minimum IAM policies required + +This section outlines the minimum IAM policies required to enable trusted access and delegate an administrator for the EKS Dashboard integration with {aws} Organizations. + +=== Policy for enabling trusted access + +To enable trusted access between EKS Dashboard and {aws} Organizations, you need the following permissions: + +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:c0b2a459-9ffe-4fd9-a859-163d9df9fec5[] +---- + +=== Policy for delegating an administrator + +To register or deregister a delegated administrator for the EKS Dashboard, you need the following permissions: + +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:e7615daa-c27a-408d-932e-36d575ae56b3[] +---- + +[#eks-dashboard-view-policy] +=== Policy to view EKS Dashboard + +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:f6e3be86-d7b9-4e46-ba5c-4e6df6ef9cfa[] +---- + +[NOTE] +==== +These policies must be attached to the IAM principal (user or role) in the management account of your {aws} Organization. Member accounts cannot enable trusted access or delegate administrators. +==== diff --git a/latest/ug/observability/cluster-dashboard.adoc b/latest/ug/observability/cluster-dashboard.adoc new file mode 100644 index 000000000..31b03eb77 --- /dev/null +++ b/latest/ug/observability/cluster-dashboard.adoc @@ -0,0 +1,161 @@ +include::../attributes.txt[] + +[.topic] +[#cluster-dashboard] += View aggregated data about cluster resources with the EKS Dashboard +:info_titleabbrev: Amazon EKS Dashboard + +include::cluster-dashboard-orgs.adoc[leveloffset=+1] + +image::images/eks-dashboard.png[screenshot of account level cluster metrics] + +== What is the Amazon EKS Dashboard? + +The Amazon EKS Dashboard provides consolidated visibility into your Kubernetes clusters across multiple {aws} Regions and {aws} Accounts. With this dashboard, you can: + +* Track clusters scheduled for end-of-support auto-upgrades within the next 90 days. +* Project EKS control plane costs for clusters in extended support. +* Review clusters with insights that need attention before upgrading. +* Identify managed node groups running specific AMI versions. +* Monitor cluster support type distribution (standard compared to extended support). + +The EKS dashboard integrates with EKS Cluster Insights to surface issues with your clusters, such as use of deprecated Kubernetes APIs. For more information, see <>. + +[NOTE] +==== +The EKS Dashboard is not real-time and updates every 12 hours. For real-time cluster monitoring, see <> +==== + +== How does the dashboard use {aws} Organizations? + +The Amazon EKS dashboard requires {aws} Organizations integration for functionality. It leverages {aws} Organizations to securely gather cluster information across accounts. This integration provides centralized management and governance as your {aws} infrastructure scales. + +If {aws} Organizations isn't enabled for your infrastructure, see the {aws}link:organizations/latest/userguide/orgs_introduction.html[Organizations User Guide, type="documentation"] for setup instructions. + +=== Cross-region and cross-account access + +The EKS Dashboard can see cluster resources in any account that is a member of the {aws} organization. To generate a list of {aws} accounts in your organization, see link:organizations/latest/userguide/orgs_manage_accounts_export.html[Export details for all accounts in an organization,type="documentation"]. + +The us-east-1 {aws} region generates the dashboard. You must log in to this region to see the dashboard. The dashboard aggregates data across {aws} regions, but this does not include `GovCloud` or China regions. + +=== Key terms + +* *{aws} Organization*: A unified management structure for multiple {aws} accounts. +* *Management account*: The primary account that controls the {aws} Organization. +* *Member account*: Any account within the organization except the management account. +* *Delegated administrator*: A member account granted specific cross-account administrative permissions. Within the management account, you can select one delegated administrator account per {aws} Service. +* *Trusted access*: Authorization for the EKS Dashboard to access cluster information across organizational accounts. +* *Service-Linked Role (SLRs)*: A unique type of IAM role directly linked to an {aws} service. The EKS Dashboard uses a SLR to read information about your accounts and organizations. +* For more information, see link:organizations/latest/userguide/orgs_getting-started_concepts.html[Terminology and concepts, type="documentation"] in the {aws} Organizations User Guide. + +=== General overview + +. Access the management account of your {aws} Organization. +** The steps to access the management account depend on how you have configured your {aws} Organization. For example, you might access the management account via {aws} link:iam/identity-center/[Identity Center,type="marketing"] or link:https://www.okta.com/partners/aws/[Okta]. +. Enable Trusted access through the EKS Console. +. Assign a Delegated administrator using their {aws} Account ID. +. Switch to the Delegated administrator account. +. Access the enhanced EKS Console with organization-wide visibility. + +== Enable the EKS Dashboard using the {aws} console + +[IMPORTANT] +==== +You must be logged in to the Management Account of your {aws} Organization to enable the EKS Dashboard. +==== + +=== Access EKS Dashboard settings + +. Confirm the following: +.. You have {aws} Organizations enabled and configured. +.. You are logged into the Management account of the organization. +.. You are viewing the {aws} Management Console in the us-east-1 region. +. Navigate to the EKS console. +. In the left sidebar, open Dashboard Settings. + +=== Set up access to the Amazon EKS Dashboard + +. Find the {aws} Account ID of the {aws} Account you want to allow to view the EKS Dashboard. +.. This step is optional, but suggested. If you don't, you can only access the dashboard from the management account. As a best practice, you should limit access to the management account. +. Click *Enable trusted access*. +.. You can now view the dashboard from the management account. +. Click *Register delegated administrator* and input the Account ID of the {aws} Account you will use to view the dashboard. +.. You can now view the dashboard from the delegated administrator account or the management account. + +For information about permissions required to enable the dashboard, see xref:dashboard-iam-policy[Minimum IAM policies required]. + +== View the EKS dashboard + +. Log in to the delegated administrator account (suggested) or the management account. +. Log in to the us-east-1 region. +. Go to the EKS service, and select Dashboard from the left sidebar. + +NOTE: xref:eks-dashboard-view-policy[Review the IAM permissions required to view the EKS dashboard.] + +== Configure the dashboard + +You can configure the view of the dashboard, and filter resources. + +=== Available resources + +* *Clusters*: View aggregated information about the status and location of EKS Clusters. +** Clusters with health issues. +** Clusters on EKS Extended Support. +** Breakdown of clusters by Kubernetes version. +* *Managed node groups*: Review Managed node groups and EC2 Instances. +** Node groups by AMI type, such as Amazon Linux or Bottlerocket. +** Node group health issues. +** Instance type distribution. +* *Add-ons*: Learn about what Amazon EKS Add-ons you have installed, and their status. +** Number of installations per add-on. +** Add-ons with health issues. +** Version distribution per add-on. + +=== Available views + +* *Graph view* +** A customizable widget view displaying graphs and visualizations of the selected resource. +** Changes to the Graph view, such as removing a widget, are visible to all users of the EKS Dashboard. +* *Resource view* +** A list view of the selected resource, supporting filters. +* *Map View* +** View the geographic distribution of the selected resource. + +=== Filter the EKS dashboard + +You can filter the EKS Dashboard by: + +* {aws} Account +* Organizational unit, defined by {aws} Organizations +* {aws} Region + +== Disable the EKS dashboard using the {aws} console + +. Confirm the following: +.. You have {aws} Organizations enabled and configured. +.. You are logged into the Management account of the organization. +.. You are viewing the {aws} Management Console in the us-east-1 region. +. Navigate to the EKS console. +. In the left sidebar, open Dashboard Settings. +. Click *Disable trusted access*. + +== Troubleshoot the EKS dashboard + +=== Issue enabling EKS dashboard + +* You must be logged in to the management account of an {aws} Organization. +** If you do not have an {aws} Organization, create one. Learn how to link:organizations/latest/userguide/orgs_tutorials_basic.html[Create and configure an organization,type="documentation"]. +** If your {aws} account is already a member of an {aws} Organization, identify the administrator of the organization. +* You must be logged in to the {aws} account with sufficient IAM permissions to create and update {aws} Organizations resources. + +=== Issue viewing the EKS dashboard + +* You must be logged in to one of the following {aws} accounts: +** The management account of the {aws} Organization +** A delegated administrator account, identified in the EKS dashboard settings of the management account. +* If you've recently enabled the EKS Dashboard, please note that initial data population may take up to 12 hours. +* xref:dashboard-enable-cli[Try re-enabling the dashboard using the CLI], including creating the service linked role. + +=== Dashboard widgets move unexpectedly + +* The EKS Dashboard saves the configurable widget view at the {aws} Account level. If you change the widget view, other people using the same {aws} account will see the changes. diff --git a/latest/ug/observability/control-plane-logs.adoc b/latest/ug/observability/control-plane-logs.adoc index 45651d68d..e9b925eaa 100644 --- a/latest/ug/observability/control-plane-logs.adoc +++ b/latest/ug/observability/control-plane-logs.adoc @@ -1,14 +1,9 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[control-plane-logs,control-plane-logs.title]] +[#control-plane-logs] = Send control plane logs to CloudWatch Logs -:info_doctype: section -:info_title: Send control plane logs to CloudWatch Logs :info_titleabbrev: Control plane logs -:keywords: control plane, logging, API, logs -:info_abstract: Learn how to configure logging for your Amazon EKS cluster. [abstract] -- @@ -21,31 +16,31 @@ You can start using Amazon EKS control plane logging by choosing which log types When you use Amazon EKS control plane logging, you're charged standard Amazon EKS pricing for each cluster that you run. You are charged the standard CloudWatch Logs data ingestion and storage costs for any logs sent to CloudWatch Logs from your clusters. You are also charged for any {aws} resources, such as Amazon EC2 instances or Amazon EBS volumes, that you provision as part of your cluster. -The following cluster control plane log types are available. Each log type corresponds to a component of the [.noloc]`Kubernetes` control plane. To learn more about these components, see https://kubernetes.io/docs/concepts/overview/components/[Kubernetes Components] in the [.noloc]`Kubernetes` documentation. +The following cluster control plane log types are available. Each log type corresponds to a component of the Kubernetes control plane. To learn more about these components, see https://kubernetes.io/docs/concepts/overview/components/[Kubernetes Components] in the Kubernetes documentation. *API server (`api`)*:: -Your cluster's API server is the control plane component that exposes the [.noloc]`Kubernetes` API. If you enable API server logs when you launch the cluster, or shortly thereafter, the logs include API server flags that were used to start the API server. For more information, see https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/[kube-apiserver] and the https://github.com/kubernetes/kubernetes/blob/master/cluster/gce/gci/configure-helper.sh#L1129-L1255[audit policy] in the [.noloc]`Kubernetes` documentation. +Your cluster's API server is the control plane component that exposes the Kubernetes API. If you enable API server logs when you launch the cluster, or shortly thereafter, the logs include API server flags that were used to start the API server. For more information, see https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/[kube-apiserver] and the https://github.com/kubernetes/kubernetes/blob/master/cluster/gce/gci/configure-helper.sh#L1129-L1255[audit policy] in the Kubernetes documentation. *Audit (`audit`)*:: -[.noloc]`Kubernetes` audit logs provide a record of the individual users, administrators, or system components that have affected your cluster. For more information, see https://kubernetes.io/docs/tasks/debug-application-cluster/audit/[Auditing] in the [.noloc]`Kubernetes` documentation. +Kubernetes audit logs provide a record of the individual users, administrators, or system components that have affected your cluster. For more information, see https://kubernetes.io/docs/tasks/debug-application-cluster/audit/[Auditing] in the Kubernetes documentation. *Authenticator (`authenticator`)*:: -Authenticator logs are unique to Amazon EKS. These logs represent the control plane component that Amazon EKS uses for [.noloc]`Kubernetes` https://kubernetes.io/docs/reference/access-authn-authz/rbac/[Role Based Access Control] (RBAC) authentication using IAM credentials. For more information, see <>. +Authenticator logs are unique to Amazon EKS. These logs represent the control plane component that Amazon EKS uses for Kubernetes https://kubernetes.io/docs/reference/access-authn-authz/rbac/[Role Based Access Control] (RBAC) authentication using IAM credentials. For more information, see <>. *Controller manager (`controllerManager`)*:: -The controller manager manages the core control loops that are shipped with [.noloc]`Kubernetes`. For more information, see https://kubernetes.io/docs/reference/command-line-tools-reference/kube-controller-manager/[kube-controller-manager] in the [.noloc]`Kubernetes` documentation. +The controller manager manages the core control loops that are shipped with Kubernetes. For more information, see https://kubernetes.io/docs/reference/command-line-tools-reference/kube-controller-manager/[kube-controller-manager] in the Kubernetes documentation. *Scheduler (`scheduler`)*:: -The scheduler component manages when and where to run [.noloc]`Pods` in your cluster. For more information, see https://kubernetes.io/docs/reference/command-line-tools-reference/kube-scheduler/[kube-scheduler] in the [.noloc]`Kubernetes` documentation. +The scheduler component manages when and where to run Pods in your cluster. For more information, see https://kubernetes.io/docs/reference/command-line-tools-reference/kube-scheduler/[kube-scheduler] in the Kubernetes documentation. -[[enabling-control-plane-log-export,enabling-control-plane-log-export.title]] +[#enabling-control-plane-log-export] == Enable or disable control plane logs By default, cluster control plane logs aren't sent to CloudWatch Logs. You must enable each log type individually to send logs for your cluster. CloudWatch Logs ingestion, archive storage, and data scanning rates apply to enabled control plane logs. For more information, see link:cloudwatch/pricing/[CloudWatch pricing,type="marketing"]. @@ -54,16 +49,16 @@ To update the control plane logging configuration, Amazon EKS requires up to fiv You can enable or disable control plane logs with either the <> or the <>. -[[control-plane-console,control-plane-console.title]] +[#control-plane-console] === {aws-management-console} . Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. . Choose the name of the cluster to display your cluster information. . Choose the *Observability* tab. -. In the *Control plane logging* section, choose *Manage logging*. +. In the *Control plane logging* section, choose *Manage logging*. . For each individual log type, choose whether the log type should be turned on or turned off. By default, each log type is turned off. . Choose *Save changes* to finish. -[[control-plane-cli,control-plane-cli.title]] +[#control-plane-cli] === {aws} CLI . Check your {aws} CLI version with the following command. + @@ -136,7 +131,7 @@ An example output is as follows. } ---- -[[viewing-control-plane-logs,viewing-control-plane-logs.title]] +[#viewing-control-plane-logs] == View cluster control plane logs After you have enabled any of the control plane log types for your Amazon EKS cluster, you can view them on the CloudWatch console. @@ -149,7 +144,7 @@ To learn more about viewing, analyzing, and managing logs in CloudWatch, see the + NOTE: As log stream data grows, the log stream names are rotated. When multiple log streams exist for a particular log type, you can view the latest log stream by looking for the log stream name with the latest *Last event time*. + -** *[.noloc]`Kubernetes` API server component logs (`api`)* – `kube-apiserver-[.replaceable]``1234567890abcdef01234567890abcde``` +** *Kubernetes API server component logs (`api`)* – `kube-apiserver-[.replaceable]``1234567890abcdef01234567890abcde``` ** *Audit (`audit`)* – `kube-apiserver-audit-[.replaceable]``1234567890abcdef01234567890abcde``` ** *Authenticator (`authenticator`)* – `authenticator-[.replaceable]``1234567890abcdef01234567890abcde``` ** *Controller manager (`controllerManager`)* – `kube-controller-manager-[.replaceable]``1234567890abcdef01234567890abcde``` @@ -160,4 +155,4 @@ For example, you should see the initial API server flags for the cluster when vi + NOTE: If you don't see the API server logs at the beginning of the log stream, then it is likely that the API server log file was rotated on the server before you enabled API server logging on the server. Any log files that are rotated before API server logging is enabled can't be exported to CloudWatch. -However, you can create a new cluster with the same [.noloc]`Kubernetes` version and enable the API server logging when you create the cluster. Clusters with the same platform version have the same flags enabled, so your flags should match the new cluster's flags. When you finish viewing the flags for the new cluster in CloudWatch, you can delete the new cluster. +However, you can create a new cluster with the same Kubernetes version and enable the API server logging when you create the cluster. Clusters with the same platform version have the same flags enabled, so your flags should match the new cluster's flags. When you finish viewing the flags for the new cluster in CloudWatch, you can delete the new cluster. \ No newline at end of file diff --git a/latest/ug/observability/deploy-prometheus.adoc b/latest/ug/observability/deploy-prometheus.adoc index b7da384d2..b83752830 100644 --- a/latest/ug/observability/deploy-prometheus.adoc +++ b/latest/ug/observability/deploy-prometheus.adoc @@ -1,20 +1,20 @@ -//!!NODE_ROOT
include::../attributes.txt[] + [.topic] -[[deploy-prometheus,deploy-prometheus.title]] -= Deploy [.noloc]`Prometheus` using [.noloc]`Helm` +[#deploy-prometheus] += Deploy Prometheus using Helm :info_titleabbrev: Deploy using Helm [abstract] -- -As an alternative to using Amazon Managed Service for Prometheus, you can deploy [.noloc]`Prometheus` into your cluster with [.noloc]`Helm` V3. +As an alternative to using Amazon Managed Service for Prometheus, you can deploy Prometheus into your cluster with https://helm.sh/docs/topics/version_skew/#supported-version-skew[supported Helm versions]. -- -As an alternative to using Amazon Managed Service for Prometheus, you can deploy [.noloc]`Prometheus` into your cluster with [.noloc]`Helm` V3. If you already have [.noloc]`Helm` installed, you can check your version with the `helm version` command. [.noloc]`Helm` is a package manager for [.noloc]`Kubernetes` clusters. For more information about [.noloc]`Helm` and how to install it, see <>. +As an alternative to using Amazon Managed Service for Prometheus, you can deploy Prometheus into your cluster with Helm. If you already have Helm installed, you can check your version with the `helm version` command. Helm is a package manager for Kubernetes clusters. For more information about Helm and how to install it, see <>. -After you configure [.noloc]`Helm` for your Amazon EKS cluster, you can use it to deploy [.noloc]`Prometheus` with the following steps. +After you configure Helm for your Amazon EKS cluster, you can use it to deploy Prometheus with the following steps. -. Create a [.noloc]`Prometheus` namespace. +. Create a Prometheus namespace. + [source,bash,subs="verbatim,attributes"] ---- @@ -26,7 +26,7 @@ kubectl create namespace prometheus ---- helm repo add prometheus-community https://prometheus-community.github.io/helm-charts ---- -. Deploy [.noloc]`Prometheus`. +. Deploy Prometheus. + [source,bash,subs="verbatim,attributes"] ---- @@ -36,11 +36,11 @@ helm upgrade -i prometheus prometheus-community/prometheus \ --set server.persistentVolume.storageClass="gp2" ---- + -NOTE: If you get the error `Error: failed to download "stable/prometheus" (hint: running `helm repo update` may help)` when executing this command, run `helm repo update prometheus-community`, and then try running the Step 2 command again. +NOTE: If you get the error `Error: failed to download "stable/prometheus" (hint: running helm repo update may help)` when executing this command, run `helm repo update prometheus-community`, and then try running the Step 2 command again. + If you get the error `Error: rendered manifests contain a resource that already exists`, run `helm uninstall [.replaceable]``your-release-name`` -n [.replaceable]``namespace```, then try running the Step 3 command again. + -. Verify that all of the [.noloc]`Pods` in the `prometheus` namespace are in the `READY` state. +. Verify that all of the Pods in the `prometheus` namespace are in the `READY` state. + [source,bash,subs="verbatim,attributes"] ---- @@ -60,18 +60,18 @@ prometheus-node-exporter-vbdks 1/1 Running 0 48 prometheus-pushgateway-76c444b68c-82tnw 1/1 Running 0 48s prometheus-server-775957f748-mmht9 1/2 Running 0 48s ---- -. Use `kubectl` to port forward the [.noloc]`Prometheus` console to your local machine. +. Use `kubectl` to port forward the Prometheus console to your local machine. + [source,bash,subs="verbatim,attributes"] ---- kubectl --namespace=prometheus port-forward deploy/prometheus-server 9090 ---- -. Point a web browser to `http://localhost:9090` to view the [.noloc]`Prometheus` console. -. Choose a metric from the *- insert metric at cursor* menu, then choose *Execute*. Choose the *Graph* tab to show the metric over time. The following image shows `container_memory_usage_bytes` over time. +. Point a web browser to `http://localhost:9090` to view the Prometheus console. +. Choose a metric from the *- insert metric at cursor* menu, then choose *Execute*. Choose the *Graph* tab to show the metric over time. The following image shows `container_memory_usage_bytes` over time. + image::images/prometheus-metric.png[Prometheus metrics,scaledwidth=100%] -. From the top navigation bar, choose *Status*, then *Targets*. +. From the top navigation bar, choose *Status*, then *Targets*. + image::images/prometheus.png[Prometheus console,scaledwidth=100%] + -All of the [.noloc]`Kubernetes` endpoints that are connected to [.noloc]`Prometheus` using service discovery are displayed. +All of the Kubernetes endpoints that are connected to Prometheus using service discovery are displayed. \ No newline at end of file diff --git a/latest/ug/observability/eks-observe.adoc b/latest/ug/observability/eks-observe.adoc index 463c58f04..5a19c5e3c 100644 --- a/latest/ug/observability/eks-observe.adoc +++ b/latest/ug/observability/eks-observe.adoc @@ -1,21 +1,8 @@ -//!!NODE_ROOT include::../attributes.txt[] -[[eks-observe,eks-observe.title]] + +[#eks-observe] = Monitor your cluster performance and view logs -:doctype: book -:sectnums: -:toc: left -:icons: font -:experimental: -:idprefix: -:idseparator: - -:sourcedir: . -:info_doctype: chapter -:info_title: Monitor your cluster performance and view logs :info_titleabbrev: Monitor clusters -:keywords: observability, monitoring, logging, logs, data -:info_abstract: You can observe your data in Amazon EKS using many available monitoring or logging \ - tools. [abstract] -- @@ -24,7 +11,7 @@ You can observe your data in Amazon EKS using many available monitoring or loggi You can observe your data in Amazon EKS using many available monitoring or logging tools. Your Amazon EKS log data can be streamed to {aws} services or to partner tools for data analysis. There are many services available in the {aws-management-console} that provide data for troubleshooting your Amazon EKS issues. You can also use an {aws}-supported open-source solution for link:grafana/latest/userguide/solution-eks.html[monitoring Amazon EKS infrastructure,type="documentation"]. -After selecting *Clusters* in the left navigation pane of the Amazon EKS console, you can view cluster health and details by choosing your cluster's name and choosing the *Observability* tab. To view details about any existing [.noloc]`Kubernetes` resources that are deployed to your cluster, see <>. +After selecting *Clusters* in the left navigation pane of the Amazon EKS console, you can view cluster health and details by choosing your cluster's name and choosing the *Observability* tab. To view details about any existing Kubernetes resources that are deployed to your cluster, see <>. Monitoring is an important part of maintaining the reliability, availability, and performance of Amazon EKS and your {aws} solutions. We recommend that you collect monitoring data from all of the parts of your {aws} solution. That way, you can more easily debug a multi-point failure if one occurs. Before you start monitoring Amazon EKS, make sure that your monitoring plan addresses the following questions. @@ -35,7 +22,7 @@ Monitoring is an important part of maintaining the reliability, availability, an * Who do you intend to perform the monitoring tasks? * Whom do you want notifications to be sent to when something goes wrong? -[[logging-monitoring,logging-monitoring.title]] +[#logging-monitoring] == Monitoring and logging on Amazon EKS Amazon EKS provides built-in tools for monitoring and logging. For supported versions, the observability dashboard gives visibility into the performance of your cluster. It helps you to quickly detect, troubleshoot, and remediate issues. In addition to monitoring features, it includes lists based on the control plane audit logs. The Kubernetes control plane exposes a number of metrics that that can also be scraped outside of the console. @@ -62,18 +49,18 @@ For low-level, customizable logging, then https://kubernetes.io/docs/concepts/cl Amazon EKS is integrated with {aws} CloudTrail, a service that provides a record of actions taken by a user, role, or an {aws} service in Amazon EKS. CloudTrail captures all API calls for Amazon EKS as events. The calls captured include calls from the Amazon EKS console and code calls to the Amazon EKS API operations. For more information, see <>. -The [.noloc]`Kubernetes` API server exposes a number of metrics that are useful for monitoring and analysis. For more information, see <>. +The Kubernetes API server exposes a number of metrics that are useful for monitoring and analysis. For more information, see <>. -To configure [.noloc]`Fluent Bit` for custom Amazon CloudWatch logs, see link:AmazonCloudWatch/latest/monitoring/Container-Insights-setup-logs-FluentBit.html#Container-Insights-FluentBit-setup[Setting up Fluent Bit,type="documentation"] in the _Amazon CloudWatch User Guide_. +To configure Fluent Bit for custom Amazon CloudWatch logs, see link:AmazonCloudWatch/latest/monitoring/Container-Insights-setup-logs-FluentBit.html#Container-Insights-FluentBit-setup[Setting up Fluent Bit,type="documentation"] in the _Amazon CloudWatch User Guide_. -[[eks-monitor-tools,eks-monitor-tools.title]] +[#eks-monitor-tools] == Amazon EKS monitoring and logging tools Amazon Web Services provides various tools that you can use to monitor Amazon EKS. You can configure some tools to set up automatic monitoring, but some require manual calls. We recommend that you automate monitoring tasks as much as your environment and existing toolset allows. The following table describes various monitoring tool options. -[cols="1,1,1,1", options="header"] +[%header,cols="4"] |=== |Areas @@ -88,7 +75,7 @@ The following table describes various monitoring tool options. |Applications / control plane |link:prometheus/latest/userguide/what-is-Amazon-Managed-Service-Prometheus.html[Prometheus,type="documentation"] -|[.noloc]`Prometheus` can be used to monitor metrics and alerts for applications and the control plane. +|Prometheus can be used to monitor metrics and alerts for applications and the control plane. |<> |Applications @@ -108,7 +95,7 @@ The following table describes various monitoring tool options. |Applications |link:xray/latest/devguide/aws-xray.html[{aws} X-Ray,type="documentation"] -|{aws} X-Ray receives trace data about your application. This trace data includes ingoing and outgoing requests and metadata about the requests. For Amazon EKS, the implementation requires the [.noloc]`OpenTelemetry` add-on. +|{aws} X-Ray receives trace data about your application. This trace data includes ingoing and outgoing requests and metadata about the requests. For Amazon EKS, the implementation requires the OpenTelemetry add-on. |link:xray/latest/devguide/xray-instrumenting-your-app.html[Setup procedure,type="documentation"] |Applications @@ -120,7 +107,7 @@ The following table describes various monitoring tool options. The following table describes various logging tool options. -[cols="1,1,1,1", options="header"] +[%header,cols="4"] |=== |Areas @@ -166,3 +153,5 @@ include::control-plane-logs.adoc[leveloffset=+1] include::logging-using-cloudtrail.adoc[leveloffset=+1] include::opentelemetry.adoc[leveloffset=+1] + +include::cluster-dashboard.adoc[leveloffset=+1] diff --git a/latest/ug/observability/enable-asg-metrics.adoc b/latest/ug/observability/enable-asg-metrics.adoc index e8ec7789d..64f91c0de 100644 --- a/latest/ug/observability/enable-asg-metrics.adoc +++ b/latest/ug/observability/enable-asg-metrics.adoc @@ -1,7 +1,7 @@ -//!!NODE_ROOT
include::../attributes.txt[] + [.topic] -[[enable-asg-metrics,enable-asg-metrics.title]] +[#enable-asg-metrics] = View metrics for Amazon EC2 Auto Scaling groups :info_titleabbrev: Auto Scaling group metrics @@ -14,4 +14,4 @@ Amazon EKS managed node groups have Amazon EC2 Auto Scaling group metrics enable With Auto Scaling group metrics collection, you're able to monitor the scaling of managed node groups. Auto Scaling group metrics report the minimum, maximum, and desired size of an Auto Scaling group. You can create an alarm if the number of nodes in a node group falls below the minimum size, which would indicate an unhealthy node group. Tracking node group size is also useful in adjusting the maximum count so that your data plane doesn't run out of capacity. -If you would prefer to not have these metrics collected, you can choose to disable all or only some of them. For example, you can do this to avoid noise in your CloudWatch dashboards. For more information, see link:autoscaling/ec2/userguide/ec2-auto-scaling-cloudwatch-monitoring.html[Amazon CloudWatch metrics for Amazon EC2 Auto Scaling,type="documentation"]. +If you would prefer to not have these metrics collected, you can choose to disable all or only some of them. For example, you can do this to avoid noise in your CloudWatch dashboards. For more information, see link:autoscaling/ec2/userguide/ec2-auto-scaling-cloudwatch-monitoring.html[Amazon CloudWatch metrics for Amazon EC2 Auto Scaling,type="documentation"]. \ No newline at end of file diff --git a/latest/ug/observability/logging-using-cloudtrail.adoc b/latest/ug/observability/logging-using-cloudtrail.adoc index 4136d6e62..58aaca584 100644 --- a/latest/ug/observability/logging-using-cloudtrail.adoc +++ b/latest/ug/observability/logging-using-cloudtrail.adoc @@ -1,14 +1,9 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[logging-using-cloudtrail,logging-using-cloudtrail.title]] +[#logging-using-cloudtrail] = Log API calls as {aws} CloudTrail events -:info_doctype: section -:info_title: Log API calls as {aws} CloudTrail events :info_titleabbrev: {aws} CloudTrail -:keywords: logging, API calls, {aws} CloudTrail -:info_abstract: Learn about logging Amazon EKS with {aws} CloudTrail. [abstract] -- @@ -29,4 +24,4 @@ include::service-name-info-in-cloudtrail.adoc[leveloffset=+1] include::understanding-service-name-entries.adoc[leveloffset=+1] -include::enable-asg-metrics.adoc[leveloffset=+1] +include::enable-asg-metrics.adoc[leveloffset=+1] \ No newline at end of file diff --git a/latest/ug/observability/observability-dashboard.adoc b/latest/ug/observability/observability-dashboard.adoc index cb949d4cc..ee07e806b 100644 --- a/latest/ug/observability/observability-dashboard.adoc +++ b/latest/ug/observability/observability-dashboard.adoc @@ -1,71 +1,48 @@ -//!!NODE_ROOT
+include::../attributes.txt[] [.topic] -[[observability-dashboard,observability-dashboard.title]] +[#observability-dashboard] = Monitor your cluster with the observability dashboard -:info_doctype: section :info_titleabbrev: Observability dashboard -:keywords: observability, dashboard -:info_abstract: Learn how to configure logging for your Amazon EKS cluster. - -include::../attributes.txt[] [abstract] -- Learn how to configure logging for your Amazon EKS cluster. -- -The Amazon EKS console includes an observability dashboard that gives visibility into the performance of your cluster. The information in this dashboard helps you to quickly detect, troubleshoot, and remediate issues. You can open the applicable section of the dashboard by choosing an item in the *Health and performance summary*. This summary is included in several places, including the *Observability* tab. +The Amazon EKS console includes an observability dashboard that gives visibility into the performance of your cluster. The information it provides helps you to quickly detect, troubleshoot, and remediate issues. You can open the applicable section of the observability dashboard by choosing an item in the *Health and performance summary*. This summary is included in several places, including the *Observability* tab. -The dashboard is split into several tabs. +The observability dashboard is split into several tabs. -[[observability-summary,observability-summary.title]] +[#observability-summary] == Summary -The *Health and performance summary* lists the quantity of items in various categories. Each number acts as a hyperlink to a location in the dashboard with a list for that category. +The *Health and performance summary* lists the quantity of items in various categories. Each number acts as a hyperlink to a location in the observability dashboard with a list for that category. -[[observability-cluster-health,observability-cluster-health.title]] -== Cluster health issues +[#observability-cluster-health] +== Cluster health -*Cluster health issues* are important notifications to be aware of, some of which you may need to take action on as soon as possible. With this list, you can see descriptions and the affected resources. To refresh the status, choose the refresh button ( ↻ ). +*Cluster health* provides important notifications to be aware of, some of which you may need to take action on as soon as possible. With this list, you can see descriptions and the affected resources. Cluster health includes two tables: *Health issues* and *Configuration insights*. To refresh the status of *Health issues*, choose the refresh button ( ↻ ). *Configuration insights* update automatically once every 24 hours and can't be manually refreshed. -For more information, see <>. +For more information about *Health issues*, see <>. For more information about *Configuration insights*, see <>. -[[observability-control-plane,observability-control-plane.title]] +[#observability-control-plane] == Control plane monitoring The *Control plane monitoring* tab is divided into three sections, each of which help you to monitor and troubleshoot your cluster's control plane. -[[observability-metrics,observability-metrics.title]] +[#observability-metrics] === Metrics -The *Metrics* section shows graphs of several metrics gathered for various control plane components. This specific feature is only available for new clusters and previous clusters with a platform version that is the same or later compared to the following table. - -[cols="1,1", options="header"] -|=== -|Kubernetes version -|Platform version -|`1.31` -|`eks.12` - -|`1.30` -|`eks.20` - -|`1.29` -|`eks.23` - -|`1.28` -|`eks.29` - -|=== +For clusters that are Kubernetes version `1.28` and above, the *Metrics* section shows graphs of several metrics gathered for various control plane components. You can set the time period used by the X-axis of every graph by making selections at the top of the section. You can refresh data with the refresh button ( ↻ ). For each separate graph, the vertical ellipses button ( ⋮ ) opens a menu with options from CloudWatch. -These metrics and more are automatically available as basic monitoring metrics in CloudWatch under the `AWS/EKS` namespace. For more information, see link:AmazonCloudWatch/latest/monitoring/cloudwatch-metrics-basic-detailed.html[Basic monitoring and detailed monitoring,type="documentation"] in the _Amazon CloudWatch User Guide_. To get more detailed metrics, visualization, and insights, see link:AmazonCloudWatch/latest/monitoring/ContainerInsights.html[Container Insights,type="documentation"] in the _Amazon CloudWatch User Guide_. Or if you prefer [.noloc]`Prometheus` based monitoring, see <>. +These metrics and more are automatically available as basic monitoring metrics in CloudWatch under the `AWS/EKS` namespace. For more information, see link:AmazonCloudWatch/latest/monitoring/cloudwatch-metrics-basic-detailed.html[Basic monitoring and detailed monitoring,type="documentation"] in the _Amazon CloudWatch User Guide_. To get more detailed metrics, visualization, and insights, see link:AmazonCloudWatch/latest/monitoring/ContainerInsights.html[Container Insights,type="documentation"] in the _Amazon CloudWatch User Guide_. Or if you prefer Prometheus based monitoring, see <>. The following table describes available metrics. -[cols="1,1", options="header"] +[%header,cols="2"] |=== |Metric |Description @@ -108,7 +85,7 @@ The following table describes available metrics. |=== -[[observability-log-insights,observability-log-insights.title]] +[#observability-log-insights] === CloudWatch Log Insights The *CloudWatch Log Insights* section shows various lists based on the control plane audit logs. The Amazon EKS control plane logs need to be turned on to use this feature, which you can do from the *View control plane logs in CloudWatch* section. @@ -116,13 +93,13 @@ When enough time has passed to collect data, you can *Run all queries* or choose For more information, see link:AmazonCloudWatch/latest/logs/AnalyzingLogData.html[Analyzing log data with CloudWatch Logs Insights,type="documentation"] in the Amazon CloudWatch Logs User Guide. -[[observability-cp-logs,observability-cp-logs.title]] +[#observability-cp-logs] === View control plane logs in CloudWatch Choose *Manage logging* to update the log types that are available. It takes several minutes for the logs to appear in CloudWatch Logs after you enable logging. When enough time has passed, choose any of the *View* links in this section to navigate to the applicable log. For more information, see <>. -[[observability-cluster-insights,observability-cluster-insights.title]] +[#observability-cluster-insights] == Cluster insights The *Upgrade insights* table both surfaces issues and recommends corrective actions, accelerating the validation process for upgrading to new Kubernetes versions. Amazon EKS automatically scans clusters against a list of potential Kubernetes version upgrade impacting issues. The *Upgrade insights* table lists the insight checks performed by Amazon EKS against this cluster, along with their associated statuses. @@ -131,11 +108,11 @@ Amazon EKS maintains and periodically refreshes the list of insight checks to be For more information, see <>. -[[observability-node-health-issues,observability-node-health-issues.title]] +[#observability-node-health-issues] == Node health issues The Amazon EKS node monitoring agent automatically reads node logs to detect health issues. Regardless of the auto repair setting, all node health issues are reported so that you can investigate as needed. If an issue type is listed without a description, you can read the description in its popover element. When you refresh the page, any resolved issues will disappear from the list. If auto repair is enabled, you could temporarily see some health issues that will be resolved without action from you. Issues that are not supported by auto repair may require manual action from you depending on the type. -For node health issues to be reported, your cluster must use Amazon EKS Auto Mode or have the node monitoring agent add-on. For more information, see <>. +For node health issues to be reported, your cluster must use Amazon EKS Auto Mode or have the node monitoring agent add-on. For more information, see <>. \ No newline at end of file diff --git a/latest/ug/observability/opentelemetry.adoc b/latest/ug/observability/opentelemetry.adoc index 7345cd7fa..cd376440a 100644 --- a/latest/ug/observability/opentelemetry.adoc +++ b/latest/ug/observability/opentelemetry.adoc @@ -1,16 +1,9 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[opentelemetry,opentelemetry.title]] +[#opentelemetry] = Send metric and trace data with ADOT Operator -:info_doctype: section -:info_title: Send metric and trace data with ADOT Operator :info_titleabbrev: ADOT Operator -:keywords: ADOT -:info_abstract: The {aws} Distro for OpenTelemetry (ADOT) Operator makes it easier to enable your \ - applications running on Amazon EKS to send metric and trace data to multiple monitoring service \ - options. [abstract] -- @@ -19,4 +12,4 @@ The {aws} Distro for OpenTelemetry (ADOT) Operator makes it easier to enable you Amazon EKS supports using the {aws-management-console}, {aws} CLI and Amazon EKS API to install and manage the https://aws-otel.github.io/[{aws} Distro for OpenTelemetry (ADOT)] Operator. This makes it easier to enable your applications running on Amazon EKS to send metric and trace data to multiple monitoring service options like link:cloudwatch[Amazon CloudWatch,type="console"], link:prometheus[Prometheus,type="console"], and link:xray[X-Ray,type="console"]. -For more information, see https://aws-otel.github.io/docs/getting-started/adot-eks-add-on[Getting Started with {aws} Distro for OpenTelemetry using EKS Add-Ons] in the {aws} Distro for [.noloc]`OpenTelemetry` documentation. +For more information, see https://aws-otel.github.io/docs/getting-started/adot-eks-add-on[Getting Started with {aws} Distro for OpenTelemetry using EKS Add-Ons] in the {aws} Distro for OpenTelemetry documentation. \ No newline at end of file diff --git a/latest/ug/observability/prometheus.adoc b/latest/ug/observability/prometheus.adoc index de795dc4c..490622724 100644 --- a/latest/ug/observability/prometheus.adoc +++ b/latest/ug/observability/prometheus.adoc @@ -1,46 +1,40 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[prometheus,prometheus.title]] -= Monitor your cluster metrics with [.noloc]`Prometheus` -:info_doctype: section -:info_title: Monitor your cluster metrics with Prometheus +[#prometheus] += Monitor your cluster metrics with Prometheus :info_titleabbrev: Prometheus metrics -:keywords: Prometheus, metrics, control plane -:info_abstract: This topic explains how to deploy Prometheus and some of the ways \ - that you can use it to view and analyze what your cluster is doing. [abstract] -- -This topic explains how to deploy [.noloc]`Prometheus` and some of the ways that you can use it to view and analyze what your cluster is doing. +This topic explains how to deploy Prometheus and some of the ways that you can use it to view and analyze what your cluster is doing. -- -https://prometheus.io/[Prometheus] is a monitoring and time series database that scrapes endpoints. It provides the ability to query, aggregate, and store collected data. You can also use it for alerting and alert aggregation. This topic explains how to set up [.noloc]`Prometheus` as either a managed or open source option. Monitoring Amazon EKS control plane metrics is a common use case. +https://prometheus.io/[Prometheus] is a monitoring and time series database that scrapes endpoints. It provides the ability to query, aggregate, and store collected data. You can also use it for alerting and alert aggregation. This topic explains how to set up Prometheus as either a managed or open source option. Monitoring Amazon EKS control plane metrics is a common use case. -Amazon Managed Service for Prometheus is a [.noloc]`Prometheus`-compatible monitoring and alerting service that makes it easy to monitor containerized applications and infrastructure at scale. It is a fully-managed service that automatically scales the ingestion, storage, querying, and alerting of your metrics. It also integrates with {aws} security services to enable fast and secure access to your data. You can use the open-source PromQL query language to query your metrics and alert on them. Also, you can use alert manager in Amazon Managed Service for Prometheus to set up alerting rules for critical alerts. You can then send these critical alerts as notifications to an Amazon SNS topic. +Amazon Managed Service for Prometheus is a Prometheus-compatible monitoring and alerting service that makes it easy to monitor containerized applications and infrastructure at scale. It is a fully-managed service that automatically scales the ingestion, storage, querying, and alerting of your metrics. It also integrates with {aws} security services to enable fast and secure access to your data. You can use the open-source PromQL query language to query your metrics and alert on them. Also, you can use alert manager in Amazon Managed Service for Prometheus to set up alerting rules for critical alerts. You can then send these critical alerts as notifications to an Amazon SNS topic. -There are several different options for using [.noloc]`Prometheus` with Amazon EKS: +There are several different options for using Prometheus with Amazon EKS: -* You can turn on [.noloc]`Prometheus` metrics when first creating an Amazon EKS cluster or you can create your own [.noloc]`Prometheus` scraper for existing clusters. Both of these options are covered by this topic. -* You can deploy [.noloc]`Prometheus` using [.noloc]`Helm`. For more information, see <>. -* You can view control plane raw metrics in [.noloc]`Prometheus` format. For more information, see <>. +* You can turn on Prometheus metrics when first creating an Amazon EKS cluster or you can create your own Prometheus scraper for existing clusters. Both of these options are covered by this topic. +* You can deploy Prometheus using Helm. For more information, see <>. +* You can view control plane raw metrics in Prometheus format. For more information, see <>. -[[turn-on-prometheus-metrics,turn-on-prometheus-metrics.title]] -== Step 1: Turn on [.noloc]`Prometheus` metrics +[#turn-on-prometheus-metrics] +== Step 1: Turn on Prometheus metrics [IMPORTANT] ==== -Amazon Managed Service for [.noloc]`Prometheus` resources are outside of the cluster lifecycle and need to be maintained independent of the cluster. When you delete your cluster, make sure to also delete any applicable scrapers to stop applicable costs. For more information, see link:prometheus/latest/userguide/AMP-collector-how-to.html#AMP-collector-list-delete[Find and delete scrapers,type="documentation"] in the _Amazon Managed Service for Prometheus User Guide_. +Amazon Managed Service for Prometheus resources are outside of the cluster lifecycle and need to be maintained independent of the cluster. When you delete your cluster, make sure to also delete any applicable scrapers to stop applicable costs. For more information, see link:prometheus/latest/userguide/AMP-collector-how-to.html#AMP-collector-list-delete[Find and delete scrapers,type="documentation"] in the _Amazon Managed Service for Prometheus User Guide_. ==== -[.noloc]`Prometheus` discovers and collects metrics from your cluster through a pull-based model called scraping. Scrapers are set up to gather data from your cluster infrastructure and containerized applications. When you turn on the option to send [.noloc]`Prometheus` metrics, Amazon Managed Service for [.noloc]`Prometheus` provides a fully managed agentless scraper. +Prometheus discovers and collects metrics from your cluster through a pull-based model called scraping. Scrapers are set up to gather data from your cluster infrastructure and containerized applications. When you turn on the option to send Prometheus metrics, Amazon Managed Service for Prometheus provides a fully managed agentless scraper. -If you haven't created the cluster yet, you can turn on the option to send metrics to [.noloc]`Prometheus` when first creating the cluster. In the Amazon EKS console, this option is in the *Configure observability* step of creating a new cluster. For more information, see <>. +If you haven't created the cluster yet, you can turn on the option to send metrics to Prometheus when first creating the cluster. In the Amazon EKS console, this option is in the *Configure observability* step of creating a new cluster. For more information, see <>. -If you already have an existing cluster, you can create your own [.noloc]`Prometheus` scraper. To do this in the Amazon EKS console, navigate to your cluster's *Observability* tab and choose the *Add scraper* button. If you would rather do so with the {aws} API or {aws} CLI, see link:prometheus/latest/userguide/AMP-collector-how-to.html#AMP-collector-create[Create a scraper,type="documentation"] in the _Amazon Managed Service for [.noloc]`Prometheus` User Guide_. +If you already have an existing cluster, you can create your own Prometheus scraper. To do this in the Amazon EKS console, navigate to your cluster's *Observability* tab and choose the *Add scraper* button. If you would rather do so with the {aws} API or {aws} CLI, see link:prometheus/latest/userguide/AMP-collector-how-to.html#AMP-collector-create[Create a scraper,type="documentation"] in the _Amazon Managed Service for Prometheus User Guide_. The following options are available when creating the scraper with the Amazon EKS console. @@ -48,46 +42,46 @@ The following options are available when creating the scraper with the Amazon EK (Optional) Enter a unique alias for the scraper. *Destination*:: -Choose an Amazon Managed Service for [.noloc]`Prometheus` workspace. A workspace is a logical space dedicated to the storage and querying of [.noloc]`Prometheus` metrics. With this workspace, you will be able to view [.noloc]`Prometheus` metrics across the accounts that have access to it. The *Create new workspace* option tells Amazon EKS to create a workspace on your behalf using the *Workspace alias* you provide. With the *Select existing workspace* option, you can select an existing workspace from a dropdown list. For more information about workspaces, see link:prometheus/latest/userguide/AMP-manage-ingest-query.html[Managing workspaces,type="documentation"] in the _Amazon Managed Service for Prometheus User Guide_. +Choose an Amazon Managed Service for Prometheus workspace. A workspace is a logical space dedicated to the storage and querying of Prometheus metrics. With this workspace, you will be able to view Prometheus metrics across the accounts that have access to it. The *Create new workspace* option tells Amazon EKS to create a workspace on your behalf using the *Workspace alias* you provide. With the *Select existing workspace* option, you can select an existing workspace from a dropdown list. For more information about workspaces, see link:prometheus/latest/userguide/AMP-manage-ingest-query.html[Managing workspaces,type="documentation"] in the _Amazon Managed Service for Prometheus User Guide_. *Service access*:: -This section summarizes the permissions you grant when sending [.noloc]`Prometheus` metrics: +This section summarizes the permissions you grant when sending Prometheus metrics: + ** Allow Amazon Managed Service for Prometheus to describe the scraped Amazon EKS cluster -** Allow remote writing to the Amazon Managed [.noloc]`Prometheus` workspace +** Allow remote writing to the Amazon Managed Prometheus workspace + -If the `AmazonManagedScraperRole` already exists, the scraper uses it. Choose the `AmazonManagedScraperRole` link to see the *Permission details*. If the `AmazonManagedScraperRole` doesn't exist already, choose the *View permission details* link to see the specific permissions you are granting by sending [.noloc]`Prometheus` metrics. +If the `AmazonManagedScraperRole` already exists, the scraper uses it. Choose the `AmazonManagedScraperRole` link to see the *Permission details*. If the `AmazonManagedScraperRole` doesn't exist already, choose the *View permission details* link to see the specific permissions you are granting by sending Prometheus metrics. *Subnets*:: Modify the subnets that the scraper will inherit as needed. If you need to add a grayed out subnet option, go back to the create cluster *Specify networking* step. *Scraper configuration*:: -Modify the scraper configuration in YAML format as needed. To do so, use the form or upload a replacement YAML file. For more information, see link:prometheus/latest/userguide/AMP-collector-how-to.html#AMP-collector-configuration[Scraper configuration,type="documentation"] in the _Amazon Managed Service for Prometheus User Guide_. +Modify the scraper configuration in YAML format as needed. To do so, use the form or upload a replacement YAML file. For more information, see link:prometheus/latest/userguide/AMP-collector-how-to.html#AMP-collector-configuration[Scraper configuration,type="documentation"] in the _Amazon Managed Service for Prometheus User Guide_. Amazon Managed Service for Prometheus refers to the agentless scraper that is created alongside the cluster as an {aws} managed collector. For more information about {aws} managed collectors, see link:prometheus/latest/userguide/AMP-collector.html[Ingest metrics with {aws} managed collectors,type="documentation"] in the _Amazon Managed Service for Prometheus User Guide_. [IMPORTANT] ==== -* If you create a [.noloc]`Prometheus` scraper using the {aws} CLI or {aws} API, you need to adjust its configuration to give the scraper in-cluster permissions. For more information, see link:prometheus/latest/userguide/AMP-collector-how-to.html#AMP-collector-eks-setup[Configuring your Amazon EKS cluster,type="documentation"] in the _Amazon Managed Service for Prometheus User Guide_. +* If you create a Prometheus scraper using the {aws} CLI or {aws} API, you need to adjust its configuration to give the scraper in-cluster permissions. For more information, see link:prometheus/latest/userguide/AMP-collector-how-to.html#AMP-collector-eks-setup[Configuring your Amazon EKS cluster,type="documentation"] in the _Amazon Managed Service for Prometheus User Guide_. -* If you have a [.noloc]`Prometheus` scraper created before November 11, 2024 that uses the `aws-auth` `ConfigMap` instead of access entries, you need to update it to access additional metrics from the Amazon EKS cluster control plane. For the updated configuration, see link:prometheus/latest/userguide/AMP-collector-how-to.html#AMP-collector-eks-manual-setup[Manually configuring Amazon EKS for scraper access,type="documentation"] in the _Amazon Managed Service for Prometheus User Guide_. +* If you have a Prometheus scraper created before November 11, 2024 that uses the `aws-auth` `ConfigMap` instead of access entries, you need to update it to access additional metrics from the Amazon EKS cluster control plane. For the updated configuration, see link:prometheus/latest/userguide/AMP-collector-how-to.html#AMP-collector-eks-manual-setup[Manually configuring Amazon EKS for scraper access,type="documentation"] in the _Amazon Managed Service for Prometheus User Guide_. ==== -[[use-prometheus-metrics,use-prometheus-metrics.title]] -== Step 2: Use the [.noloc]`Prometheus` metrics +[#use-prometheus-metrics] +== Step 2: Use the Prometheus metrics -For more information about how to use the [.noloc]`Prometheus` metrics after you turn them on for your cluster, see the link:prometheus/latest/userguide/what-is-Amazon-Managed-Service-Prometheus.html[Amazon Managed Service for Prometheus User Guide,type="documentation"]. +For more information about how to use the Prometheus metrics after you turn them on for your cluster, see the link:prometheus/latest/userguide/what-is-Amazon-Managed-Service-Prometheus.html[Amazon Managed Service for Prometheus User Guide,type="documentation"]. -[[viewing-prometheus-scraper-details,viewing-prometheus-scraper-details.title]] -== Step 3: Manage [.noloc]`Prometheus` scrapers +[#viewing-prometheus-scraper-details] +== Step 3: Manage Prometheus scrapers -To manage scrapers, choose the *Observability* tab in the Amazon EKS console. A table shows a list of scrapers for the cluster, including information such as the scraper ID, alias, status, and creation date. You can add more scrapers, delete scrapers, or view more information about the current scrapers. +To manage scrapers, choose the *Observability* tab in the Amazon EKS console. A table shows a list of scrapers for the cluster, including information such as the scraper ID, alias, status, and creation date. You can add more scrapers, edit scrapers, delete scrapers, or view more information about the current scrapers. -To see more details about a scraper, choose the scraper ID link. For example, you can view the ARN, environment, workspace ID, IAM role, configuration, and networking information. You can use the scraper ID as input to Amazon Managed Service for Prometheus API operations like `DescribeScraper` and `DeleteScraper`. For more information on using the [.noloc]`Prometheus` API, see the link:prometheus/latest/userguide/AMP-APIReference.html[Amazon Managed Service for Prometheus API Reference,type="documentation"]. +To see more details about a scraper, choose the scraper ID link. For example, you can view the ARN, environment, workspace ID, IAM role, configuration, and networking information. You can use the scraper ID as input to Amazon Managed Service for Prometheus API operations like link:prometheus/latest/APIReference/API_DescribeScraper.html[`DescribeScraper`,type="documentation"], link:prometheus/latest/APIReference/API_UpdateScraper.html[`UpdateScraper`,type="documentation"], and link:prometheus/latest/APIReference/API_DeleteScraper.html[`DeleteScraper`,type="documentation"]. For more information on using the Prometheus API, see the link:prometheus/latest/userguide/AMP-APIReference.html[Amazon Managed Service for Prometheus API Reference,type="documentation"]. include::deploy-prometheus.adoc[leveloffset=+1] -include::view-raw-metrics.adoc[leveloffset=+1] +include::view-raw-metrics.adoc[leveloffset=+1] \ No newline at end of file diff --git a/latest/ug/observability/service-name-info-in-cloudtrail.adoc b/latest/ug/observability/service-name-info-in-cloudtrail.adoc index a096de03b..6b4ea4d66 100644 --- a/latest/ug/observability/service-name-info-in-cloudtrail.adoc +++ b/latest/ug/observability/service-name-info-in-cloudtrail.adoc @@ -1,7 +1,7 @@ -//!!NODE_ROOT
include::../attributes.txt[] + [.topic] -[[service-name-info-in-cloudtrail,service-name-info-in-cloudtrail.title]] +[#service-name-info-in-cloudtrail] = View helpful references for {aws} CloudTrail :info_titleabbrev: References @@ -10,17 +10,17 @@ include::../attributes.txt[] When any activity occurs in Amazon EKS, that activity is recorded in a CloudTrail event. -- -When you create your {aws} account, CloudTrail is also enabled on your {aws} account. When any activity occurs in Amazon EKS, that activity is recorded in a CloudTrail event along with other {aws} service events in *Event history*. You can view, search, and download recent events in your {aws} account. For more information, see link:awscloudtrail/latest/userguide/view-cloudtrail-events.html[Viewing events with CloudTrail event history,type="documentation"]. +When you create your {aws} account, CloudTrail is also enabled on your {aws} account. When any activity occurs in Amazon EKS, that activity is recorded in a CloudTrail event along with other {aws} service events in *Event history*. You can view, search, and download recent events in your {aws} account. For more information, see link:awscloudtrail/latest/userguide/view-cloudtrail-events.html[Viewing events with CloudTrail event history,type="documentation"]. For an ongoing record of events in your {aws} account, including events for Amazon EKS, create a trail. A _trail_ enables CloudTrail to deliver log files to an Amazon S3 bucket. By default, when you create a trail in the console, the trail applies to all {aws} Regions. The trail logs events from all {aws} Regions in the {aws} partition and delivers the log files to the Amazon S3 bucket that you specify. Additionally, you can configure other {aws} services to further analyze and act upon the event data that's collected in CloudTrail logs. For more information, see the following resources. * link:awscloudtrail/latest/userguide/cloudtrail-create-and-update-a-trail.html[Overview for creating a trail,type="documentation"] -* link:awscloudtrail/latest/userguide/cloudtrail-aws-service-specific-topics.html#cloudtrail-aws-service-specific-topics-integrations[CloudTrail supported services and integrations,type="documentation"] -* link:awscloudtrail/latest/userguide/getting_notifications_top_level.html[Configuring Amazon SNS notifications for CloudTrail,type="documentation"] +* link:awscloudtrail/latest/userguide/cloudtrail-aws-service-specific-topics.html#cloudtrail-aws-service-specific-topics-integrations[CloudTrail supported services and integrations,type="documentation"] +* link:awscloudtrail/latest/userguide/getting_notifications_top_level.html[Configuring Amazon SNS notifications for CloudTrail,type="documentation"] * link:awscloudtrail/latest/userguide/receive-cloudtrail-log-files-from-multiple-regions.html[Receiving CloudTrail log files from multiple regions,type="documentation"] and link:awscloudtrail/latest/userguide/cloudtrail-receive-logs-from-multiple-accounts.html[Receiving CloudTrail log files from multiple accounts,type="documentation"] -All Amazon EKS actions are logged by CloudTrail and are documented in the link:eks/latest/APIReference/[Amazon EKS API Reference,type="documentation"]. For example, calls to the link:eks/latest/APIReference/API_CreateCluster.html[CreateCluster,type="documentation"], link:eks/latest/APIReference/API_ListClusters.html[ListClusters,type="documentation"] and link:eks/latest/APIReference/API_DeleteCluster.html[DeleteCluster,type="documentation"] sections generate entries in the CloudTrail log files. +All Amazon EKS actions are logged by CloudTrail and are documented in the link:eks/latest/APIReference/[Amazon EKS API Reference,type="documentation"]. For example, calls to the link:eks/latest/APIReference/API_CreateCluster.html[CreateCluster,type="documentation"], link:eks/latest/APIReference/API_ListClusters.html[ListClusters,type="documentation"] and link:eks/latest/APIReference/API_DeleteCluster.html[DeleteCluster,type="documentation"] sections generate entries in the CloudTrail log files. Every event or log entry contains information about the type of IAM identity that made the request, and which credentials were used. If temporary credentials were used, the entry shows how the credentials were obtained. -For more information, see the link:awscloudtrail/latest/userguide/cloudtrail-event-reference-user-identity.html[CloudTrail userIdentity element,type="documentation"]. +For more information, see the link:awscloudtrail/latest/userguide/cloudtrail-event-reference-user-identity.html[CloudTrail userIdentity element,type="documentation"]. \ No newline at end of file diff --git a/latest/ug/observability/understanding-service-name-entries.adoc b/latest/ug/observability/understanding-service-name-entries.adoc index f8b044d3b..0516c972f 100644 --- a/latest/ug/observability/understanding-service-name-entries.adoc +++ b/latest/ug/observability/understanding-service-name-entries.adoc @@ -1,7 +1,7 @@ -//!!NODE_ROOT
include::../attributes.txt[] + [.topic] -[[understanding-service-name-entries,understanding-service-name-entries.title]] +[#understanding-service-name-entries] = Analyze {aws} CloudTrail log file entries :info_titleabbrev: Log file entries @@ -12,7 +12,7 @@ A trail is a configuration that enables delivery of events as log files to an Am A trail is a configuration that enables delivery of events as log files to an Amazon S3 bucket that you specify. CloudTrail log files contain one or more log entries. An event represents a single request from any source and includes information about the requested action. This include information such as the date and time of the action and the request parameters that were used. CloudTrail log files aren't an ordered stack trace of the public API calls, so they don't appear in any specific order. -The following example shows a CloudTrail log entry that demonstrates the link:eks/latest/APIReference/API_CreateCluster.html[CreateCluster,type="documentation"] action. +The following example shows a CloudTrail log entry that demonstrates the link:eks/latest/APIReference/API_CreateCluster.html[`CreateCluster`,type="documentation"] action. [source,json,subs="verbatim,attributes"] ---- @@ -70,12 +70,12 @@ The following example shows a CloudTrail log entry that demonstrates the link:e ---- -[[eks-service-linked-role-ct,eks-service-linked-role-ct.title]] +[#eks-service-linked-role-ct] == Log Entries for Amazon EKS Service Linked Roles The Amazon EKS service linked roles make API calls to {aws} resources. CloudTrail log entries with `username: AWSServiceRoleForAmazonEKS` and `username: AWSServiceRoleForAmazonEKSNodegroup` appears for calls made by the Amazon EKS service linked roles. For more information about Amazon EKS and service linked roles, see <>. -The following example shows a CloudTrail log entry that demonstrates a ` link:IAM/latest/APIReference/API_DeleteInstanceProfile.html[DeleteInstanceProfile,type="documentation"]` action that's made by the `AWSServiceRoleForAmazonEKSNodegroup` service linked role, noted in the `sessionContext`. +The following example shows a CloudTrail log entry that demonstrates a link:IAM/latest/APIReference/API_DeleteInstanceProfile.html[`DeleteInstanceProfile`,type="documentation"] action that's made by the `AWSServiceRoleForAmazonEKSNodegroup` service linked role, noted in the `sessionContext`. [source,json,subs="verbatim,attributes"] ---- @@ -118,4 +118,4 @@ The following example shows a CloudTrail log entry that demonstrates a ` link:IA "eventType": "AwsApiCall", "recipientAccountId": "111122223333" } ----- +---- \ No newline at end of file diff --git a/latest/ug/observability/view-raw-metrics.adoc b/latest/ug/observability/view-raw-metrics.adoc index e373899bf..27b470fb3 100644 --- a/latest/ug/observability/view-raw-metrics.adoc +++ b/latest/ug/observability/view-raw-metrics.adoc @@ -1,21 +1,21 @@ -//!!NODE_ROOT
include::../attributes.txt[] + [.topic] -[[view-raw-metrics,view-raw-metrics.title]] -= Fetch control plane raw metrics in [.noloc]`Prometheus` format +[#view-raw-metrics] += Fetch control plane raw metrics in Prometheus format :info_titleabbrev: Control plane [abstract] -- -The [.noloc]`Kubernetes` control plane exposes a number of metrics that are represented in a [.noloc]`Prometheus` format. +The Kubernetes control plane exposes a number of metrics that are represented in a Prometheus format. -- -The [.noloc]`Kubernetes` control plane exposes a number of metrics that are represented in a https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md[Prometheus format]. These metrics are useful for monitoring and analysis. They are exposed internally through metrics endpoints, and can be accessed without fully deploying [.noloc]`Prometheus`. However, deploying [.noloc]`Prometheus` more easily allows analyzing metrics over time. +The Kubernetes control plane exposes a number of metrics that are represented in a https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md[Prometheus format]. These metrics are useful for monitoring and analysis. They are exposed internally through metrics endpoints, and can be accessed without fully deploying Prometheus. However, deploying Prometheus more easily allows analyzing metrics over time. -To view the raw metrics output, run the following command. -[source,bash,subs="verbatim,attributes,quotes"] +To view the raw metrics output, replace `endpoint` and run the following command. +[source,bash,subs="verbatim,attributes"] ---- -kubectl get --raw [.replaceable]`endpoint` +kubectl get --raw endpoint ---- This command allows you to pass any endpoint path and returns the raw response. The output lists different metrics line-by-line, with each line including a metric name, tags, and a value. @@ -25,7 +25,7 @@ This command allows you to pass any endpoint path and returns the raw response. metric_name{tag="value"[,...]} value ---- -[[fetch-metrics,fetch-metrics.title]] +[#fetch-metrics] == Fetch metrics from the API server The general API server endpoint is exposed on the Amazon EKS control plane. This endpoint is primarily useful when looking at a specific metric. @@ -59,36 +59,17 @@ ssh_tunnel_open_fail_count 0 This raw output returns verbatim what the API server exposes. -[[fetch-metrics-prometheus,fetch-metrics-prometheus.title]] +[#fetch-metrics-prometheus] == Fetch control plane metrics with `metrics.eks.amazonaws.com` -For new clusters that are [.noloc]`Kubernetes` version `1.28` and above, Amazon EKS also exposes metrics under the API group `metrics.eks.amazonaws.com`. These metrics include control plane components such as `kube-scheduler` and `kube-controller-manager`. These metrics are also available for existing clusters that have a platform version that is the same or later compared to the following table. - -[cols="1,1", options="header"] -|=== -|Kubernetes version -|Platform version - -|`1.31` -|`eks.10` - -|`1.30` -|`eks.18` - -|`1.29` -|`eks.21` - -|`1.28` -|`eks.27` - -|=== +For clusters that are Kubernetes version `1.28` and above, Amazon EKS also exposes metrics under the API group `metrics.eks.amazonaws.com`. These metrics include control plane components such as `kube-scheduler` and `kube-controller-manager`. [NOTE] ==== If you have a webhook configuration that could block the creation of the new `APIService` resource `v1.metrics.eks.amazonaws.com` on your cluster, the metrics endpoint feature might not be available. You can verify that in the `kube-apiserver` audit log by searching for the `v1.metrics.eks.amazonaws.com` keyword. ==== -[[fetch-metrics-scheduler,fetch-metrics-scheduler.title]] +[#fetch-metrics-scheduler] === Fetch `kube-scheduler` metrics To retrieve `kube-scheduler` metrics, use the following command. @@ -118,7 +99,7 @@ scheduler_pod_scheduling_attempts_bucket{le="+Inf"} 81 [...] ---- -[[fetch-metrics-controller,fetch-metrics-controller.title]] +[#fetch-metrics-controller] === Fetch `kube-controller-manager` metrics To retrieve `kube-controller-manager` metrics, use the following command. @@ -150,12 +131,12 @@ workqueue_work_duration_seconds_sum{name="replicaset"} 4.265655885000002 [...] ---- -[[scheduler-controller-metrics,scheduler-controller-metrics.title]] +[#scheduler-controller-metrics] === Understand the scheduler and controller manager metrics -The following table describes the scheduler and controller manager metrics that are made available for [.noloc]`Prometheus` style scraping. For more information about these metrics, see https://kubernetes.io/docs/reference/instrumentation/metrics/[Kubernetes Metrics Reference] in the [.noloc]`Kubernetes` documentation. +The following table describes the scheduler and controller manager metrics that are made available for Prometheus style scraping. For more information about these metrics, see https://kubernetes.io/docs/reference/instrumentation/metrics/[Kubernetes Metrics Reference] in the Kubernetes documentation. -[cols="1,1,1", options="header"] +[%header,cols="3"] |=== |Metric |Control plane component @@ -189,14 +170,6 @@ The following table describes the scheduler and controller manager metrics that |scheduler |The end-to-end latency for a Pod being scheduled, from the time the Pod enters the scheduling queue. This might involve multiple scheduling attempts. -|kube_pod_resource_request -|scheduler -|The resources requested by workloads on the cluster, broken down by Pod. This shows the resource usage the scheduler and kubelet expect per Pod for resources along with the unit for the resource if any. - -|kube_pod_resource_limit -|scheduler -|The resources limit for workloads on the cluster, broken down by Pod. This shows the resource usage the scheduler and kubelet expect per Pod for resources along with the unit for the resource if any. - |cronjob_controller_job_creation_skew_duration_seconds |controller manager |The time between when a cronjob is scheduled to be run, and when the corresponding job is created. @@ -219,7 +192,7 @@ The following table describes the scheduler and controller manager metrics that |=== -[[deploy-prometheus-scraper,deploy-prometheus-scraper.title]] +[#deploy-prometheus-scraper] == Deploy a Prometheus scraper to consistently scrape metrics To deploy a Prometheus scraper to consistently scrape the metrics, use the following configuration: @@ -344,10 +317,10 @@ kubectl patch clusterrole --type=json -p='[ ]' ---- -Then you can view the [.noloc]`Prometheus` dashboard by proxying the port of the [.noloc]`Prometheus` scraper to your local port. +Then you can view the Prometheus dashboard by proxying the port of the Prometheus scraper to your local port. [source,bash,subs="verbatim,attributes"] ---- kubectl port-forward pods/prom-pod 9090:9090 ---- -For your Amazon EKS cluster, the core Kubernetes control plane metrics are also ingested into Amazon CloudWatch Metrics under the `AWS/EKS` namespace. To view them, open the link:cloudwatch/home#logs:prefix=/aws/eks[CloudWatch console,type="console"] and select *All metrics* from the left navigation pane. On the *Metrics* selection page, choose the `AWS/EKS` namespace and a metrics dimension for your cluster. +For your Amazon EKS cluster, the core Kubernetes control plane metrics are also ingested into Amazon CloudWatch Metrics under the `AWS/EKS` namespace. To view them, open the link:cloudwatch/home#logs:prefix=/aws/eks[CloudWatch console,type="console"] and select *All metrics* from the left navigation pane. On the *Metrics* selection page, choose the `AWS/EKS` namespace and a metrics dimension for your cluster. \ No newline at end of file diff --git a/latest/ug/outposts/eks-outposts-capacity-considerations.adoc b/latest/ug/outposts/eks-outposts-capacity-considerations.adoc index a79039d99..55f750517 100644 --- a/latest/ug/outposts/eks-outposts-capacity-considerations.adoc +++ b/latest/ug/outposts/eks-outposts-capacity-considerations.adoc @@ -1,8 +1,7 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[eks-outposts-capacity-considerations,eks-outposts-capacity-considerations.title]] +[#eks-outposts-capacity-considerations] = Select instance types and placement groups for Amazon EKS clusters on {aws} Outposts based on capacity considerations :info_titleabbrev: Capacity considerations @@ -11,9 +10,9 @@ include::../attributes.txt[] Learn how to select instance types and optionally use placement groups to meet high availability requirements for your Amazon EKS local cluster on {aws} Outposts. -- -This topic provides guidance for selecting the [.noloc]`Kubernetes` control plane instance type and (optionally) using placement groups to meet high-availability requirements for your local Amazon EKS cluster on an Outpost. +This topic provides guidance for selecting the Kubernetes control plane instance type and (optionally) using placement groups to meet high-availability requirements for your local Amazon EKS cluster on an Outpost. -Before you select an instance type (such as `m5`, `c5`, or `r5`) to use for your local cluster's [.noloc]`Kubernetes` control plane on Outposts, confirm the instance types that are available on your Outpost configuration. After you identify the available instance types, select the instance size (such as `large`, `xlarge`, or `2xlarge`) based on the number of nodes that your workloads require. The following table provides recommendations for choosing an instance size. +Before you select an instance type (such as `m5`, `c5`, or `r5`) to use for your local cluster's Kubernetes control plane on Outposts, confirm the instance types that are available on your Outpost configuration. After you identify the available instance types, select the instance size (such as `large`, `xlarge`, or `2xlarge`) based on the number of nodes that your workloads require. The following table provides recommendations for choosing an instance size. [NOTE] ==== @@ -22,7 +21,7 @@ The instance sizes must be slotted on your Outposts. Make sure that you have eno ==== -[cols="1,1", options="header"] +[%header,cols="2"] |=== |Number of nodes |Kubernetes control plane instance size @@ -41,27 +40,27 @@ The instance sizes must be slotted on your Outposts. Make sure that you have eno |`4xlarge` |=== -The storage for the [.noloc]`Kubernetes` control plane requires 246 GB of Amazon EBS storage for each local cluster to meet the required IOPS for `etcd`. When the local cluster is created, the Amazon EBS volumes are provisioned automatically for you. +The storage for the Kubernetes control plane requires 246 GB of Amazon EBS storage for each local cluster to meet the required IOPS for `etcd`. When the local cluster is created, the Amazon EBS volumes are provisioned automatically for you. -[[outpost-capacity-considerations-control-plane-placement,outpost-capacity-considerations-control-plane-placement.title]] +[#outpost-capacity-considerations-control-plane-placement] == Control plane placement -When you don't specify a placement group with the `OutpostConfig.ControlPlanePlacement.GroupName` property, the Amazon EC2 instances provisioned for your [.noloc]`Kubernetes` control plane don't receive any specific hardware placement enforcement across the underlying capacity available on your Outpost. +When you don't specify a placement group with the `OutpostConfig.ControlPlanePlacement.GroupName` property, the Amazon EC2 instances provisioned for your Kubernetes control plane don't receive any specific hardware placement enforcement across the underlying capacity available on your Outpost. -You can use placement groups to meet the high-availability requirements for your local Amazon EKS cluster on an Outpost. By specifying a placement group during cluster creation, you influence the placement of the [.noloc]`Kubernetes` control plane instances. The instances are spread across independent underlying hardware (racks or hosts), minimizing correlated instance impact on the event of hardware failures. +You can use placement groups to meet the high-availability requirements for your local Amazon EKS cluster on an Outpost. By specifying a placement group during cluster creation, you influence the placement of the Kubernetes control plane instances. The instances are spread across independent underlying hardware (racks or hosts), minimizing correlated instance impact on the event of hardware failures. The type of spread that you can configure depends on the number of Outpost racks you have in your deployment. -* *Deployments with one or two physical racks in a single logical Outpost* – You must have at least three hosts that are configured with the instance type that you choose for your [.noloc]`Kubernetes` control plane instances. A _spread_ placement group using _host-level spread_ ensures that all [.noloc]`Kubernetes` control plane instances run on distinct hosts within the underlying racks available in your Outpost deployment. -* *Deployments with three or more physical racks in a single logical Outpost* – You must have at least three hosts configured with the instance type you choose for your [.noloc]`Kubernetes` control plane instances. A _spread_ placement group using _rack-level spread_ ensures that all [.noloc]`Kubernetes` control plane instances run on distinct racks in your Outpost deployment. You can alternatively use the _host-level spread_ placement group as described in the previous option. +* *Deployments with one or two physical racks in a single logical Outpost* – You must have at least three hosts that are configured with the instance type that you choose for your Kubernetes control plane instances. A _spread_ placement group using _host-level spread_ ensures that all Kubernetes control plane instances run on distinct hosts within the underlying racks available in your Outpost deployment. +* *Deployments with three or more physical racks in a single logical Outpost* – You must have at least three hosts configured with the instance type you choose for your Kubernetes control plane instances. A _spread_ placement group using _rack-level spread_ ensures that all Kubernetes control plane instances run on distinct racks in your Outpost deployment. You can alternatively use the _host-level spread_ placement group as described in the previous option. You are responsible for creating the desired placement group. You specify the placement group when calling the `CreateCluster` API. For more information about placement groups and how to create them, see link:AWSEC2/latest/UserGuide/placement-groups.html[Placement Groups,type="documentation"] in the Amazon EC2 User Guide. * When a placement group is specified, there must be available slotted capacity on your Outpost to successfully create a local Amazon EKS cluster. The capacity varies based on whether you use the host or rack spread type. If there isn't enough capacity, the cluster remains in the `Creating` state. You are able to check the `Insufficient Capacity Error` on the health field of the link:eks/latest/APIReference/API_DescribeCluster.html[DescribeCluster,type="documentation"] API response. You must free capacity for the creation process to progress. -* During Amazon EKS local cluster platform and version updates, the [.noloc]`Kubernetes` control plane instances from your cluster are replaced by new instances using a rolling update strategy. During this replacement process, each control plane instance is terminated, freeing up its respective slot. A new updated instance is provisioned in its place. The updated instance might be placed in the slot that was released. If the slot is consumed by another unrelated instance and there is no more capacity left that respects the required spread topology requirement, then the cluster remains in the `Updating` state. You are able to see the respective `Insufficient Capacity Error` on the health field of the link:eks/latest/APIReference/API_DescribeCluster.html[DescribeCluster,type="documentation"] API response. You must free capacity so the update process can progress and reestablish prior high availability levels. -* You can create a maximum of 500 placement groups per account in each {aws} Region. For more information, see link:AWSEC2/latest/UserGuide/placement-groups.html#placement-groups-limitations-general[General rules and limitations,type="documentation"] in the Amazon EC2 User Guide. +* During Amazon EKS local cluster platform and version updates, the Kubernetes control plane instances from your cluster are replaced by new instances using a rolling update strategy. During this replacement process, each control plane instance is terminated, freeing up its respective slot. A new updated instance is provisioned in its place. The updated instance might be placed in the slot that was released. If the slot is consumed by another unrelated instance and there is no more capacity left that respects the required spread topology requirement, then the cluster remains in the `Updating` state. You are able to see the respective `Insufficient Capacity Error` on the health field of the link:eks/latest/APIReference/API_DescribeCluster.html[DescribeCluster,type="documentation"] API response. You must free capacity so the update process can progress and reestablish prior high availability levels. +* You can create a maximum of 500 placement groups per account in each {aws} Region. For more information, see link:AWSEC2/latest/UserGuide/placement-groups.html#placement-groups-limitations-general[General rules and limitations,type="documentation"] in the Amazon EC2 User Guide. \ No newline at end of file diff --git a/latest/ug/outposts/eks-outposts-local-cluster-create.adoc b/latest/ug/outposts/eks-outposts-local-cluster-create.adoc index fe3aeef13..6465b41ec 100644 --- a/latest/ug/outposts/eks-outposts-local-cluster-create.adoc +++ b/latest/ug/outposts/eks-outposts-local-cluster-create.adoc @@ -1,8 +1,7 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[eks-outposts-local-cluster-create,eks-outposts-local-cluster-create.title]] +[#eks-outposts-local-cluster-create] = Deploy an Amazon EKS cluster on {aws} Outposts :info_titleabbrev: Deploy a local cluster @@ -22,22 +21,21 @@ This topic provides an overview of what to consider when running a local cluster ==== * Local clusters support Outpost racks only. A single local cluster can run across multiple physical Outpost racks that comprise a single logical Outpost. A single local cluster can't run across multiple logical Outposts. Each logical Outpost has a single Outpost ARN. -* Local clusters run and manage the [.noloc]`Kubernetes` control plane in your account on the Outpost. You can't run workloads on the [.noloc]`Kubernetes` control plane instances or modify the [.noloc]`Kubernetes` control plane components. These nodes are managed by the Amazon EKS service. Changes to the [.noloc]`Kubernetes` control plane don't persist through automatic Amazon EKS management actions, such as patching. +* Local clusters run and manage the Kubernetes control plane in your account on the Outpost. You can't run workloads on the Kubernetes control plane instances or modify the Kubernetes control plane components. These nodes are managed by the Amazon EKS service. Changes to the Kubernetes control plane don't persist through automatic Amazon EKS management actions, such as patching. * Local clusters support self-managed add-ons and self-managed Amazon Linux node groups. The <>, <>, and <> add-ons are automatically installed on local clusters. -* Local clusters require the use of Amazon EBS on Outposts. Your Outpost must have Amazon EBS available for the [.noloc]`Kubernetes` control plane storage. -* Local clusters use Amazon EBS on Outposts. Your Outpost must have Amazon EBS available for the [.noloc]`Kubernetes` control plane storage. Outposts support Amazon EBS `gp2` volumes only. -* Amazon EBS backed [.noloc]`Kubernetes` `PersistentVolumes` are supported using the Amazon EBS CSI driver. +* Local clusters require the use of Amazon EBS on Outposts. Your Outpost must have Amazon EBS available for the Kubernetes control plane storage. Outposts support Amazon EBS `gp2` volumes only. +* Amazon EBS backed Kubernetes `PersistentVolumes` are supported using the Amazon EBS CSI driver. * The control plane instances of local clusters are set up in https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/ha-topology/[stacked highly available topology]. Two out of the three control plane instances must be healthy at all times to maintain quorum. If quorum is lost, contact {aws} support, as some service-side actions will be required to enable the new managed instances. -**Prerequisites** +*Prerequisites* * Familiarity with the <>, <>, and <>. * An existing Outpost. For more information, see link:outposts/latest/userguide/what-is-outposts.html[What is {aws} Outposts,type="documentation"]. -* The `kubectl` command line tool is installed on your computer or {aws} CloudShell. The version can be the same as or up to one minor version earlier or later than the [.noloc]`Kubernetes` version of your cluster. For example, if your cluster version is `1.29`, you can use `kubectl` version `1.28`, `1.29`, or `1.30` with it. To install or upgrade `kubectl`, see <>. -* Version `2.12.3` or later or version `1.27.160` or later of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device or {aws} CloudShell. To check your current version, use `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or [.noloc]`Homebrew` for [.noloc]`macOS` are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the _{aws} Command Line Interface User Guide_. The {aws} CLI version that is installed in {aws} CloudShell might also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the _{aws} CloudShell User Guide_. +* The `kubectl` command line tool is installed on your computer or {aws} CloudShell. The version can be the same as or up to one minor version earlier or later than the Kubernetes version of your cluster. For example, if your cluster version is `1.29`, you can use `kubectl` version `1.28`, `1.29`, or `1.30` with it. To install or upgrade `kubectl`, see <>. +* Version `2.12.3` or later or version `1.27.160` or later of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device or {aws} CloudShell. To check your current version, use `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or Homebrew for macOS are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the _{aws} Command Line Interface User Guide_. The {aws} CLI version that is installed in {aws} CloudShell might also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the _{aws} CloudShell User Guide_. * An IAM principal (user or role) with permissions to `create` and `describe` an Amazon EKS cluster. For more information, see <> and <>. -When a local Amazon EKS cluster is created, the link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] that creates the cluster is permanently added. The principal is specifically added to the [.noloc]`Kubernetes` RBAC authorization table as the administrator. This entity has `system:masters` permissions. The identity of this entity isn't visible in your cluster configuration. So, it's important to note the entity that created the cluster and make sure that you never delete it. Initially, only the principal that created the server can make calls to the [.noloc]`Kubernetes` API server using `kubectl`. If you use the console to create the cluster, make sure that the same IAM credentials are in the {aws} SDK credential chain when you run `kubectl` commands on your cluster. After your cluster is created, you can grant other IAM principals access to your cluster. +When a local Amazon EKS cluster is created, the link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] that creates the cluster is permanently added. The principal is specifically added to the Kubernetes RBAC authorization table as the administrator. This entity has `system:masters` permissions. The identity of this entity isn't visible in your cluster configuration. So, it's important to note the entity that created the cluster and make sure that you never delete it. Initially, only the principal that created the server can make calls to the Kubernetes API server using `kubectl`. If you use the console to create the cluster, make sure that the same IAM credentials are in the {aws} SDK credential chain when you run `kubectl` commands on your cluster. After your cluster is created, you can grant other IAM principals access to your cluster. == Create an Amazon EKS local cluster You can create a local cluster with the following tools described in this page: @@ -70,7 +68,7 @@ kind: ClusterConfig metadata: name: my-cluster region: region-code - version: "1.24" + version: "{k8s-n}" vpc: clusterEndpoints: @@ -104,10 +102,10 @@ Cluster provisioning takes several minutes. While the cluster is being created, + [TIP] ==== -To see the most options that you can specify when creating a cluster with `eksctl`, use the `eksctl create cluster --help` command. To see all the available options, you can use a `config` file. For more information, see https://eksctl.io/usage/creating-and-managing-clusters/#using-config-files[Using config files] and the https://eksctl.io/usage/schema/[config file schema] in the `eksctl` documentation. You can find https://github.com/weaveworks/eksctl/tree/master/examples[config file examples] on [.noloc]`GitHub`. +To see the most options that you can specify when creating a cluster with `eksctl`, use the `eksctl create cluster --help` command. To see all the available options, you can use a `config` file. For more information, see https://eksctl.io/usage/creating-and-managing-clusters/#using-config-files[Using config files] and the https://eksctl.io/usage/schema/[config file schema] in the `eksctl` documentation. You can find https://github.com/weaveworks/eksctl/tree/master/examples[config file examples] on GitHub. ==== + -The `eksctl` command automatically created an <> for the IAM principal (user or role) that created the cluster and granted the IAM principal administrator permissions to [.noloc]`Kubernetes` objects on the cluster. If you don't want the cluster creator to have administrator access to [.noloc]`Kubernetes` objects on the cluster, add the following text to the previous configuration file: `bootstrapClusterCreatorAdminPermissions: false` (at the same level as `metadata`, `vpc`, and `outpost`). If you added the option, then after cluster creation, you need to create an access entry for at least one IAM principal, or no IAM principals will have access to [.noloc]`Kubernetes` objects on the cluster. +The `eksctl` command automatically created an <> for the IAM principal (user or role) that created the cluster and granted the IAM principal administrator permissions to Kubernetes objects on the cluster. If you don't want the cluster creator to have administrator access to Kubernetes objects on the cluster, add the following text to the previous configuration file: `bootstrapClusterCreatorAdminPermissions: false` (at the same level as `metadata`, `vpc`, and `outpost`). If you added the option, then after cluster creation, you need to create an access entry for at least one IAM principal, or no IAM principals will have access to Kubernetes objects on the cluster. === {aws-management-console} [[console_create_cluster_outpost]] @@ -119,28 +117,15 @@ The `eksctl` command automatically created an <> fo + [source,json,subs="verbatim,attributes"] ---- -cat >eks-local-cluster-role-trust-policy.json <>. -. Choose *Add cluster* and then choose *Create*. +. Choose *Add cluster* and then choose *Create*. . On the *Configure cluster* page, enter or select values for the following fields: + -* *[.noloc]`Kubernetes` control plane location* – Choose {aws} Outposts. +* *Kubernetes control plane location* – Choose {aws} Outposts. * *Outpost ID* – Choose the ID of the Outpost that you want to create your control plane on. * *Instance type* – Select an instance type. Only the instance types available in your Outpost are displayed. In the dropdown list, each instance type describes how many nodes the instance type is recommended for. Before choosing an instance type, see <>. All replicas are deployed using the same instance type. You can't change the instance type after your cluster is created. Three control plane instances are deployed. You can't change this number. * *Name* – A name for your cluster. It must be unique in your {aws} account. The name can contain only alphanumeric characters (case-sensitive) and hyphens. It must start with an alphanumeric character and can't be longer than 100 characters. The name must be unique within the {aws} Region and {aws} account that you're creating the cluster in. The name must be unique within the {aws} Region and {aws} account that you're creating the cluster in. -* *[.noloc]`Kubernetes` version* – Choose the [.noloc]`Kubernetes` version that you want to use for your cluster. We recommend selecting the latest version, unless you need to use an earlier version. -* *Cluster service role* – Choose the Amazon EKS cluster IAM role that you created in a previous step to allow the [.noloc]`Kubernetes` control plane to manage {aws} resources. -* *[.noloc]`Kubernetes` cluster administrator access* – If you want the IAM principal (role or user) that's creating the cluster to have administrator access to the [.noloc]`Kubernetes` objects on the cluster, accept the default (allow). Amazon EKS creates an access entry for the IAM principal and grants cluster administrator permissions to the access entry. For more information about access entries, see <>. +* *Kubernetes version* – Choose the Kubernetes version that you want to use for your cluster. We recommend selecting the latest version, unless you need to use an earlier version. +* *Cluster service role* – Choose the Amazon EKS cluster IAM role that you created in a previous step to allow the Kubernetes control plane to manage {aws} resources. +* *Kubernetes cluster administrator access* – If you want the IAM principal (role or user) that's creating the cluster to have administrator access to the Kubernetes objects on the cluster, accept the default (allow). Amazon EKS creates an access entry for the IAM principal and grants cluster administrator permissions to the access entry. For more information about access entries, see <>. + -If you want a different IAM principal than the principal creating the cluster to have administrator access to [.noloc]`Kubernetes` cluster objects, choose the disallow option. After cluster creation, any IAM principal that has IAM permissions to create access entries can add an access entries for any IAM principals that need access to [.noloc]`Kubernetes` cluster objects. For more information about the required IAM permissions, see link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html#amazonelastickubernetesservice-actions-as-permissions[Actions defined by Amazon Elastic Kubernetes Service,type="documentation"] in the Service Authorization Reference. If you choose the disallow option and don't create any access entries, then no IAM principals will have access to the [.noloc]`Kubernetes` objects on the cluster. -* *Tags* – (Optional) Add any tags to your cluster. For more information, see <>. When you're done with this page, choose *Next*. +If you want a different IAM principal than the principal creating the cluster to have administrator access to Kubernetes cluster objects, choose the disallow option. After cluster creation, any IAM principal that has IAM permissions to create access entries can add an access entries for any IAM principals that need access to Kubernetes cluster objects. For more information about the required IAM permissions, see link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html#amazonelastickubernetesservice-actions-as-permissions[Actions defined by Amazon Elastic Kubernetes Service,type="documentation"] in the Service Authorization Reference. If you choose the disallow option and don't create any access entries, then no IAM principals will have access to the Kubernetes objects on the cluster. +* *Tags* – (Optional) Add any tags to your cluster. For more information, see <>. When you're done with this page, choose *Next*. . On the *Specify networking* page, select values for the following fields: + -* *VPC* – Choose an existing VPC. The VPC must have a sufficient number of IP addresses available for the cluster, any nodes, and other [.noloc]`Kubernetes` resources that you want to create. Your VPC must meet the requirements in <>. -* *Subnets* – By default, all available subnets in the VPC specified in the previous field are preselected. The subnets that you choose must meet the requirements in <>. +* *VPC* – Choose an existing VPC. The VPC must have a sufficient number of IP addresses available for the cluster, any nodes, and other Kubernetes resources that you want to create. Your VPC must meet the requirements in <>. +* *Subnets* – By default, all available subnets in the VPC specified in the previous field are preselected. The subnets that you choose must meet the requirements in <>. * *Security groups* – (Optional) Specify one or more security groups that you want Amazon EKS to associate to the network interfaces that it creates. Amazon EKS automatically creates a security group that enables communication between your cluster and your VPC. Amazon EKS associates this security group, and any that you choose, to the network interfaces that it creates. For more information about the cluster security group that Amazon EKS creates, see <>. You can modify the rules in the cluster security group that Amazon EKS creates. If you choose to add your own security groups, you can't change the ones that you choose after cluster creation. For on-premises hosts to communicate with the cluster endpoint, you must allow inbound traffic from the cluster security group. For clusters that don't have an ingress and egress internet connection (also knows as private clusters), you must do one of the following: + ** Add the security group associated with required VPC endpoints. For more information about the required endpoints, see <> in <>. + -** Modify the security group that Amazon EKS created to allow traffic from the security group associated with the VPC endpoints. When you're done with this page, choose *Next*. -. On the *Configure observability* page, you can optionally choose which *Metrics* and *Control plane logging* options that you want to turn on. By default, each log type is turned off. +** Modify the security group that Amazon EKS created to allow traffic from the security group associated with the VPC endpoints. When you're done with this page, choose *Next*. +. On the *Configure observability* page, you can optionally choose which *Metrics* and *Control plane logging* options that you want to turn on. By default, each log type is turned off. + -**** For more information on the [.noloc]`Prometheus` metrics option, see <>. -**** For more information on the *Control plane logging* options, see <>. When you're done with this page, choose *Next*. -. On the *Review and create* page, review the information that you entered or selected on the previous pages. If you need to make changes, choose *Edit*. When you're satisfied, choose *Create*. The *Status* field shows *CREATING* while the cluster is provisioned. +**** For more information on the Prometheus metrics option, see <>. +**** For more information on the *Control plane logging* options, see <>. When you're done with this page, choose *Next*. +. On the *Review and create* page, review the information that you entered or selected on the previous pages. If you need to make changes, choose *Edit*. When you're satisfied, choose *Create*. The *Status* field shows *CREATING* while the cluster is provisioned. + Cluster provisioning takes several minutes. @@ -196,7 +181,7 @@ An example output is as follows. "Name": "my-cluster-control-plane-id3" ---- + -Each instance is tainted with `node-role.eks-local.amazonaws.com/control-plane` so that no workloads are ever scheduled on the control plane instances. For more information about taints, see https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/[Taints and Tolerations] in the [.noloc]`Kubernetes` documentation. Amazon EKS continuously monitors the state of local clusters. We perform automatic management actions, such as security patches and repairing unhealthy instances. When local clusters are disconnected from the cloud, we complete actions to ensure that the cluster is repaired to a healthy state upon reconnect. +Each instance is tainted with `node-role.eks-local.amazonaws.com/control-plane` so that no workloads are ever scheduled on the control plane instances. For more information about taints, see https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/[Taints and Tolerations] in the Kubernetes documentation. Amazon EKS continuously monitors the state of local clusters. We perform automatic management actions, such as security patches and repairing unhealthy instances. When local clusters are disconnected from the cloud, we complete actions to ensure that the cluster is repaired to a healthy state upon reconnect. . If you created your cluster using `eksctl`, then you can skip this step. `eksctl` completes this step for you. Enable `kubectl` to communicate with your cluster by adding a new context to the `kubectl` `config` file. For instructions on how to create and update the file, see <>. + [source,bash,subs="verbatim,attributes"] @@ -210,9 +195,9 @@ An example output is as follows. ---- Added new context {arn-aws}eks:region-code:111122223333:cluster/my-cluster to /home/username/.kube/config ---- -. To connect to your local cluster's [.noloc]`Kubernetes` API server, have access to the local gateway for the subnet, or connect from within the VPC. For more information about connecting an Outpost rack to your on-premises network, see link:outposts/latest/userguide/how-racks-work.html[How local gateways for racks work,type="documentation"] in the {aws} Outposts User Guide. If you use Direct VPC Routing and the Outpost subnet has a route to your local gateway, the private IP addresses of the [.noloc]`Kubernetes` control plane instances are automatically broadcasted over your local network. The local cluster's [.noloc]`Kubernetes` API server endpoint is hosted in Amazon Route 53 (Route 53). The API service endpoint can be resolved by public DNS servers to the Kubernetes API servers' private IP addresses. +. To connect to your local cluster's Kubernetes API server, have access to the local gateway for the subnet, or connect from within the VPC. For more information about connecting an Outpost rack to your on-premises network, see link:outposts/latest/userguide/how-racks-work.html[How local gateways for racks work,type="documentation"] in the {aws} Outposts User Guide. If you use Direct VPC Routing and the Outpost subnet has a route to your local gateway, the private IP addresses of the Kubernetes control plane instances are automatically broadcasted over your local network. The local cluster's Kubernetes API server endpoint is hosted in Amazon Route 53 (Route 53). The API service endpoint can be resolved by public DNS servers to the Kubernetes API servers' private IP addresses. + -Local clusters' [.noloc]`Kubernetes` control plane instances are configured with static elastic network interfaces with fixed private IP addresses that don't change throughout the cluster lifecycle. Machines that interact with the [.noloc]`Kubernetes` API server might not have connectivity to Route 53 during network disconnects. If this is the case, we recommend configuring `/etc/hosts` with the static private IP addresses for continued operations. We also recommend setting up local DNS servers and connecting them to your Outpost. For more information, see the link:outposts/latest/userguide/how-outposts-works.html#dns[{aws} Outposts documentation,type="documentation"]. Run the following command to confirm that communication's established with your cluster. +Local clusters' Kubernetes control plane instances are configured with static elastic network interfaces with fixed private IP addresses that don't change throughout the cluster lifecycle. Machines that interact with the Kubernetes API server might not have connectivity to Route 53 during network disconnects. If this is the case, we recommend configuring `/etc/hosts` with the static private IP addresses for continued operations. We also recommend setting up local DNS servers and connecting them to your Outpost. For more information, see the link:outposts/latest/userguide/how-outposts-works.html#dns[{aws} Outposts documentation,type="documentation"]. Run the following command to confirm that communication's established with your cluster. + [source,bash,subs="verbatim,attributes"] ---- @@ -229,7 +214,7 @@ kubernetes ClusterIP 10.100.0.1 443/TCP 28h . (Optional) Test authentication to your local cluster when it's in a disconnected state from the {aws} Cloud. For instructions, see <>. -[[outposts-control-plan-internal-resources,outposts-control-plan-internal-resources.title]] +[#outposts-control-plan-internal-resources] === Internal resources Amazon EKS creates the following resources on your cluster. The resources are for Amazon EKS internal use. For proper functioning of your cluster, don't edit or modify these resources. @@ -248,7 +233,7 @@ Amazon EKS creates the following resources on your cluster. The resources are fo + ** `kube-system/coredns` ** `kube-system/` `kube-proxy` (not created until you add your first node) -** `kube-system/aws-node` (not created until you add your first node). Local clusters use the [.noloc]`Amazon VPC CNI plugin for Kubernetes` plugin for cluster networking. Do not change the configuration for control plane instances (Pods named `aws-node-controlplane-*`). There are configuration variables that you can use to change the default value for when the plugin creates new network interfaces. For more information, see the https://github.com/aws/amazon-vpc-cni-k8s/blob/master/README.md[documentation] on GitHub. +** `kube-system/aws-node` (not created until you add your first node). Local clusters use the Amazon VPC CNI plugin for Kubernetes plugin for cluster networking. Do not change the configuration for control plane instances (Pods named `aws-node-controlplane-*`). There are configuration variables that you can use to change the default value for when the plugin creates new network interfaces. For more information, see the https://github.com/aws/amazon-vpc-cni-k8s/blob/master/README.md[documentation] on GitHub. * The following services: + ** `default/kubernetes` @@ -256,14 +241,13 @@ Amazon EKS creates the following resources on your cluster. The resources are fo * A `PodSecurityPolicy` named `eks.system` * A `ClusterRole` named `eks:system:podsecuritypolicy` * A `ClusterRoleBinding` named `eks:system` -* A default <> -* In addition to the <>, Amazon EKS creates a security group in your {aws} account that's named `eks-local-internal-do-not-use-or-edit-[.replaceable]``cluster-name``-[.replaceable]``uniqueid```. This security group allows traffic to flow freely between [.noloc]`Kubernetes` components running on the control plane instances. +* In addition to the <>, Amazon EKS creates a security group in your {aws} account that's named `eks-local-internal-do-not-use-or-edit-[.replaceable]``cluster-name``-[.replaceable]``uniqueid```. This security group allows traffic to flow freely between Kubernetes components running on the control plane instances. Recommended next steps: * <> -* <>. If you want the entities to view [.noloc]`Kubernetes` resources in the Amazon EKS console, grant the <> to the entities. +* <>. If you want the entities to view Kubernetes resources in the Amazon EKS console, grant the <> to the entities. * <> * Familiarize yourself with what happens during <>. * <> -* Consider setting up a backup plan for your `etcd`. Amazon EKS doesn't support automated backup and restore of `etcd` for local clusters. For more information, see https://kubernetes.io/docs/tasks/administer-cluster/configure-upgrade-etcd/#backing-up-an-etcd-cluster[Backing up an etcd cluster] in the [.noloc]`Kubernetes` documentation. The two main options are using `etcdctl` to automate taking snapshots or using Amazon EBS storage volume backup. +* Consider setting up a backup plan for your `etcd`. Amazon EKS doesn't support automated backup and restore of `etcd` for local clusters. For more information, see https://kubernetes.io/docs/tasks/administer-cluster/configure-upgrade-etcd/#backing-up-an-etcd-cluster[Backing up an etcd cluster] in the Kubernetes documentation. The two main options are using `etcdctl` to automate taking snapshots or using Amazon EBS storage volume backup. diff --git a/latest/ug/outposts/eks-outposts-local-cluster-overview.adoc b/latest/ug/outposts/eks-outposts-local-cluster-overview.adoc index 5ebe6411e..045bc2b9f 100644 --- a/latest/ug/outposts/eks-outposts-local-cluster-overview.adoc +++ b/latest/ug/outposts/eks-outposts-local-cluster-overview.adoc @@ -1,29 +1,16 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[eks-outposts-local-cluster-overview,eks-outposts-local-cluster-overview.title]] +[#eks-outposts-local-cluster-overview] = Create local Amazon EKS clusters on {aws} Outposts for high availability :info_titleabbrev: Run local clusters -include::eks-outposts-local-cluster-create.adoc[leveloffset=+1] - -include::eks-outposts-platform-versions.adoc[leveloffset=+1] - -include::eks-outposts-vpc-subnet-requirements.adoc[leveloffset=+1] - -include::eks-outposts-network-disconnects.adoc[leveloffset=+1] - -include::eks-outposts-capacity-considerations.adoc[leveloffset=+1] - -include::eks-outposts-troubleshooting.adoc[leveloffset=+1] - [abstract] -- Learn to create and manage local Amazon EKS clusters on {aws} Outposts for high availability across multiple regions. -- -You can use local clusters to run your entire Amazon EKS cluster locally on {aws} Outposts. This helps mitigate the risk of application downtime that might result from temporary network disconnects to the cloud. These disconnects can be caused by fiber cuts or weather events. Because the entire [.noloc]`Kubernetes` cluster runs locally on Outposts, applications remain available. You can perform cluster operations during network disconnects to the cloud. For more information, see <>. The following diagram shows a local cluster deployment. +You can use local clusters to run your entire Amazon EKS cluster locally on {aws} Outposts. This helps mitigate the risk of application downtime that might result from temporary network disconnects to the cloud. These disconnects can be caused by fiber cuts or weather events. Because the entire Kubernetes cluster runs locally on Outposts, applications remain available. You can perform cluster operations during network disconnects to the cloud. For more information, see <>. The following diagram shows a local cluster deployment. @@ -31,10 +18,23 @@ image::images/outposts-local-cluster.png[Outpost local cluster,scaledwidth=100%] Local clusters are generally available for use with Outposts racks. -[[outposts-control-plane-supported-regions,outposts-control-plane-supported-regions.title]] +[#outposts-control-plane-supported-regions] == Supported {aws} Regions You can create local clusters in the following {aws} Regions: US East (Ohio), US East (N. Virginia), US West (N. California), US West (Oregon), Asia Pacific (Seoul), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), Europe (London), Middle East (Bahrain), and South America (São Paulo). For detailed information about supported features, see <>. [.topiclist] [[Topic List]] + +include::eks-outposts-local-cluster-create.adoc[leveloffset=+1] + +include::eks-outposts-platform-versions.adoc[leveloffset=+1] + +include::eks-outposts-vpc-subnet-requirements.adoc[leveloffset=+1] + +include::eks-outposts-network-disconnects.adoc[leveloffset=+1] + +include::eks-outposts-capacity-considerations.adoc[leveloffset=+1] + +include::eks-outposts-troubleshooting.adoc[leveloffset=+1] + diff --git a/latest/ug/outposts/eks-outposts-network-disconnects.adoc b/latest/ug/outposts/eks-outposts-network-disconnects.adoc index 330c3cf1b..d6992a9dd 100644 --- a/latest/ug/outposts/eks-outposts-network-disconnects.adoc +++ b/latest/ug/outposts/eks-outposts-network-disconnects.adoc @@ -1,10 +1,9 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[eks-outposts-network-disconnects,eks-outposts-network-disconnects.title]] +[#eks-outposts-network-disconnects] = Prepare local Amazon EKS clusters on {aws} Outposts for network disconnects -:info_titleabbrev: Prepare for network disconnects +:info_titleabbrev: Prepare for disconnects [abstract] -- @@ -16,20 +15,20 @@ If your local network has lost connectivity with the {aws} Cloud, you can contin * Local clusters enable stability and continued operations during temporary, unplanned network disconnects. {aws} Outposts remains a fully connected offering that acts as an extension of the {aws} Cloud in your data center. In the event of network disconnects between your Outpost and {aws} Cloud, we recommend attempting to restore your connection. For instruction, see link:outposts/latest/userguide/network-troubleshoot.html[{aws} Outposts rack network troubleshooting checklist,type="documentation"] in the _{aws} Outposts User Guide_. For more information about how to troubleshoot issues with local clusters, see <>. -* Outposts emit a `ConnectedStatus` metric that you can use to monitor the connectivity state of your Outpost. For more information, see link:outposts/latest/userguide/outposts-cloudwatch-metrics.html#outposts-metrics[Outposts Metrics,type="documentation"] in the _{aws} Outposts User Guide_. +* Outposts emit a `ConnectedStatus` metric that you can use to monitor the connectivity state of your Outpost. For more information, see link:outposts/latest/userguide/outposts-cloudwatch-metrics.html#outposts-metrics[Outposts Metrics,type="documentation"] in the _{aws} Outposts User Guide_. * Local clusters use IAM as the default authentication mechanism using the https://github.com/kubernetes-sigs/aws-iam-authenticator[{aws} Identity and Access Management authenticator for Kubernetes]. IAM isn't available during network disconnects. So, local clusters support an alternative authentication mechanism using `x.509` certificates that you can use to connect to your cluster during network disconnects. For information about how to obtain and use an `x.509` certificate for your cluster, see <>. -* If you can't access Route 53 during network disconnects, consider using local DNS servers in your on-premises environment. The [.noloc]`Kubernetes` control plane instances use static IP addresses. You can configure the hosts that you use to connect to your cluster with the endpoint hostname and IP addresses as an alternative to using local DNS servers. For more information, see link:outposts/latest/userguide/how-outposts-works.html#dns[DNS,type="documentation"] in the _{aws} Outposts User Guide_. +* If you can't access Route 53 during network disconnects, consider using local DNS servers in your on-premises environment. The Kubernetes control plane instances use static IP addresses. You can configure the hosts that you use to connect to your cluster with the endpoint hostname and IP addresses as an alternative to using local DNS servers. For more information, see link:outposts/latest/userguide/how-outposts-works.html#dns[DNS,type="documentation"] in the _{aws} Outposts User Guide_. * If you expect increases in application traffic during network disconnects, you can provision spare compute capacity in your cluster when connected to the cloud. Amazon EC2 instances are included in the price of {aws} Outposts. So, running spare instances doesn't impact your {aws} usage cost. -* During network disconnects to enable create, update, and scale operations for workloads, your application's container images must be accessible over the local network and your cluster must have enough capacity. Local clusters don't host a container registry for you. If the [.noloc]`Pods` have previously run on those nodes, container images are cached on the nodes. If you typically pull your application's container images from Amazon ECR in the cloud, consider running a local cache or registry. A local cache or registry is helpful if you require create, update, and scale operations for workload resources during network disconnects. -* Local clusters use Amazon EBS as the default storage class for persistent volumes and the Amazon EBS CSI driver to manage the lifecycle of Amazon EBS persistent volumes. During network disconnects, [.noloc]`Pods` that are backed by Amazon EBS can't be created, updated, or scaled. This is because these operations require calls to the Amazon EBS API in the cloud. If you're deploying stateful workloads on local clusters and require create, update, or scale operations during network disconnects, consider using an alternative storage mechanism. +* During network disconnects to enable create, update, and scale operations for workloads, your application's container images must be accessible over the local network and your cluster must have enough capacity. Local clusters don't host a container registry for you. If the Pods have previously run on those nodes, container images are cached on the nodes. If you typically pull your application's container images from Amazon ECR in the cloud, consider running a local cache or registry. A local cache or registry is helpful if you require create, update, and scale operations for workload resources during network disconnects. +* Local clusters use Amazon EBS as the default storage class for persistent volumes and the Amazon EBS CSI driver to manage the lifecycle of Amazon EBS persistent volumes. During network disconnects, Pods that are backed by Amazon EBS can't be created, updated, or scaled. This is because these operations require calls to the Amazon EBS API in the cloud. If you're deploying stateful workloads on local clusters and require create, update, or scale operations during network disconnects, consider using an alternative storage mechanism. * Amazon EBS snapshots can't be created or deleted if {aws} Outposts can't access the relevant {aws} in-region APIs (such as the APIs for Amazon EBS or Amazon S3). * When integrating ALB (Ingress) with {aws} Certificate Manager (ACM), certificates are pushed and stored in memory of the {aws} Outposts ALB Compute instance. Current TLS termination will continue to operate in the event of a disconnect from the {aws} Region. Mutating operations in this context will fail (such as new ingress definitions, new ACM based certificates API operations, ALB compute scale, or certificate rotation). For more information, see link:acm/latest/userguide/troubleshooting-renewal.html[Troubleshooting managed certificate renewal,type="documentation"] in the _{aws} Certificate Manager User Guide_. -* The Amazon EKS control plane logs are cached locally on the [.noloc]`Kubernetes` control plane instances during network disconnects. Upon reconnect, the logs are sent to CloudWatch Logs in the parent {aws} Region. You can use https://prometheus.io/[Prometheus], https://grafana.com/[Grafana], or Amazon EKS partner solutions to monitor the cluster locally using the [.noloc]`Kubernetes` API server's metrics endpoint or using [.noloc]`Fluent Bit` for logs. -* If you're using the [.noloc]`{aws} Load Balancer Controller` on Outposts for application traffic, existing [.noloc]`Pods` fronted by the [.noloc]`{aws} Load Balancer Controller` continue to receive traffic during network disconnects. New [.noloc]`Pods` created during network disconnects don't receive traffic until the Outpost is reconnected to the {aws} Cloud. Consider setting the replica count for your applications while connected to the {aws} Cloud to accommodate your scaling needs during network disconnects. -* The [.noloc]`Amazon VPC CNI plugin for Kubernetes` defaults to https://aws.github.io/aws-eks-best-practices/networking/vpc-cni/#overview[secondary IP mode]. It's configured with `WARM_ENI_TARGET`=``1``, which allows the plugin to keep "a full elastic network interface" of available IP addresses available. Consider changing `WARM_ENI_TARGET`, `WARM_IP_TARGET`, and `MINIMUM_IP_TARGET` values according to your scaling needs during a disconnected state. For more information, see the https://github.com/aws/amazon-vpc-cni-k8s/blob/master/README.md[readme] file for the plugin on GitHub. For a list of the maximum number of [.noloc]`Pods` that's supported by each instance type, see the https://github.com/aws/amazon-vpc-cni-k8s/blob/master/misc/eni-max-pods.txt[eni-max-pods.txt] file on GitHub. +* The Amazon EKS control plane logs are cached locally on the Kubernetes control plane instances during network disconnects. Upon reconnect, the logs are sent to CloudWatch Logs in the parent {aws} Region. You can use https://prometheus.io/[Prometheus], https://grafana.com/[Grafana], or Amazon EKS partner solutions to monitor the cluster locally using the Kubernetes API server's metrics endpoint or using Fluent Bit for logs. +* If you're using the {aws} Load Balancer Controller on Outposts for application traffic, existing Pods fronted by the {aws} Load Balancer Controller continue to receive traffic during network disconnects. New Pods created during network disconnects don't receive traffic until the Outpost is reconnected to the {aws} Cloud. Consider setting the replica count for your applications while connected to the {aws} Cloud to accommodate your scaling needs during network disconnects. +* The Amazon VPC CNI plugin for Kubernetes defaults to https://aws.github.io/aws-eks-best-practices/networking/vpc-cni/#overview[secondary IP mode]. It's configured with `WARM_ENI_TARGET`=``1``, which allows the plugin to keep "a full elastic network interface" of available IP addresses available. Consider changing `WARM_ENI_TARGET`, `WARM_IP_TARGET`, and `MINIMUM_IP_TARGET` values according to your scaling needs during a disconnected state. For more information, see the https://github.com/aws/amazon-vpc-cni-k8s/blob/master/README.md[readme] file for the plugin on GitHub. For a list of the maximum number of Pods that's supported by each instance type, see the https://github.com/aws/amazon-vpc-cni-k8s/blob/master/misc/eni-max-pods.txt[eni-max-pods.txt] file on GitHub. -[[outposts-network-disconnects-authentication,outposts-network-disconnects-authentication.title]] +[#outposts-network-disconnects-authentication] == Authenticating to your local cluster during a network disconnect [abstract] @@ -48,7 +47,7 @@ Learn how to work with your cluster during a network disconnect. openssl req -new -newkey rsa:4096 -nodes -days 365 \ -keyout admin.key -out admin.csr -subj "/CN=admin" ---- -.. Create a certificate signing request in [.noloc]`Kubernetes`. +.. Create a certificate signing request in Kubernetes. + [source,bash,subs="verbatim,attributes"] ---- @@ -86,7 +85,7 @@ NAME AGE REQUESTOR CONDITION admin-csr 11m kubernetes-admin Pending ---- + -[.noloc]`Kubernetes` created the certificate signing request. +Kubernetes created the certificate signing request. . Approve the certificate signing request. + [source,bash,subs="verbatim,attributes"] @@ -178,4 +177,4 @@ kubectl config use-context admin@my-cluster --kubeconfig admin.kubeconfig ---- + -If you notice any issues with your local clusters while they're in a disconnected state, we recommend opening a support ticket. +If you notice any issues with your local clusters while they're in a disconnected state, we recommend opening a support ticket. \ No newline at end of file diff --git a/latest/ug/outposts/eks-outposts-platform-versions.adoc b/latest/ug/outposts/eks-outposts-platform-versions.adoc index 2211b5daf..1c47edad6 100644 --- a/latest/ug/outposts/eks-outposts-platform-versions.adoc +++ b/latest/ug/outposts/eks-outposts-platform-versions.adoc @@ -1,302 +1,282 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[eks-outposts-platform-versions,eks-outposts-platform-versions.title]] -= Learn [.noloc]`Kubernetes` and Amazon EKS platform versions for {aws} Outposts -:info_titleabbrev: Learn Kubernetes platform versions +[#eks-outposts-platform-versions] += Learn Kubernetes and Amazon EKS platform versions for {aws} Outposts +:info_titleabbrev: EKS platform versions [abstract] -- -Learn the relationship between Amazon EKS and [.noloc]`Kubernetes` versions available on {aws} Outposts. +Learn the relationship between Amazon EKS and Kubernetes versions available on {aws} Outposts. -- -Local cluster platform versions represent the capabilities of the Amazon EKS cluster on {aws} Outposts. The versions include the components that run on the [.noloc]`Kubernetes` control plane, which [.noloc]`Kubernetes` API server flags are enabled. They also include the current [.noloc]`Kubernetes` patch version. Each [.noloc]`Kubernetes` minor version has one or more associated platform versions. The platform versions for different [.noloc]`Kubernetes` minor versions are independent. The platform versions for local clusters and Amazon EKS clusters in the cloud are independent. +Local cluster platform versions represent the capabilities of the Amazon EKS cluster on {aws} Outposts. The versions include the components that run on the Kubernetes control plane, which Kubernetes API server flags are enabled. They also include the current Kubernetes patch version. Each Kubernetes minor version has one or more associated platform versions. The platform versions for different Kubernetes minor versions are independent. The platform versions for local clusters and Amazon EKS clusters in the cloud are independent. -When a new [.noloc]`Kubernetes` minor version is available for local clusters, such as `1.30`, the initial platform version for that [.noloc]`Kubernetes` minor version starts at `eks-local-outposts.1`. However, Amazon EKS releases new platform versions periodically to enable new [.noloc]`Kubernetes` control plane settings and to provide security fixes. +When a new Kubernetes minor version is available for local clusters, such as `1.31`, the initial platform version for that Kubernetes minor version starts at `eks-local-outposts.1`. However, Amazon EKS releases new platform versions periodically to enable new Kubernetes control plane settings and to provide security fixes. When new local cluster platform versions become available for a minor version: * The platform version number is incremented (`eks-local-outposts.n+1`). -* Amazon EKS automatically updates all existing local clusters to the latest platform version for their corresponding [.noloc]`Kubernetes` minor version. Automatic updates of existing platform versions are rolled out incrementally. The roll-out process might take some time. If you need the latest platform version features immediately, we recommend that you create a new local cluster. -* Amazon EKS might publish a new node AMI with a corresponding patch version. All patch versions are compatible between the [.noloc]`Kubernetes` control plane and node AMIs for a single [.noloc]`Kubernetes` minor version. +* Amazon EKS automatically updates all existing local clusters to the latest platform version for their corresponding Kubernetes minor version. Automatic updates of existing platform versions are rolled out incrementally. The roll-out process consists of the replacement of the managed Kubernetes control-plane instances running on the Outpost, one at a time, until all 3 instances get replaced by new ones. +* The Kubernetes control-plane instance replacement process will stop progressing if there is risk of service interruption. Amazon EKS will only attempt to replace an instance in case the other 2 Kubernetes control-plane instances are healthy and passing all readiness conditions as a cluster node. +* A platform version rollout will typically take less than 30 minutes to complete. If a cluster remains on `UPDATING` state for an extended amount of time, see the <> and seek help from {aws} Support. Never manually terminate Kubernetes control-plane instances unless instructed by {aws} Support. +* Amazon EKS might publish a new node AMI with a corresponding patch version. All patch versions are compatible between the Kubernetes control plane and node AMIs for a single Kubernetes minor version. New platform versions don't introduce breaking changes or cause service interruptions. -Local clusters are always created with the latest available platform version (`eks-local-outposts.n`) for the specified [.noloc]`Kubernetes` version. +Local clusters are always created with the latest available platform version (`eks-local-outposts.n`) for the specified Kubernetes version. The current and recent platform versions are described in the following tables. -[[outposts-platform-versions-1.30,outposts-platform-versions-1.30.title]] -== [.noloc]`Kubernetes` version `1.30` +To receive notifications of all source file changes to this specific documentation page, you can subscribe to the following URL with an RSS reader: -The following admission controllers are enabled for all `1.30` platform versions: `CertificateApproval`, `CertificateSigning`, `CertificateSubjectRestriction`, `DefaultIngressClass`, `DefaultStorageClass`, `DefaultTolerationSeconds`, `ExtendedResourceToleration`, `LimitRanger`, `MutatingAdmissionWebhook`, `NamespaceLifecycle`, `NodeRestriction`, `PersistentVolumeClaimResize`, `Priority`, `PodSecurity`, `ResourceQuota`, `RuntimeClass`, `ServiceAccount`, `StorageObjectInUseProtection`, `TaintNodesByCondition`, `ValidatingAdmissionPolicy`, and `ValidatingAdmissionWebhook`. +[source,none] +---- +https://github.com/awsdocs/amazon-eks-user-guide/commits/mainline/latest/ug/outposts/eks-outposts-platform-versions.adoc.atom +---- -[cols="1,1,1,1", options="header"] -|=== -|Kubernetes version -|Amazon EKS platform version -|Release notes -|Release date +[#outposts-platform-versions-1-31] +== Kubernetes version `1.31` -|`1.30.5` -|`eks-local-outposts.1` -|Initial release of Kubernetes version `v1.30`. for local Amazon EKS clusters on Outpost -|November 13, 2024 -|=== - -[[outposts-platform-versions-1.29,outposts-platform-versions-1.29.title]] -== [.noloc]`Kubernetes` version `1.29` +The following admission controllers are enabled for all `1.31` platform versions: `CertificateApproval`, `CertificateSigning`, `CertificateSubjectRestriction`, `ClusterTrustBundleAttest`, `DefaultIngressClass`, `DefaultStorageClass`, `DefaultTolerationSeconds`, `ExtendedResourceToleration`, `LimitRanger`, `MutatingAdmissionWebhook`, `NamespaceLifecycle`, `NodeRestriction`, `PersistentVolumeClaimResize`, `PodSecurity`, `Priority`, `ResourceQuota`, `RuntimeClass`, `ServiceAccount`, `StorageObjectInUseProtection`, `TaintNodesByCondition`, `ValidatingAdmissionPolicy`, and `ValidatingAdmissionWebhook`. -The following admission controllers are enabled for all `1.29` platform versions: `CertificateApproval`, `CertificateSigning`, `CertificateSubjectRestriction`, `DefaultIngressClass`, `DefaultStorageClass`, `DefaultTolerationSeconds`, `ExtendedResourceToleration`, `LimitRanger`, `MutatingAdmissionWebhook`, `NamespaceLifecycle`, `NodeRestriction`, `PersistentVolumeClaimResize`, `Priority`, `PodSecurity`, `ResourceQuota`, `RuntimeClass`, `ServiceAccount`, `StorageObjectInUseProtection`, `TaintNodesByCondition`, `ValidatingAdmissionPolicy`, and `ValidatingAdmissionWebhook`. -[cols="1,1,1,1", options="header"] +[%header,cols="4"] |=== |Kubernetes version |Amazon EKS platform version |Release notes |Release date - -|`1.29.6` -|`eks-local-outposts.1` -|Initial release of Kubernetes version `v1.29`. for local Amazon EKS clusters on Outpost -|August 20, 2024 -|=== - -[[outposts-platform-versions-1.28,outposts-platform-versions-1.28.title]] -== [.noloc]`Kubernetes` version `1.28` - -The following admission controllers are enabled for all `1.28` platform versions: `CertificateApproval`, `CertificateSigning`, `CertificateSubjectRestriction`, `DefaultIngressClass`, `DefaultStorageClass`, `DefaultTolerationSeconds`, `ExtendedResourceToleration`, `LimitRanger`, `MutatingAdmissionWebhook`, `NamespaceLifecycle`, `NodeRestriction`, `PersistentVolumeClaimResize`, `Priority`, `PodSecurity`, `ResourceQuota`, `RuntimeClass`, `ServiceAccount`, `StorageObjectInUseProtection`, `TaintNodesByCondition`, `ValidatingAdmissionPolicy`, and `ValidatingAdmissionWebhook`. - -[cols="1,1,1,1", options="header"] -|=== -|Kubernetes version -|Amazon EKS platform version -|Release notes -|Release date - - -|`1.28.6` +|`1.31.12` |`eks-local-outposts.5` -|Updated Bottlerocket version to v1.19.3 containing newest bugfixes to support local boot in Outposts. -|April 18, 2024 +|New platform version with security fixes and enhancements. kube-proxy updated to `v1.31.10`. {aws} IAM Authenticator updated to `v0.7.4`. Amazon VPC CNI plugin for Kubernetes updated to `v1.20.2`. Bottlerocket version updated to `v1.47.0`. +|October 3, 2025 -|`1.28.6` +|`1.31.9` |`eks-local-outposts.4` -|New platform version with security fixes and enhancements. Restored support or local boot in Outposts. Downgraded [.noloc]`Bottlerocket` version to `v1.15.1` for compatibility. -|April 2, 2024 +|New platform version with security fixes and enhancements. kube-proxy updated to `v1.31.9`. {aws} IAM Authenticator updated to `v0.7.2`. Amazon VPC CNI plugin for Kubernetes updated to `v1.20.0` Bottlerocket version updated to `v1.43.0`. +|August 9, 2025 -|`1.28.6` +|`1.31.7` |`eks-local-outposts.3` -|New platform version with security fixes and enhancements. -|March 22, 2024 +|New platform version with security fixes and enhancements. kube-proxy updated to `v1.31.9`. {aws} IAM Authenticator updated to `v0.7.1`. Amazon VPC CNI plugin for Kubernetes updated to `v1.19.5`. Bottlerocket version updated to `v1.40.0`. +|June 19, 2025 -|`1.28.6` +|`1.31.6` |`eks-local-outposts.2` -|New platform version with security fixes and enhancements kube-proxy updated to `v1.28.6`. {aws} IAM Authenticator updated to `v0.6.17`. Amazon VPC CNI plugin for Kubernetes downgraded to `v1.13.2` for compatibility reasons. Updated [.noloc]`Bottlerocket` version to `v1.19.2`. -|March 8, 2024 +|New platform version with security fixes and enhancements. Bottlerocket version updated to `v1.36.0`. +|April 24, 2025 -|`1.28.1` +|`1.31.6` |`eks-local-outposts.1` -|Initial release of Kubernetes version `v1.28`. for local Amazon EKS clusters on Outpost -|October 4, 2023 +|Initial release of Kubernetes version `v1.31` for local Amazon EKS clusters on Outposts. +|April 9, 2025 |=== -[[outposts-platform-versions-1.27,outposts-platform-versions-1.27.title]] -== [.noloc]`Kubernetes` version `1.27` +[#outposts-platform-versions-1-30] +== Kubernetes version `1.30` -The following admission controllers are enabled for all `1.27` platform versions: `CertificateApproval`, `CertificateSigning`, `CertificateSubjectRestriction`, `DefaultIngressClass`, `DefaultStorageClass`, `DefaultTolerationSeconds`, `ExtendedResourceToleration`, `LimitRanger`, `MutatingAdmissionWebhook`, `NamespaceLifecycle`, `NodeRestriction`, `PersistentVolumeClaimResize`, `Priority`, `PodSecurity`, `ResourceQuota`, `RuntimeClass`, `ServiceAccount`, `StorageObjectInUseProtection`, `TaintNodesByCondition`, `ValidatingAdmissionPolicy`, and `ValidatingAdmissionWebhook`. +The following admission controllers are enabled for all `1.30` platform versions: `CertificateApproval`, `CertificateSigning`, `CertificateSubjectRestriction`, `ClusterTrustBundleAttest`, `DefaultIngressClass`, `DefaultStorageClass`, `DefaultTolerationSeconds`, `ExtendedResourceToleration`, `LimitRanger`, `MutatingAdmissionWebhook`, `NamespaceLifecycle`, `NodeRestriction`, `PersistentVolumeClaimResize`, `PodSecurity`, `Priority`, `ResourceQuota`, `RuntimeClass`, `ServiceAccount`, `StorageObjectInUseProtection`, `TaintNodesByCondition`, `ValidatingAdmissionPolicy`, and `ValidatingAdmissionWebhook`. -[cols="1,1,1,1", options="header"] +[%header,cols="4"] |=== |Kubernetes version |Amazon EKS platform version |Release notes |Release date +|`1.30.14` +|`eks-local-outposts.7` +|New platform version with security fixes and enhancements. kube-proxy updated to `v1.30.14`. {aws} IAM Authenticator updated to `v0.7.4`. Amazon VPC CNI plugin for Kubernetes updated to `v1.20.2`. Bottlerocket version updated to `v1.47.0`. +|October 3, 2025 -|`1.27.10` +|`1.30.13` +|`eks-local-outposts.6` +|New platform version with security fixes and enhancements. kube-proxy updated to `v1.30.13`. {aws} IAM Authenticator updated to `v0.7.2`. Amazon VPC CNI plugin for Kubernetes updated to `v1.20.0`. Bottlerocket version updated to `v1.43.0`. +|August 09, 2025 + +|`1.30.11` |`eks-local-outposts.5` -|New platform with security fixes and enhancements. -|April 2, 2024 +|New platform version with security fixes and enhancements. kube-proxy updated to `v1.30.11`. {aws} IAM Authenticator updated to `v0.7.1`. Amazon VPC CNI plugin for Kubernetes updated to `v1.19.5` Bottlerocket version updated to `v1.40.0`. +|June 19, 2025 -|`1.27.10` +|`1.30.10` |`eks-local-outposts.4` -|New platform with security fixes and enhancements. kube-proxy updated to `v1.27.10`. {aws} IAM Authenticator updated to `v0.6.17`. Updated [.noloc]`Bottlerocket` version to `v1.19.2`. -|March 22, 2024 +|New platform version with security fixes and enhancements. Bottlerocket version updated to `v1.36.0`. +|April 24, 2025 -|`1.27.3` +|`1.30.10` |`eks-local-outposts.3` -|New platform version with security fixes and enhancements. `kube-proxy` updated to `v1.27.3`. Amazon VPC CNI plugin for Kubernetes updated to `v1.13.2`. -|July 14, 2023 +|New platform version with security fixes and enhancements. kube-proxy updated to `v1.30.10`. {aws} IAM Authenticator updated to `v0.6.29`. Amazon VPC CNI plugin for Kubernetes updated to `v1.19.2`. CoreDNS updated to `v1.11.4`. {aws} Cloud Controller Manager updated to `v1.30.8`. Bottlerocket version updated to `v1.34.0`. +|March 27, 2025 -|`1.27.1` +|`1.30.7` |`eks-local-outposts.2` -|Updated CoreDNS image to `v1.10.1` -|June 22, 2023 +|New platform version with security fixes and enhancements. kube-proxy updated to `v1.30.7`. {aws} IAM Authenticator updated to `v0.6.28`. Amazon VPC CNI plugin for Kubernetes updated to `v1.19.0`. Updated Bottlerocket version to `v1.29.0`. +|January 10, 2025 -|`1.27.1` +|`1.30.5` |`eks-local-outposts.1` -|Initial release of Kubernetes version `1.27` for local Amazon EKS clusters on Outposts. -|May 30, 2023 +|Initial release of Kubernetes version `v1.30` for local Amazon EKS clusters on Outposts. +|November 13, 2024 |=== -[[outposts-platform-versions-1.26,outposts-platform-versions-1.26.title]] -== [.noloc]`Kubernetes` version `1.26` +[#outposts-platform-versions-1-29] +== Kubernetes version `1.29` -The following admission controllers are enabled for all `1.26` platform versions: `CertificateApproval`, `CertificateSigning`, `CertificateSubjectRestriction`, `DefaultIngressClass`, `DefaultStorageClass`, `DefaultTolerationSeconds`, `ExtendedResourceToleration`, `LimitRanger`, `MutatingAdmissionWebhook`, `NamespaceLifecycle`, `NodeRestriction`, `PersistentVolumeClaimResize`, `Priority`, `PodSecurity`, `ResourceQuota`, `RuntimeClass`, `ServiceAccount`, `StorageObjectInUseProtection`, `TaintNodesByCondition`, `ValidatingAdmissionPolicy`, and `ValidatingAdmissionWebhook`. +The following admission controllers are enabled for all `1.29` platform versions: `CertificateApproval`, `CertificateSigning`, `CertificateSubjectRestriction`, `ClusterTrustBundleAttest`, `DefaultIngressClass`, `DefaultStorageClass`, `DefaultTolerationSeconds`, `ExtendedResourceToleration`, `LimitRanger`, `MutatingAdmissionWebhook`, `NamespaceLifecycle`, `NodeRestriction`, `PersistentVolumeClaimResize`, `PodSecurity`, `Priority`, `ResourceQuota`, `RuntimeClass`, `ServiceAccount`, `StorageObjectInUseProtection`, `TaintNodesByCondition`, `ValidatingAdmissionPolicy`, and `ValidatingAdmissionWebhook`. -[cols="1,1,1,1", options="header"] +[%header,cols="4"] |=== |Kubernetes version |Amazon EKS platform version |Release notes |Release date +|`v1.29.15` +|`eks-local-outposts.10` +|New platform version with security fixes and enhancements. {aws} IAM Authenticator updated to `v0.7.4`. Amazon VPC CNI plugin for Kubernetes updated to `v1.20.2`. Bottlerocket version updated to `v1.47.0`. +|October 3, 2025 -|`1.26.13` -|`eks-local-outposts.5` -|New platform version with security fixes and enhancements. kube-proxy updated to `v1.26.13`. {aws} IAM Authenticator updated to `v0.6.17`. Updated [.noloc]`Bottlerocket` version to `v1.19.2`. -|March 22, 2024 -|=== +|`v1.29.15` +|`eks-local-outposts.9` +|New platform version with security fixes and enhancements. {aws} IAM Authenticator updated to `v0.7.2`. Amazon VPC CNI plugin for Kubernetes updated to `v1.20.0`. Bottlerocket version updated to `v1.43.0`. +|August 9, 2025 -[[outposts-platform-versions-1.25,outposts-platform-versions-1.25.title]] -== [.noloc]`Kubernetes` version `1.25` +|`v1.29.15` +|`eks-local-outposts.8` +|New platform version with security fixes and enhancements. kube-proxy updated to `v1.29.15`. {aws} IAM Authenticator updated to `v0.7.1`. Amazon VPC CNI plugin for Kubernetes updated to `v1.19.5`. Bottlerocket version updated to `v1.40.0`. +|June 19, 2025 -The following admission controllers are enabled for all `1.25` platform versions: `CertificateApproval`, `CertificateSigning`, `CertificateSubjectRestriction`, `DefaultIngressClass`, `DefaultStorageClass`, `DefaultTolerationSeconds`, `ExtendedResourceToleration`, `LimitRanger`, `MutatingAdmissionWebhook`, `NamespaceLifecycle`, `NodeRestriction`, `PersistentVolumeClaimResize`, `Priority`, `PodSecurity`, `ResourceQuota`, `RuntimeClass`, `ServiceAccount`, `StorageObjectInUseProtection`, `TaintNodesByCondition`, and `ValidatingAdmissionWebhook`. - -[cols="1,1,1,1", options="header"] -|=== -|Kubernetes version -|Amazon EKS platform version -|Release notes -|Release date - - -|`1.25.16` +|`v1.29.14` |`eks-local-outposts.7` -|New platform version with security fixes and enhancements. kube-proxy updated to `v1.25.16`. {aws} IAM Authenticator updated to `v0.6.17`. Updated [.noloc]`Bottlerocket` version to `v1.19.2`. -|March 22, 2024 +|New platform version with security fixes and enhancements. Bottlerocket version updated to `v1.36.0`. +|March 24, 2025 -|`1.25.11` +|`v1.29.14` |`eks-local-outposts.6` -|New platform version with security fixes and enhancements. `kube-proxy` updated to `v1.25.11`. Amazon VPC CNI plugin for Kubernetes updated to `v1.13.2`. -|July 14, 2023 +|New platform version with security fixes and enhancements. kube-proxy updated to `v1.29.14`. Amazon VPC CNI plugin for Kubernetes updated to `v1.19.2`. CoreDNS updated to `v1.11.4`. {aws} Cloud Controller Manager updated to `v1.29.8`. Bottlerocket version updated to `v1.34.0`. +|March 27, 2025 -|`1.25.9` +|`v1.29.11` |`eks-local-outposts.5` -|New platform version with security fixes and enhancements. -|July 13, 2023 +|New platform version with security fixes and enhancements. kube-proxy updated to `v1.29.11`. Amazon VPC CNI plugin for Kubernetes updated to `v1.19.0`. Updated CoreDNS image to `v1.11.3`. Updated Bottlerocket version to `v1.29.0`. +|January 10, 2025 -|`1.25.6` +|`1.29.9` |`eks-local-outposts.4` -|Updated Bottlerocket version to `1.13.2` -|May 2, 2023 +|New platform version with security fixes and enhancements. kube-proxy updated to `v1.29.9`. {aws} IAM Authenticator updated to `v0.6.26`. Updated Bottlerocket version to `v1.26.0`. +|November 8, 2024 -|`1.25.6` +|`1.29.6` |`eks-local-outposts.3` -|Amazon EKS control plane instance operating system updated to Bottlerocket version `v1.13.1` and Amazon VPC CNI plugin for Kubernetes updated to version `v1.12.6`. -|April 14, 2023 +|New platform version with security fixes and enhancements. Updated Bottlerocket version to `v1.22.0`. +|October 22, 2024 -|`1.25.6` +|`1.29.6` |`eks-local-outposts.2` -|Improved diagnostics collection for Kubernetes control plane instances. -|March 8, 2023 +|New platform version with security fixes and enhancements. Updated Bottlerocket version to `v1.21.0`. +|August 27, 2024 -|`1.25.6` +|`1.29.6` |`eks-local-outposts.1` -|Initial release of Kubernetes version `1.25` for local Amazon EKS clusters on Outposts. -|March 1, 2023 +|Initial release of Kubernetes version `v1.29` for local Amazon EKS clusters on Outposts. +|August 20, 2024 |=== -[[outposts-platform-versions-1.24,outposts-platform-versions-1.24.title]] -== [.noloc]`Kubernetes` version `1.24` +[#outposts-platform-versions-1-28] +== Kubernetes version `1.28` -The following admission controllers are enabled for all `1.24` platform versions: `DefaultStorageClass`, `DefaultTolerationSeconds`, `LimitRanger`, `MutatingAdmissionWebhook`, `NamespaceLifecycle`, `NodeRestriction`, `ResourceQuota`, `ServiceAccount`, `ValidatingAdmissionWebhook`, `PodSecurityPolicy`, `TaintNodesByCondition`, `StorageObjectInUseProtection`, `PersistentVolumeClaimResize`, `ExtendedResourceToleration`, `CertificateApproval`, `PodPriority`, `CertificateSigning`, `CertificateSubjectRestriction`, `RuntimeClass`, and `DefaultIngressClass`. +The following admission controllers are enabled for all `1.28` platform versions: `CertificateApproval`, `CertificateSigning`, `CertificateSubjectRestriction`, `ClusterTrustBundleAttest`, `DefaultIngressClass`, `DefaultStorageClass`, `DefaultTolerationSeconds`, `ExtendedResourceToleration`, `LimitRanger`, `MutatingAdmissionWebhook`, `NamespaceLifecycle`, `NodeRestriction`, `PersistentVolumeClaimResize`, `PodSecurity`, `Priority`, `ResourceQuota`, `RuntimeClass`, `ServiceAccount`, `StorageObjectInUseProtection`, `TaintNodesByCondition`, `ValidatingAdmissionPolicy`, and `ValidatingAdmissionWebhook`. -[cols="1,1,1,1", options="header"] +[%header,cols="4"] |=== |Kubernetes version |Amazon EKS platform version |Release notes |Release date +|`1.28.15` +|`eks-local-outposts.17` +|New platform version with security fixes and enhancements. {aws} IAM Authenticator updated to `v0.7.4`. Amazon VPC CNI plugin for Kubernetes updated to `v1.20.2`. Bottlerocket version updated to `v1.47.0`. +|October 3, 2025 + +|`1.28.15` +|`eks-local-outposts.16` +|New platform version with security fixes and enhancements. {aws} IAM Authenticator updated to `v0.7.2`. Amazon VPC CNI plugin for Kubernetes updated to `v1.20.0`. Bottlerocket version updated to `v1.43.0`. +|August 9, 2025 + +|`1.28.15` +|`eks-local-outposts.15` +|New platform version with security fixes and enhancements. {aws} IAM Authenticator updated to `v0.7.1`. Amazon VPC CNI plugin for Kubernetes updated to `v1.19.5`. CoreDNS updated to `v1.11.4`. Bottlerocket version updated to `v1.40.0`. +|June 19, 2025 + +|`1.28.15` +|`eks-local-outposts.14` +|New platform version with security fixes and enhancements. Bottlerocket version updated to `v1.36.0`. +|April 24, 2025 + +|`1.28.15` +|`eks-local-outposts.13` +|New platform version with security fixes and enhancements. kube-proxy `v1.28.15` build updated. Amazon VPC CNI plugin for Kubernetes updated to `v1.19.0`. {aws} Cloud Controller Manager updated to `v1.28.11`. +|March 27, 2025 + +|`1.28.15` +|`eks-local-outposts.12` +|New platform version with security fixes and enhancements. kube-proxy updated to `v1.28.15`. Amazon VPC CNI plugin for Kubernetes updated to `v1.19.0`. Updated Bottlerocket version to `v1.29.0`. +|January 10, 2025 + +|`1.28.14` +|`eks-local-outposts.11` +|New platform version with security fixes and enhancements. kube-proxy updated to `v1.28.14`. {aws} IAM Authenticator updated to `v0.6.26`. Updated CoreDNS image to `v1.11.1`. Updated Bottlerocket version to `v1.26.0`. +|November 8, 2024 + +|`1.28.10` +|`eks-local-outposts.10` +|New platform version with security fixes and enhancements. Updated Bottlerocket version to `v1.22.0`. +|October 22, 2024 + +|`1.28.10` +|`eks-local-outposts.9` +|New platform version with security fixes and enhancements. Updated Bottlerocket version to `v1.21.0`. +|August 27, 2024 + +|`1.28.10` +|`eks-local-outposts.8` +|New platform version with security fixes and enhancements. +|July 30, 2024 -|`1.24.17` -|`eks-local-outposts.7` -|New platform version with security fixes and enhancements. kube-proxy updated to `v1.25.16`. {aws} IAM Authenticator updated `v0.6.17`. Updated [.noloc]`Bottlerocket` version to `v1.19.2`. -|March 22, 2024 - -|`1.24.15` +|`1.28.10` |`eks-local-outposts.6` -|New platform version with security fixes and enhancements. `kube-proxy` updated to `v1.24.15`. Amazon VPC CNI plugin for Kubernetes updated to `v1.13.2`. -|July 14, 2023 +|New platform version with security fixes and enhancements. kube-proxy updated to `v1.28.10`. {aws} IAM Authenticator updated to `v0.6.20`. Updated Bottlerocket version to `v1.20.2`. +|June 19, 2024 -|`1.24.13` +|`1.28.6` |`eks-local-outposts.5` -|New platform version with security fixes and enhancements. -|July 13, 2023 +|Updated Bottlerocket version to v1.19.3 containing newest bugfixes to support local boot in Outposts. +|April 18, 2024 -|`1.24.9` +|`1.28.6` |`eks-local-outposts.4` -|Updated Bottlerocket version to `1.13.2` -|May 2, 2023 +|New platform version with security fixes and enhancements. Restored support or local boot in Outposts. Downgraded Bottlerocket version to `v1.15.1` for compatibility. +|April 2, 2024 -|`1.24.9` +|`1.28.6` |`eks-local-outposts.3` -|Amazon EKS control plane instance operating system updated to Bottlerocket version `v1.13.1` and Amazon VPC CNI plugin for Kubernetes updated to version `v1.12.6`. -|April 14, 2023 +|New platform version with security fixes and enhancements. +|March 22, 2024 -|`1.24.9` +|`1.28.6` |`eks-local-outposts.2` -|Improved diagnostics collection for Kubernetes control plane instances. -|March 8, 2023 +|New platform version with security fixes and enhancements. kube-proxy updated to `v1.28.6`. {aws} IAM Authenticator updated to `v0.6.17`. Amazon VPC CNI plugin for Kubernetes downgraded to `v1.13.2` for compatibility reasons. Updated Bottlerocket version to `v1.19.2`. +|March 8, 2024 -|`1.24.9` +|`1.28.1` |`eks-local-outposts.1` -|Initial release of Kubernetes version `1.24` for local Amazon EKS clusters on Outposts. -|January 17, 2023 -|=== - -[[outposts-platform-versions-1.23,outposts-platform-versions-1.23.title]] -== [.noloc]`Kubernetes` version `1.23` - -The following admission controllers are enabled for all `1.23` platform versions: `DefaultStorageClass`, `DefaultTolerationSeconds`, `LimitRanger`, `MutatingAdmissionWebhook`, `NamespaceLifecycle`, `NodeRestriction`, `ResourceQuota`, `ServiceAccount`, `ValidatingAdmissionWebhook`, `PodSecurityPolicy`, `TaintNodesByCondition`, `StorageObjectInUseProtection`, `PersistentVolumeClaimResize`, `ExtendedResourceToleration`, `CertificateApproval`, `PodPriority`, `CertificateSigning`, `CertificateSubjectRestriction`, `RuntimeClass`, and `DefaultIngressClass`. - -[cols="1,1,1,1", options="header"] -|=== -|Kubernetes version -|Amazon EKS platform version -|Release notes -|Release date - - -|`1.23.17` -|`eks-local-outposts.6` -|New platform version with security fixes and enhancements. -|July 13, 2023 - -|`1.23.17` -|`eks-local-outposts.5` -|New platform version with security fixes and enhancements. kube-proxy updated to `v1.23.17`. Updated [.noloc]`Bottlerocket` version to `v1.14.1`. -|July 6, 2023 - -|`1.23.15` -|`eks-local-outposts.4` -|Amazon EKS control plane instance operating system updated to Bottlerocket version `v1.13.1` and Amazon VPC CNI plugin for Kubernetes updated to version `v1.12.6`. -|April 14, 2023 - -|`1.23.15` -|`eks-local-outposts.3` -|Improved diagnostics collection for Kubernetes control plane instances. -|March 8, 2023 - -|`1.23.15` -|`eks-local-outposts.2` -|Initial release of Kubernetes version `1.23` for local Amazon EKS clusters on Outposts. -|January 17, 2023 +|Initial release of Kubernetes version `v1.28` for local Amazon EKS clusters on Outposts. +|October 4, 2023 |=== diff --git a/latest/ug/outposts/eks-outposts-self-managed-nodes.adoc b/latest/ug/outposts/eks-outposts-self-managed-nodes.adoc index d16c40965..37b1da142 100644 --- a/latest/ug/outposts/eks-outposts-self-managed-nodes.adoc +++ b/latest/ug/outposts/eks-outposts-self-managed-nodes.adoc @@ -1,43 +1,56 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[eks-outposts-self-managed-nodes,eks-outposts-self-managed-nodes.title]] +[#eks-outposts-self-managed-nodes] = Create Amazon Linux nodes on {aws} Outposts -:info_doctype: section -:info_title: Create Amazon Linux nodes on {aws} Outposts :info_titleabbrev: Nodes -:keywords: launch, start, self-managed, Linux, node -:info_abstract: Learn how to launch Auto Scaling groups of Amazon Linux nodes on an Outpost that register with \ - your Amazon EKS cluster. The cluster can be on the {aws} Cloud or on an Outpost. [abstract] -- Learn how to launch Auto Scaling groups of Amazon Linux nodes on an Outpost that register with your Amazon EKS cluster. The cluster can be on the {aws} Cloud or on an Outpost. -- +[IMPORTANT] +==== +Amazon EKS Local Clusters on Outposts only supports nodes created from the following Amazon EKS-optimized Amazon Linux 2 AMIs: + +* Standard Amazon Linux 2 (`amazon-linux-2`) +* GPU-enabled Amazon Linux 2 (`amazon-linux-2-gpu`) +* Arm64-based Amazon Linux 2 (`amazon-linux-2-arm64`) + +Nodes on Local Clusters that run Amazon Linux 2023 (AL2023) AMIs aren't supported and fail to join the cluster. +==== + This topic describes how you can launch Auto Scaling groups of Amazon Linux nodes on an Outpost that register with your Amazon EKS cluster. The cluster can be on the {aws} Cloud or on an Outpost. * An existing Outpost. For more information, see link:outposts/latest/userguide/what-is-outposts.html[What is {aws} Outposts,type="documentation"]. * An existing Amazon EKS cluster. To deploy a cluster on the {aws} Cloud, see <>. To deploy a cluster on an Outpost, see <>. * Suppose that you're creating your nodes in a cluster on the {aws} Cloud and you have subnets in the {aws} Region where you have {aws} Outposts, {aws} Wavelength, or {aws} Local Zones enabled. Then, those subnets must not have been passed in when you created your cluster. If you're creating your nodes in a cluster on an Outpost, you must have passed in an Outpost subnet when creating your cluster. -* (Recommended for clusters on the {aws} Cloud) The [.noloc]`Amazon VPC CNI plugin for Kubernetes` add-on configured with its own IAM role that has the necessary IAM policy attached to it. For more information, see <>. Local clusters do not support IAM roles for service accounts. +* (Recommended for clusters on the {aws} Cloud) The Amazon VPC CNI plugin for Kubernetes add-on configured with its own IAM role that has the necessary IAM policy attached to it. For more information, see <>. Local clusters do not support IAM roles for service accounts. -You can create a self-managed Amazon Linux node group with `eksctl` or the {aws-management-console} (with an {aws} CloudFormation template). You can also use https://registry.terraform.io/modules/terraform-aws-modules/eks/aws/latest[Terraform]. +You can create a self-managed Amazon Linux node group with `eksctl` or the {aws-management-console} (with an {aws} CloudFormation template). You can also use Terraform. -You can create a local cluster with the following tools described in this page: +You can create a self-managed node group for local cluster with the following tools described in this page: * <> * <> +[IMPORTANT] +==== +* Self-managed node group includes Amazon EC2 instances in your account. These instances aren't automatically upgraded when you or Amazon EKS update the control plane version on your behalf. A self-managed node group doesn't have any indication in the console that it needs updating. You can view the `kubelet` version installed on a node by selecting the node in the *Nodes* list on the *Overview* tab of your cluster to determine which nodes need updating. You must manually update the nodes. For more information, see <>. +* The certificates used by kubelet on your self-managed nodes are issued with one year expiration. By default certificate rotation is *not* enabled (see: https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/#kubelet-config-k8s-io-v1beta1-KubeletConfiguration), this means if you have a self-managed node running for more than one year, it will no longer be able to authenticate to the Kubernetes API. +* As a best practice we recommend customers to regularly update their self-managed node groups to receive CVEs and security patches from latest Amazon EKS optimized AMI. Updating AMI used in self-managed node groups also triggers re-creation of nodes and make sure they do not run into issue due to expired kubelet certificates. +* Alternatively you can also enable client certificate rotation (see: https://kubernetes.io/docs/tasks/tls/certificate-rotation/) when creating the self-managed node groups to make sure kubelet certificates are renewed as the current certificate approaches expiration. +==== + == `eksctl` [[eksctl_create_nodes_outpost]] *To launch self-managed Linux nodes using `eksctl`* . Install version `{eksctl-min-version}` or later of the `eksctl` command line tool installed on your device or {aws} CloudShell. To install or update `eksctl`, see https://eksctl.io/installation[Installation] in the `eksctl` documentation. + -. If your cluster is on the {aws} Cloud and the *AmazonEKS_CNI_Policy* managed IAM policy is attached to your <>, we recommend assigning it to an IAM role that you associate to the [.noloc]`Kubernetes` `aws-node` service account instead. For more information, see <>. If your cluster in on your Outpost, the policy must be attached to your node role. -. The following command creates a node group in an existing cluster. The cluster must have been created using `eksctl`. Replace [.replaceable]`al-nodes` with a name for your node group. The node group name can't be longer than 63 characters. It must start with letter or digit, but can also include hyphens and underscores for the remaining characters. Replace [.replaceable]`my-cluster` with the name of your cluster. The name can contain only alphanumeric characters (case-sensitive) and hyphens. It must start with an alphanumeric character and can't be longer than 100 characters. The name must be unique within the {aws} Region and {aws} account that you're creating the cluster in. If your cluster exists on an Outpost, replace [.replaceable]`id` with the ID of an Outpost subnet. If your cluster exists on the {aws} Cloud, replace [.replaceable]`id` with the ID of a subnet that you didn't specify when you created your cluster. Replace [.replaceable]`instance-type` with an instance type supported by your Outpost. Replace the remaining [.replaceable]`example values` with your own values. The nodes are created with the same [.noloc]`Kubernetes` version as the control plane, by default. +. If your cluster is on the {aws} Cloud and the *AmazonEKS_CNI_Policy* managed IAM policy is attached to your <>, we recommend assigning it to an IAM role that you associate to the Kubernetes `aws-node` service account instead. For more information, see <>. If your cluster in on your Outpost, the policy must be attached to your node role. +. The following command creates a node group in an existing cluster. The cluster must have been created using `eksctl`. Replace [.replaceable]`al-nodes` with a name for your node group. The node group name can't be longer than 63 characters. It must start with letter or digit, but can also include hyphens and underscores for the remaining characters. Replace [.replaceable]`my-cluster` with the name of your cluster. The name can contain only alphanumeric characters (case-sensitive) and hyphens. It must start with an alphanumeric character and can't be longer than 100 characters. The name must be unique within the {aws} Region and {aws} account that you're creating the cluster in. If your cluster exists on an Outpost, replace [.replaceable]`id` with the ID of an Outpost subnet. If your cluster exists on the {aws} Cloud, replace [.replaceable]`id` with the ID of a subnet that you didn't specify when you created your cluster. Replace [.replaceable]`instance-type` with an instance type supported by your Outpost. Replace the remaining example values with your own values. The nodes are created with the same Kubernetes version as the control plane, by default. + Replace [.replaceable]`instance-type` with an instance type available on your Outpost. + @@ -53,7 +66,7 @@ eksctl create nodegroup --cluster my-cluster --name al-nodes --node-type instanc + If your cluster is deployed on the {aws} Cloud: + -** The node group that you deploy can assign `IPv4` addresses to [.noloc]`Pods` from a different [.noloc]`CIDR` block than that of the instance. For more information, see <>. +** The node group that you deploy can assign `IPv4` addresses to Pods from a different CIDR block than that of the instance. For more information, see <>. ** The node group that you deploy doesn't require outbound internet access. For more information, see <>. + @@ -65,12 +78,12 @@ For a complete list of all available options and defaults, see https://eksctl.io ---- [✔] created 1 nodegroup(s) in cluster "my-cluster" ---- -. (Optional) Deploy a <> to test your cluster and [.noloc]`Linux` nodes. +. (Optional) Deploy a <> to test your cluster and Linux nodes. == {aws-management-console} [[console_create_nodes_outpost]] -*Step 1: Launch self-managed Linux nodes using {aws-management-console}*` +*Step 1: Launch self-managed Linux nodes using {aws-management-console}* . Download the latest version of the {aws} CloudFormation template. + @@ -79,47 +92,59 @@ For a complete list of all available options and defaults, see https://eksctl.io curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/cloudformation/2022-12-23/amazon-eks-nodegroup.yaml ---- . Open the link:cloudformation/[{aws} CloudFormation console,type="console"]. -. Choose *Create stack* and then select *With new resources (standard)*. -. For *Specify template*, select *Upload a template file* and then select *Choose file*. Select the `amazon-eks-nodegroup.yaml` file that you downloaded in a previous step and then select *Next*. -. On the *Specify stack details* page, enter the following parameters accordingly, and then choose *Next*: +. Choose *Create stack* and then select *With new resources (standard)*. +. For *Specify template*, select *Upload a template file* and then select *Choose file*. Select the `amazon-eks-nodegroup.yaml` file that you downloaded in a previous step and then select *Next*. +. On the *Specify stack details* page, enter the following parameters accordingly, and then choose *Next*: + ** *Stack name*: Choose a stack name for your {aws} CloudFormation stack. For example, you can call it [.replaceable]`al-nodes`. The name can contain only alphanumeric characters (case-sensitive) and hyphens. It must start with an alphanumeric character and can't be longer than 100 characters. The name must be unique within the {aws} Region and {aws} account that you're creating the cluster in. ** *ClusterName*: Enter the name of your cluster. If this name doesn't match your cluster name, your nodes can't join the cluster. -** *ClusterControlPlaneSecurityGroup*: Choose the *SecurityGroups* value from the {aws} CloudFormation output that you generated when you created your <>. +** *ClusterControlPlaneSecurityGroup*: Choose the *SecurityGroups* value from the {aws} CloudFormation output that you generated when you created your <>. + The following steps show one operation to retrieve the applicable group. + ... Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. ... Choose the name of the cluster. ... Choose the *Networking* tab. -... Use the *Additional security groups* value as a reference when selecting from the *ClusterControlPlaneSecurityGroup* dropdown list. +... Use the *Additional security groups* value as a reference when selecting from the *ClusterControlPlaneSecurityGroup* dropdown list. ** *NodeGroupName*: Enter a name for your node group. This name can be used later to identify the Auto Scaling node group that's created for your nodes. ** *NodeAutoScalingGroupMinSize*: Enter the minimum number of nodes that your node Auto Scaling group can scale in to. ** *NodeAutoScalingGroupDesiredCapacity*: Enter the desired number of nodes to scale to when your stack is created. ** *NodeAutoScalingGroupMaxSize*: Enter the maximum number of nodes that your node Auto Scaling group can scale out to. ** *NodeInstanceType*: Choose an instance type for your nodes. If your cluster is running on the {aws} Cloud, then for more information, see <>. If your cluster is running on an Outpost, then you can only select an instance type that is available on your Outpost. -** *NodeImageIdSSMParam*: Pre-populated with the Amazon EC2 Systems Manager parameter of a recent Amazon EKS optimized AMI for a variable [.noloc]`Kubernetes` version. To use a different [.noloc]`Kubernetes` minor version supported with Amazon EKS, replace [.replaceable]`1.XX` with a different <>. We recommend specifying the same [.noloc]`Kubernetes` version as your cluster. +** *NodeImageIdSSMParam*: Pre-populated with the Amazon EC2 Systems Manager parameter of a recent Amazon EKS optimized AMI for a variable Kubernetes version. To use a different Kubernetes minor version supported with Amazon EKS, replace [.replaceable]`1.XX` with a different link:eks/latest/userguide/kubernetes-versions.html[supported version,type="documentation"]. We recommend specifying the same Kubernetes version as your cluster. + To use an Amazon EKS optimized accelerated AMI, replace [.replaceable]`amazon-linux-2` with `amazon-linux-2-gpu`. To use an Amazon EKS optimized Arm AMI, replace [.replaceable]`amazon-linux-2` with `amazon-linux-2-arm64`. + NOTE: The Amazon EKS node AMIs are based on Amazon Linux. You can track security or privacy events for Amazon Linux at the https://alas.aws.amazon.com/[Amazon Linux security center] by choosing the tab for your desired version. You can also subscribe to the applicable RSS feed. Security and privacy events include an overview of the issue, what packages are affected, and how to update your instances to correct the issue. -** *NodeImageId*: (Optional) If you're using your own custom AMI (instead of an Amazon EKS optimized AMI), enter a node AMI ID for your {aws} Region. If you specify a value here, it overrides any values in the *NodeImageIdSSMParam* field. +** *NodeImageId*: (Optional) If you're using your own custom AMI (instead of an Amazon EKS optimized AMI), enter a node AMI ID for your {aws} Region. If you specify a value here, it overrides any values in the *NodeImageIdSSMParam* field. ** *NodeVolumeSize*: Specify a root volume size for your nodes, in GiB. ** *NodeVolumeType*: Specify a root volume type for your nodes. ** *KeyName*: Enter the name of an Amazon EC2 SSH key pair that you can use to connect using SSH into your nodes with after they launch. If you don't already have an Amazon EC2 key pair, you can create one in the {aws-management-console}. For more information, see link:AWSEC2/latest/UserGuide/ec2-key-pairs.html[Amazon EC2 key pairs,type="documentation"] in the _Amazon EC2 User Guide_. + NOTE: If you don't provide a key pair here, the {aws} CloudFormation stack creation fails. -** *BootstrapArguments*: There are several optional arguments that you can pass to your nodes. For more information, view the https://github.com/awslabs/amazon-eks-ami/blob/main/templates/al2/runtime/bootstrap.sh[bootstrap script usage information] on [.noloc]`GitHub`. If you're adding nodes to an Amazon EKS Local Cluster on {aws} Outposts (where the [.noloc]`Kubernetes` control plane instances run on {aws} Outposts) and the cluster doesn't have ingress and egress internet connection (also known as private clusters), then you must provide the following bootstrap arguments (as a single line). +** *BootstrapArguments*: There are several optional arguments that you can pass to your nodes. For more information, view the https://github.com/awslabs/amazon-eks-ami/blob/main/templates/al2/runtime/bootstrap.sh[bootstrap script usage information] on GitHub. If you're adding nodes to an Amazon EKS Local Cluster on {aws} Outposts (where the Kubernetes control plane instances run on {aws} Outposts) and the cluster doesn't have ingress and egress internet connection (also known as private clusters), then you must provide the following bootstrap arguments (as a single line). + [source,bash,subs="verbatim,attributes"] ---- --b64-cluster-ca ${CLUSTER_CA} --apiserver-endpoint https://${APISERVER_ENDPOINT} --enable-local-outpost true --cluster-id ${CLUSTER_ID} ---- -** *DisableIMDSv1*: By default, each node supports the Instance Metadata Service Version 1 (IMDSv1) and IMDSv2. You can disable IMDSv1. To prevent future nodes and [.noloc]`Pods` in the node group from using IMDSv1, set *DisableIMDSv1* to *true*. For more information about IMDS, see link:AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html[Configuring the instance metadata service,type="documentation"]. For more information about restricting access to it on your nodes, see https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node[Restrict access to the instance profile assigned to the worker node]. -** *VpcId*: Enter the ID for the <> that you created. Before choosing a VPC, review <>. -** *Subnets*: If your cluster is on an Outpost, then choose at least one private subnet in your VPC. Before choosing subnets, review <>. You can see which subnets are private by opening each subnet link from the *Networking* tab of your cluster. -. Select your desired choices on the *Configure stack options* page, and then choose *Next*. -. Select the check box to the left of *I acknowledge that {aws} CloudFormation might create IAM resources.*, and then choose *Create stack*. +To retrieve the values for `CLUSTER_CA`, `APISERVER_ENDPOINT`, and `CLUSTER_ID` of your Amazon EKS local cluster, run the following {aws} CLI commands. Replace cluster-name with the name of your cluster and region (for example, us-east-1) with your cluster's {aws} Region. ++ +[source,bash,subs="verbatim,attributes"] +---- +echo "CLUSTER_CA=$(aws eks describe-cluster --name cluster-name --region region --query cluster.certificateAuthority.data --output text)" + +echo "APISERVER_ENDPOINT=$(aws eks describe-cluster --name cluster-name --region region --query cluster.endpoint --output text)" + +echo "CLUSTER_ID=$(aws eks describe-cluster --name cluster-name --region region --query cluster.id --output text)" +---- + + +** *DisableIMDSv1*: By default, each node supports the Instance Metadata Service Version 1 (IMDSv1) and IMDSv2. You can disable IMDSv1. To prevent future nodes and Pods in the node group from using IMDSv1, set *DisableIMDSv1* to *true*. For more information about IMDS, see link:AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html[Configuring the instance metadata service,type="documentation"]. For more information about restricting access to it on your nodes, see https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node[Restrict access to the instance profile assigned to the worker node]. +** *VpcId*: Enter the ID for the <> that you created. Before choosing a VPC, review <>. +** *Subnets*: If your cluster is on an Outpost, then choose at least one private subnet in your VPC. Before choosing subnets, review <>. You can see which subnets are private by opening each subnet link from the *Networking* tab of your cluster. +. Select your desired choices on the *Configure stack options* page, and then choose *Next*. +. Select the check box to the left of *I acknowledge that {aws} CloudFormation might create IAM resources.*, and then choose *Create stack*. . When your stack has finished creating, select it in the console and choose *Outputs*. . Record the *NodeInstanceRole* for the node group that was created. You need this when you configure your Amazon EKS nodes. @@ -215,7 +240,7 @@ EOF ---- kubectl apply -f gp2-storage-class.yaml ---- -. (GPU nodes only) If you chose a GPU instance type and an Amazon EKS optimized accelerated AMI, you must apply the https://github.com/NVIDIA/k8s-device-plugin[NVIDIA device plugin for Kubernetes] as a [.noloc]`DaemonSet` on your cluster. Replace [.replaceable]`vX.X.X` with your desired https://github.com/NVIDIA/k8s-device-plugin/releases[NVIDIA/k8s-device-plugin] version before running the following command. +. (GPU nodes only) If you chose a GPU instance type and an Amazon EKS optimized accelerated AMI, you must apply the https://github.com/NVIDIA/k8s-device-plugin[NVIDIA device plugin for Kubernetes] as a DaemonSet on your cluster. Replace [.replaceable]`vX.X.X` with your desired https://github.com/NVIDIA/k8s-device-plugin/releases[NVIDIA/k8s-device-plugin] version before running the following command. + [source,bash,subs="verbatim,attributes"] ---- @@ -224,5 +249,5 @@ kubectl apply -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/vX.X *Step3: Additional actions* -. (Optional) Deploy a <> to test your cluster and [.noloc]`Linux` nodes. -. If your cluster is deployed on an Outpost, then skip this step. If your cluster is deployed on the {aws} Cloud, the following information is optional. If the *AmazonEKS_CNI_Policy* managed IAM policy is attached to your <>, we recommend assigning it to an IAM role that you associate to the [.noloc]`Kubernetes` `aws-node` service account instead. For more information, see <>. +. (Optional) Deploy a <> to test your cluster and Linux nodes. +. If your cluster is deployed on an Outpost, then skip this step. If your cluster is deployed on the {aws} Cloud, the following information is optional. If the *AmazonEKS_CNI_Policy* managed IAM policy is attached to your <>, we recommend assigning it to an IAM role that you associate to the Kubernetes `aws-node` service account instead. For more information, see <>. \ No newline at end of file diff --git a/latest/ug/outposts/eks-outposts-troubleshooting.adoc b/latest/ug/outposts/eks-outposts-troubleshooting.adoc index 5df0da231..9940b6fd2 100644 --- a/latest/ug/outposts/eks-outposts-troubleshooting.adoc +++ b/latest/ug/outposts/eks-outposts-troubleshooting.adoc @@ -1,8 +1,7 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[eks-outposts-troubleshooting,eks-outposts-troubleshooting.title]] +[#eks-outposts-troubleshooting] = Troubleshoot local Amazon EKS clusters on {aws} Outposts :info_titleabbrev: Troubleshoot clusters @@ -14,17 +13,25 @@ Learn how to troubleshoot common issues with Amazon EKS local clusters on {aws} This topic covers some common errors that you might see while using local clusters and how to troubleshoot them. Local clusters are similar to Amazon EKS clusters in the cloud, but there are some differences in how they're managed by Amazon EKS. -[[outposts-troubleshooting-api-behavior,outposts-troubleshooting-api-behavior.title]] +[IMPORTANT] +==== + +Never terminate any managed EKS local cluster `Kubernetes` control-plane instance running on Outpost unless explicitly instructed by {aws} Support. Terminating these instances impose a risk to local cluster service availability, including loss of the local cluster in case multiple instances are simultaneously terminated. EKS local cluster `Kubernetes` control-plane instances are identified by the tag `eks-local:controlplane-name` on the EC2 instance console. + +==== + + +[#outposts-troubleshooting-api-behavior] .API behavior [%collapsible] ==== -Local clusters are created through the Amazon EKS API, but are run in an asynchronous manner. This means that requests to the Amazon EKS API return immediately for local clusters. However, these requests might succeed, fail fast because of input validation errors, or fail and have descriptive validation errors. This behavior is similar to the [.noloc]`Kubernetes` API. +Local clusters are created through the Amazon EKS API, but are run in an asynchronous manner. This means that requests to the Amazon EKS API return immediately for local clusters. However, these requests might succeed, fail fast because of input validation errors, or fail and have descriptive validation errors. This behavior is similar to the Kubernetes API. Local clusters don't transition to a `FAILED` status. Amazon EKS attempts to reconcile the cluster state with the user-requested desired state in a continuous manner. As a result, a local cluster might remain in the `CREATING` state for an extended period of time until the underlying issue is resolved. ==== -[[outposts-troubleshooting-describe-cluster-health-field,outposts-troubleshooting-describe-cluster-health-field.title]] +[#outposts-troubleshooting-describe-cluster-health-field] .Describe cluster health field [%collapsible] ==== @@ -55,7 +62,7 @@ An example output is as follows. If the problem is beyond repair, you might need to delete the local cluster and create a new one. For example, trying to provision a cluster with an instance type that's not available on your Outpost. The following table includes common health related errors. -[cols="1,1,1,1", options="header"] +[%header,cols="4"] |=== |Error scenario |Code @@ -136,7 +143,7 @@ If the problem is beyond repair, you might need to delete the local cluster and The following table lists errors from other {aws} services that are presented in the health field of the `describe-cluster` response. -[cols="1,1,1", options="header"] +[%header,cols="3"] |=== |Amazon EC2 error code |Cluster health issue code @@ -197,7 +204,7 @@ The following table lists errors from other {aws} services that are presented in |=== ==== -[[outposts-troubleshooting-unable-to-create-or-modify-clusters,outposts-troubleshooting-unable-to-create-or-modify-clusters.title]] +[#outposts-troubleshooting-unable-to-create-or-modify-clusters] .Unable to create or modify clusters [%collapsible] ==== @@ -205,7 +212,7 @@ The following table lists errors from other {aws} services that are presented in Local clusters require different permissions and policies than Amazon EKS clusters that are hosted in the cloud. When a cluster fails to create and produces an `InvalidPermissions` error, double check that the cluster role that you're using has the <> managed policy attached to it. All other API calls require the same set of permissions as Amazon EKS clusters in the cloud. ==== -[[outposts-troubleshooting-cluster-stuck-in-creating-state,outposts-troubleshooting-cluster-stuck-in-creating-state.title]] +[#outposts-troubleshooting-cluster-stuck-in-creating-state] .Cluster is stuck in `CREATING` state [%collapsible] ==== @@ -217,6 +224,7 @@ The most common issues are the following: * Your cluster can't connect to the control plane instance from the {aws} Region that Systems Manager is in. You can verify this by calling `aws ssm start-session --target [.replaceable]``instance-id``` from an in-Region bastion host. If that command doesn't work, check if Systems Manager is running on the control plane instance. Or, another work around is to delete the cluster and then recreate it. +* The control plane instances fail to create due to KMS key permissions for EBS volumes. With user managed KMS keys for encrypted EBS volumes, the control plane instances will terminate if the key is not accessible. If the instances are terminated, either switch to an {aws} managed KMS key or ensure that your user managed key policy grants the necessary permissions to the cluster role. * Systems Manager control plane instances might not have internet access. Check if the subnet that you provided when you created the cluster has a NAT gateway and a VPC with an internet gateway. Use VPC reachability analyzer to verify that the control plane instance can reach the internet gateway. For more information, see link:vpc/latest/reachability/getting-started.html[Getting started with VPC Reachability Analyzer,type="documentation"]. * The role ARN that you provided is missing policies. Check if the <> was removed from the role. This can also occur if an {aws} CloudFormation stack is misconfigured. @@ -229,37 +237,57 @@ The most common issues are the following: Review <>. ==== -[[outposts-troubleshooting-unable-to-join-nodes-to-a-cluster,outposts-troubleshooting-unable-to-join-nodes-to-a-cluster.title]] +[#outposts-troubleshooting-cluster-stuck-in-updating-state] +.Cluster is stuck in `UPDATING` state +[%collapsible] +==== + +Amazon EKS automatically updates all existing local clusters to the latest platform versions for their corresponding Kubernetes minor version. For more information about platform versions, please refer to <>. + +During an automatic platform-version rollout a cluster status changes to `UPDATING`. The update process consists of the replacement of all Kubernetes control-plane instances with new ones containing the latest security pathces and bugfixes released for the respective Kubernetes minor version. In general, a local cluster platform update process completes within less than 30 minutes and the cluster changes back to `ACTIVE` status. If a local cluster remains in the `UPDATING` state for an extended period of time, you may call `describe-cluster` to check for information about the cause in the `cluster.health` output field. + +Amazon EKS ensures at least 2 out of 3 Kubernetes control-plane instances are healthy and operational cluster nodes in order to maintain the local cluster availability and prevent service interruption. If a local cluster is stalled in `UPDATING` state it is usually because there is some infrastructure or configuration issue preventing the two-instances minimum availability to be guaranteed in case the process continues. So the update process stops progressing to protect the local cluster service interruption. + +It is important to troubleshoot a local cluster stuck in `UPDATING` status and address the root-cause so that the update process can complete and restore the local cluster back to `ACTIVE` with the high-availability of 3 Kubernetes control-plane instances. + +Do not terminate any managed EKS local cluster `Kubernetes` instances on Outposts unless explicitly instructed by {aws} Support. This is specially important for local clusters stuck in `UPDATING` state because there's a high probability that another control-plane nodes is not completely healthy and terminating the wrong instance could cause service interruption and risk local-cluster data loss. + +The most common issues are the following: + + + +* One or more control-plane instances are unable to connect to System Manager because of a networking configuration change since the local cluster was first created. You can verify this by calling `aws ssm start-session --target [.replaceable]``instance-id``` from an in-Region bastion host. If that command doesn't work, check if Systems Manager is running on the control plane instance. +* New control plane instances fail to be created due to KMS key permissions for EBS volumes. With user managed KMS keys for encrypted EBS volumes, the control plane instances will terminate if the key is not accessible. If the instances are terminated, either switch to an {aws} managed KMS key or ensure that your user managed key policy grants the necessary permissions to the cluster role. +* Systems Manager control plane instances might have lost internet access. Check if the subnet that was provided when you created the cluster has a NAT gateway and a VPC with an internet gateway. Use VPC reachability analyzer to verify that the control plane instance can reach the internet gateway. For more information, see link:vpc/latest/reachability/getting-started.html[Getting started with VPC Reachability Analyzer,type="documentation"]. If your private networks don't have outbound internet connection, ensure that all the required VPC endpoints and gateway endpoint are still present in the Regional subnet from your cluster (see <>). +* The role ARN that you provided is missing policies. Check if the <> was not removed from the role. +* One of the new Kubernetes control-plane instances may have experienced an unexpected bootstrapping failure. Please file a ticket with link:support/home[{aws} Support Center,type="console"] for further guidance on troubleshooting and log-collection in this exceptional case. + +==== + +[#outposts-troubleshooting-unable-to-join-nodes-to-a-cluster] .Can't join nodes to a cluster [%collapsible] ==== * AMI issues: + -** You're using an unsupported AMI. You must use https://github.com/awslabs/amazon-eks-ami/releases/tag/v20220620[v20220620] or later for the <> Amazon EKS optimized Amazon Linux. +** You're using an incompatible AMI. Only Amazon EKS optimized Amazon Linux 2 AMIs are supported (`amazon-linux-2`,`amazon-linux-2-gpu`, `amazon-linux-2-arm64`). If you attempt to join AL2023 nodes to EKS LocalClusters on {aws} Outposts, the nodes fail to join the cluster. For more information, see <>. ** If you used an {aws} CloudFormation template to create your nodes, make sure it wasn't using an unsupported AMI. * Missing the {aws} IAM Authenticator `ConfigMap` – If it's missing, you must create it. For more information, see <> . * The wrong security group is used – Make sure to use `eks-cluster-sg-[.replaceable]``cluster-name``-[.replaceable]``uniqueid``` for your worker nodes' security group. The selected security group is changed by {aws} CloudFormation to allow a new security group each time the stack is used. * Following unexpected private link VPC steps – Wrong CA data (`--b64-cluster-ca`) or API Endpoint (`--apiserver-endpoint`) are passed. -* Misconfigured [.noloc]`Pod` security policy: -+ -** The [.noloc]`CoreDNS` and [.noloc]`Amazon VPC CNI plugin for Kubernetes` Daemonsets must run on nodes for nodes to join and communicate with the cluster. -** The [.noloc]`Amazon VPC CNI plugin for Kubernetes` requires some privileged networking features to work properly. You can view the privileged networking features with the following command: `kubectl describe psp eks.privileged`. - -+ -We don't recommend modifying the default pod security policy. For more information, see <>. ==== -[[outposts-troubleshooting-collecting-logs,outposts-troubleshooting-collecting-logs.title]] +[#outposts-troubleshooting-collecting-logs] .Collecting logs [%collapsible] ==== -When an Outpost gets disconnected from the {aws} Region that it's associated with, the [.noloc]`Kubernetes` cluster likely will continue working normally. However, if the cluster doesn't work properly, follow the troubleshooting steps in <>. If you encounter other issues, contact {aws} Support. {aws} Support can guide you on downloading and running a log collection tool. That way, you can collect logs from your [.noloc]`Kubernetes` cluster control plane instances and send them to {aws} Support support for further investigation. +When an Outpost gets disconnected from the {aws} Region that it's associated with, the Kubernetes cluster likely will continue working normally. However, if the cluster doesn't work properly, follow the troubleshooting steps in <>. If you encounter other issues, contact {aws} Support. {aws} Support can guide you on downloading and running a log collection tool. That way, you can collect logs from your Kubernetes cluster control plane instances and send them to {aws} Support support for further investigation. ==== -[[outposts-troubleshooting-control-plane-instances-ssm,outposts-troubleshooting-control-plane-instances-ssm.title]] +[#outposts-troubleshooting-control-plane-instances-ssm] .Control plane instances aren't reachable through {aws} Systems Manager [%collapsible] ==== @@ -271,5 +299,5 @@ When the Amazon EKS control plane instances aren't reachable through {aws} Syste Amazon EKS control plane instances are not reachable through SSM. Please verify your SSM and network configuration, and reference the EKS on Outposts troubleshooting documentation. ---- -To resolve this issue, make sure that your VPC and subnets meet the requirements in <> and that you completed the steps in link:systems-manager/latest/userguide/session-manager-getting-started.html[Setting up Session Manager,type="documentation"] in the {aws} Systems Manager User Guide. -==== +To resolve this issue, make sure that your VPC and subnets meet the requirements in <> and that you completed the steps in link:systems-manager/latest/userguide/session-manager-getting-started.html[Setting up Session Manager,type="documentation"] in the {aws} Systems Manager User Guide. +==== \ No newline at end of file diff --git a/latest/ug/outposts/eks-outposts-vpc-subnet-requirements.adoc b/latest/ug/outposts/eks-outposts-vpc-subnet-requirements.adoc index 801a99dc6..ed8ecef98 100644 --- a/latest/ug/outposts/eks-outposts-vpc-subnet-requirements.adoc +++ b/latest/ug/outposts/eks-outposts-vpc-subnet-requirements.adoc @@ -1,8 +1,7 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[eks-outposts-vpc-subnet-requirements,eks-outposts-vpc-subnet-requirements.title]] +[#eks-outposts-vpc-subnet-requirements] = Create a VPC and subnets for Amazon EKS clusters on {aws} Outposts :info_titleabbrev: Create a VPC and subnets @@ -13,48 +12,48 @@ Learn about VPC and subnet requirements and considerations, then to create a VPC When you create a local cluster, you specify a VPC and at least one private subnet that runs on Outposts. This topic provides an overview of the VPC and subnets requirements and considerations for your local cluster. -[[outposts-vpc-requirements,outposts-vpc-requirements.title]] +[#outposts-vpc-requirements] == VPC requirements and considerations When you create a local cluster, the VPC that you specify must meet the following requirements and considerations: -* Make sure that the VPC has enough IP addresses for the local cluster, any nodes, and other [.noloc]`Kubernetes` resources that you want to create. If the VPC that you want to use doesn't have enough IP addresses, increase the number of available IP addresses. You can do this by link:vpc/latest/userguide/working-with-vpcs.html#add-ipv4-cidr[associating additional Classless Inter-Domain Routing (CIDR) blocks,type="documentation"] with your VPC. You can associate private (RFC 1918) and public (non-RFC 1918) CIDR blocks to your VPC either before or after you create your cluster. It can take a cluster up to 5 hours for a CIDR block that you associated with a VPC to be recognized. +* Make sure that the VPC has enough IP addresses for the local cluster, any nodes, and other Kubernetes resources that you want to create. If the VPC that you want to use doesn't have enough IP addresses, increase the number of available IP addresses. You can do this by link:vpc/latest/userguide/working-with-vpcs.html#add-ipv4-cidr[associating additional Classless Inter-Domain Routing (CIDR) blocks,type="documentation"] with your VPC. You can associate private (RFC 1918) and public (non-RFC 1918) CIDR blocks to your VPC either before or after you create your cluster. It can take a cluster up to 5 hours for a CIDR block that you associated with a VPC to be recognized. * The VPC can't have assigned IP prefixes or IPv6 CIDR blocks. Because of these constraints, the information that's covered in <> and <> isn't applicable to your VPC. * The VPC has a DNS hostname and DNS resolution enabled. Without these features, the local cluster fails to create, and you need to enable the features and recreate your cluster. For more information, see link:vpc/latest/userguide/vpc-dns.html[DNS attributes for your VPC,type="documentation"] in the Amazon VPC User Guide. -* To access your local cluster over your local network, the VPC must be associated with your Outpost's local gateway route table. For more information, see link:outposts/latest/userguide/outposts-local-gateways.html#vpc-associations[VPC associations,type="documentation"] in the {aws} Outposts User Guide. +* To access your local cluster over your local network, the VPC must be associated with your Outpost's local gateway route table. For more information, see link:outposts/latest/userguide/outposts-local-gateways.html#vpc-associations[VPC associations,type="documentation"] in the {aws} Outposts User Guide. -[[outposts-subnet-requirements,outposts-subnet-requirements.title]] +[#outposts-subnet-requirements] == Subnet requirements and considerations -When you create the cluster, specify at least one private subnet. If you specify more than one subnet, the [.noloc]`Kubernetes` control plane instances are evenly distributed across the subnets. If more than one subnet is specified, the subnets must exist on the same Outpost. Moreover, the subnets must also have proper routes and security group permissions to communicate with each other. When you create a local cluster, the subnets that you specify must meet the following requirements: +When you create the cluster, specify at least one private subnet. If you specify more than one subnet, the Kubernetes control plane instances are evenly distributed across the subnets. If more than one subnet is specified, the subnets must exist on the same Outpost. Moreover, the subnets must also have proper routes and security group permissions to communicate with each other. When you create a local cluster, the subnets that you specify must meet the following requirements: * The subnets are all on the same logical Outpost. -* The subnets together have at least three available IP addresses for the [.noloc]`Kubernetes` control plane instances. If three subnets are specified, each subnet must have at least one available IP address. If two subnets are specified, each subnet must have at least two available IP addresses. If one subnet is specified, the subnet must have at least three available IP addresses. -* The subnets have a route to the Outpost rack's link:outposts/latest/userguide/outposts-local-gateways.html[local gateway,type="documentation"] to access the [.noloc]`Kubernetes` API server over your local network. If the subnets don't have a route to the Outpost rack's local gateway, you must communicate with your [.noloc]`Kubernetes` API server from within the VPC. -* The subnets must use IP address-based naming. Amazon EC2 link:AWSEC2/latest/UserGuide/ec2-instance-naming.html#instance-naming-rbn[resource-based naming,type="documentation"] isn't supported by Amazon EKS. +* The subnets together have at least three available IP addresses for the Kubernetes control plane instances. If three subnets are specified, each subnet must have at least one available IP address. If two subnets are specified, each subnet must have at least two available IP addresses. If one subnet is specified, the subnet must have at least three available IP addresses. +* The subnets have a route to the Outpost rack's link:outposts/latest/userguide/outposts-local-gateways.html[local gateway,type="documentation"] to access the Kubernetes API server over your local network. If the subnets don't have a route to the Outpost rack's local gateway, you must communicate with your Kubernetes API server from within the VPC. +* The subnets must use IP address-based naming. Amazon EC2 link:AWSEC2/latest/UserGuide/ec2-instance-naming.html#instance-naming-rbn[resource-based naming,type="documentation"] isn't supported by Amazon EKS. -[[subnet-access-to-services,subnet-access-to-services.title]] +[#subnet-access-to-services] == Subnet access to {aws} services The local cluster's private subnets on Outposts must be able to communicate with Regional {aws} services. You can achieve this by using a link:vpc/latest/userguide/vpc-nat-gateway.html[NAT gateway,type="documentation"] for outbound internet access or, if you want to keep all traffic private within your VPC, using link:vpc/latest/privatelink/create-interface-endpoint.html[interface VPC endpoints,type="documentation"]. -[[subnet-access-nat-gateway,subnet-access-nat-gateway.title]] +[#subnet-access-nat-gateway] === Using a NAT gateway -The local cluster's private subnets on Outposts must have an associated route table that has a route to a NAT gateway in a public subnet that is in the Outpost's parent Availability Zone. The public subnet must have a route to an link:vpc/latest/userguide/VPC_Internet_Gateway.html[internet gateway,type="documentation"]. The NAT gateway enables outbound internet access and prevents unsolicited inbound connections from the internet to instances on the Outpost. +The local cluster's private subnets on Outposts must have an associated route table that has a route to a NAT gateway in a public subnet that is in the Outpost's parent Availability Zone. The public subnet must have a route to an link:vpc/latest/userguide/VPC_Internet_Gateway.html[internet gateway,type="documentation"]. The NAT gateway enables outbound internet access and prevents unsolicited inbound connections from the internet to instances on the Outpost. -[[vpc-subnet-requirements-vpc-endpoints,vpc-subnet-requirements-vpc-endpoints.title]] +[#vpc-subnet-requirements-vpc-endpoints] === Using interface VPC endpoints If the local cluster's private subnets on Outposts don't have an outbound internet connection, or if you want to keep all traffic private within your VPC, then you must create the following interface VPC endpoints and link:vpc/latest/privatelink/gateway-endpoints.html[gateway endpoint,type="documentation"] in a Regional subnet before creating your cluster. -[cols="1,1", options="header"] +[%header,cols="2"] |=== |Endpoint |Endpoint type @@ -99,9 +98,9 @@ The endpoints must meet the following requirements: * Have private DNS names enabled * Have an attached security group that permits inbound HTTPS traffic from the CIDR range of the private outpost subnet. -Creating endpoints incurs charges. For more information, see link:privatelink/pricing/[{aws} PrivateLink pricing,type="marketing"]. If your [.noloc]`Pods` need access to other {aws} services, then you need to create additional endpoints. For a comprehensive list of endpoints, see link:vpc/latest/privatelink/aws-services-privatelink-support.html[{aws} services that integrate with {aws} PrivateLink,type="documentation"]. +Creating endpoints incurs charges. For more information, see link:privatelink/pricing/[{aws} PrivateLink pricing,type="marketing"]. If your Pods need access to other {aws} services, then you need to create additional endpoints. For a comprehensive list of endpoints, see link:vpc/latest/privatelink/aws-services-privatelink-support.html[{aws} services that integrate with {aws} PrivateLink,type="documentation"]. -[[outposts-create-vpc,outposts-create-vpc.title]] +[#outposts-create-vpc] == Create a VPC You can create a VPC that meets the previous requirements using one of the following {aws} CloudFormation templates: @@ -109,4 +108,4 @@ You can create a VPC that meets the previous requirements using one of the follo * *https://s3.us-west-2.amazonaws.com/amazon-eks/cloudformation/2022-09-20/amazon-eks-local-outposts-vpc-subnet.yaml[Template 1]* – This template creates a VPC with one private subnet on the Outpost and one public subnet in the {aws} Region. The private subnet has a route to an internet through a NAT Gateway that resides in the public subnet in the {aws} Region. This template can be used to create a local cluster in a subnet with egress internet access. -* *https://s3.us-west-2.amazonaws.com/amazon-eks/cloudformation/2023-03-20/amazon-eks-local-outposts-fully-private-vpc-subnet.yaml[Template 2]* – This template creates a VPC with one private subnet on the Outpost and the minimum set of VPC Endpoints required to create a local cluster in a subnet that doesn't have ingress or egress internet access (also referred to as a private subnet). +* *https://s3.us-west-2.amazonaws.com/amazon-eks/cloudformation/2023-03-20/amazon-eks-local-outposts-fully-private-vpc-subnet.yaml[Template 2]* – This template creates a VPC with one private subnet on the Outpost and the minimum set of VPC Endpoints required to create a local cluster in a subnet that doesn't have ingress or egress internet access (also referred to as a private subnet). \ No newline at end of file diff --git a/latest/ug/outposts/eks-outposts.adoc b/latest/ug/outposts/eks-outposts.adoc index 13a940afc..1df3b6bae 100644 --- a/latest/ug/outposts/eks-outposts.adoc +++ b/latest/ug/outposts/eks-outposts.adoc @@ -1,34 +1,22 @@ -//!!NODE_ROOT include::../attributes.txt[] -[[eks-outposts,eks-outposts.title]] + +[#eks-outposts] = Deploy Amazon EKS on-premises with {aws} Outposts -:doctype: book -:sectnums: -:toc: left -:icons: font -:experimental: -:idprefix: -:idseparator: - -:sourcedir: . -:info_doctype: chapter -:info_title: Deploy Amazon EKS on-premises with {aws} Outposts :info_titleabbrev: Amazon EKS on {aws} Outposts -:keywords: Amazon EKS, {aws} Outposts, extended clusters, local clusters -:info_abstract: Learn to deploy Amazon EKS on {aws} Outposts for local or extended clusters to run on-premises Kubernetes applications with a fully managed control plane. [abstract] -- -Learn to deploy Amazon EKS on {aws} Outposts for local or extended clusters to run on-premises [.noloc]`Kubernetes` applications with a fully managed control plane. +Learn to deploy Amazon EKS on {aws} Outposts for local or extended clusters to run on-premises Kubernetes applications with a fully managed control plane. -- -You can use Amazon EKS to run on-premises [.noloc]`Kubernetes` applications on {aws} Outposts. You can deploy Amazon EKS on Outposts in the following ways: +You can use Amazon EKS to run on-premises Kubernetes applications on {aws} Outposts. You can deploy Amazon EKS on Outposts in the following ways: -* *Extended clusters* – Run the [.noloc]`Kubernetes` control plane in an {aws} Region and nodes on your Outpost. -* *Local clusters* – Run the [.noloc]`Kubernetes` control plane and nodes on your Outpost. +* *Extended clusters* – Run the Kubernetes control plane in an {aws} Region and nodes on your Outpost. +* *Local clusters* – Run the Kubernetes control plane and nodes on your Outpost. -For both deployment options, the [.noloc]`Kubernetes` control plane is fully managed by {aws}. You can use the same Amazon EKS APIs, tools, and console that you use in the cloud to create and run Amazon EKS on Outposts. +For both deployment options, the Kubernetes control plane is fully managed by {aws}. You can use the same Amazon EKS APIs, tools, and console that you use in the cloud to create and run Amazon EKS on Outposts. The following diagram shows these deployment options. @@ -37,45 +25,45 @@ The following diagram shows these deployment options. image::images/outposts-deployment-options.png[Outpost deployment options ,scaledwidth=100%] -[[outposts-overview-when-deployment-options,outposts-overview-when-deployment-options.title]] +[#outposts-overview-when-deployment-options] == When to use each deployment option Both local and extended clusters are general-purpose deployment options and can be used for a range of applications. With local clusters, you can run the entire Amazon EKS cluster locally on Outposts. This option can mitigate the risk of application downtime that might result from temporary network disconnects to the cloud. These network disconnects can be caused by fiber cuts or weather events. Because the entire Amazon EKS cluster runs locally on Outposts, applications remain available. You can perform cluster operations during network disconnects to the cloud. For more information, see <>. If you're concerned about the quality of the network connection from your Outposts to the parent {aws} Region and require high availability through network disconnects, use the local cluster deployment option. -With extended clusters, you can conserve capacity on your Outpost because the [.noloc]`Kubernetes` control plane runs in the parent {aws} Region. This option is suitable if you can invest in reliable, redundant network connectivity from your Outpost to the {aws} Region. The quality of the network connection is critical for this option. The way that [.noloc]`Kubernetes` handles network disconnects between the [.noloc]`Kubernetes` control plane and nodes might lead to application downtime. For more information on the behavior of [.noloc]`Kubernetes`, see https://kubernetes.io/docs/concepts/scheduling-eviction/[Scheduling, Preemption, and Eviction] in the [.noloc]`Kubernetes` documentation. +With extended clusters, you can conserve capacity on your Outpost because the Kubernetes control plane runs in the parent {aws} Region. This option is suitable if you can invest in reliable, redundant network connectivity from your Outpost to the {aws} Region. The quality of the network connection is critical for this option. The way that Kubernetes handles network disconnects between the Kubernetes control plane and nodes might lead to application downtime. For more information on the behavior of Kubernetes, see https://kubernetes.io/docs/concepts/scheduling-eviction/[Scheduling, Preemption, and Eviction] in the Kubernetes documentation. -[[outposts-overview-comparing-deployment-options,outposts-overview-comparing-deployment-options.title]] +[#outposts-overview-comparing-deployment-options] == Comparing the deployment options The following table compares the differences between the two options. -[cols="1,1,1", options="header"] +[%header,cols="3"] |=== |Feature |Extended cluster |Local cluster -|[.noloc]`Kubernetes` control plane location +|Kubernetes control plane location |{aws} Region |Outpost -|[.noloc]`Kubernetes` control plane account +|Kubernetes control plane account |{aws} account |Your account |Regional availability -|See link:general/latest/gr/eks.html#eks_region[Service endpoints,type="documentation"] +|see link:general/latest/gr/eks.html#eks_region[Service endpoints,type="documentation"] |US East (Ohio), US East (N. Virginia), US West (N. California), US West (Oregon), Asia Pacific (Seoul), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), Europe (London), Middle East (Bahrain), and South America (São Paulo) |Kubernetes minor versions -|<>. -|<>. +|eks/latest/userguide/kubernetes-versions.html[Supported Amazon EKS versions,type="documentation"]. +|eks/latest/userguide/kubernetes-versions.html[Supported Amazon EKS versions,type="documentation"]. |Platform versions -|See <> +|See link:eks/latest/userguide/platform-versions.html[EKS platform-versions,type="documentation"] |See <> |Outpost form factors @@ -98,8 +86,8 @@ The following table compares the differences between the two options. |Public or private or both |Private only -|[.noloc]`Kubernetes` API server authentication -|{aws} Identity and Access Management (IAM) and [.noloc]`OIDC` +|Kubernetes API server authentication +|{aws} Identity and Access Management (IAM) and OIDC |IAM and `x.509` certificates |Node types @@ -127,8 +115,8 @@ The following table compares the differences between the two options. |Self-managed add-ons only |Default Container Network Interface -|[.noloc]`Amazon VPC CNI plugin for Kubernetes` -|[.noloc]`Amazon VPC CNI plugin for Kubernetes` +|Amazon VPC CNI plugin for Kubernetes +|Amazon VPC CNI plugin for Kubernetes |Kubernetes control plane logs |Amazon CloudWatch Logs @@ -157,4 +145,4 @@ The following table compares the differences between the two options. include::eks-outposts-local-cluster-overview.adoc[leveloffset=+1] -include::eks-outposts-self-managed-nodes.adoc[leveloffset=+1] +include::eks-outposts-self-managed-nodes.adoc[leveloffset=+1] \ No newline at end of file diff --git a/latest/ug/quickstart.adoc b/latest/ug/quickstart.adoc index 475439e67..b85035c60 100644 --- a/latest/ug/quickstart.adoc +++ b/latest/ug/quickstart.adoc @@ -1,14 +1,9 @@ -//!!NODE_ROOT +include::attributes.txt[] + [.topic] -[[quickstart,quickstart.title]] +[#quickstart] = Quickstart: Deploy a web app and store data -:info_doctype: chapter -:info_title: Quickstart: Deploy a web app and store data :info_titleabbrev: Quickstart -:keywords: quickstart, web, cluster -:info_abstract: Deploy a game application and persist its data on Amazon EKS - -include::attributes.txt[] [abstract] -- @@ -25,23 +20,23 @@ As we progress, we'll walk you through the cluster setup process. Amazon EKS Aut Overall, you'll deploy a sample workload with the custom annotations required to fully integrate with {aws} services. -## In this tutorial +== In this tutorial Using the `eksctl` cluster template that follows, you'll build a cluster with EKS Auto Mode for automated node provisioning. -**VPC Configuration** +*VPC Configuration* When using the eksctl cluster template that follows, eksctl automatically creates an IPv4 Virtual Private Cloud (VPC) for the cluster. By default, eksctl configures a VPC that addresses all networking requirements, in addition to creating both public and private endpoints. -**Instance Management** +*Instance Management* EKS Auto Mode dynamically adds or removes nodes in your EKS cluster based on the demands of your Kubernetes applications. -**Data Persistence** +*Data Persistence* Use the block storage capability of EKS Auto Mode to ensure the persistence of application data, even in scenarios involving pod restarts or failures. -**External App Access** +*External App Access* Use the load balancing capability of EKS Auto Mode to dynamically provision an Application Load Balancer (ALB). -## Prerequisites +== Prerequisites Before you begin, ensure you have the following prerequisites set up to use Amazon EKS: @@ -51,7 +46,7 @@ Before you begin, ensure you have the following prerequisites set up to use Amaz For more information, see <>. -## Configure the cluster +== Configure the cluster In this section, you'll create a cluster using EKS Auto Mode for dynamic node provisioning. @@ -82,7 +77,7 @@ If you do not use eksctl to create the cluster, you need to manually tag the VPC ==== -## Create IngressClass +== Create IngressClass Create a Kubernetes `IngressClass` for EKS Auto Mode. The IngressClass defines how EKS Auto Mode handles Ingress resources. This step configures the load balancing capability of EKS Auto Mode. When you create Ingress resources for your applications, EKS Auto Mode uses this IngressClass to automatically provision and manage load balancers, integrating your Kubernetes applications with {aws} load balancing services. @@ -105,9 +100,14 @@ Apply the IngressClass to your cluster: kubectl apply -f ingressclass.yaml ``` -## Deploy the 2048 game sample application +== Deploy the 2048 game sample application + +In this section, we walk you through the steps to deploy the popular "`2048 game`" as a sample application within the cluster. The provided manifest includes custom annotations for the Application Load Balancer (ALB). These annotations integrate with and instruct the EKS to handle incoming HTTP traffic as "internet-facing" and route it to the appropriate service in the `game-2048` namespace using the target type "ip". -In this section, we walk you through the steps to deploy the popular "`2048 game`" as a sample application within the cluster. The provided manifest includes custom annotations for the Application Load Balancer (ALB). These annotations integrate with and instruct the EKS to handle incoming HTTP traffic as "internet-facing" and route it to the appropriate service in the 'game-2048' namespace using the target type "ip". +[NOTE] +==== +The `docker-2048` image in the example is an `x86_64` container image and will not run on other architectures. +==== . Create a Kubernetes namespace called `game-2048` with the `--save-config` flag. + @@ -147,18 +147,18 @@ kubectl get ingress -n game-2048 + You should see the following response output: + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- NAME CLASS HOSTS ADDRESS PORTS AGE -ingress-2048 alb * k8s-game2048-ingress2-eb379a0f83-378466616.[.replaceable]`region-code`.elb.amazonaws.com 80 31s +ingress-2048 alb * k8s-game2048-ingress2-eb379a0f83-378466616.region-code.elb.amazonaws.com 80 31s ---- + You'll need to wait several minutes for the Application Load Balancer (ALB) to provision before you begin the following steps. . Open a web browser and enter the `ADDRESS` from the previous step to access the web application. For example: + -[source,bash,subs="verbatim,attributes,quotes"] +[source,bash,subs="verbatim,attributes"] ---- -k8s-game2048-ingress2-eb379a0f83-378466616.[.replaceable]`region-code`.elb.amazonaws.com +k8s-game2048-ingress2-eb379a0f83-378466616.region-code.elb.amazonaws.com ---- + You should see the 2048 game in your browser. Play! @@ -166,7 +166,7 @@ You should see the 2048 game in your browser. Play! image::images/quick2048.png[Play the 2048 game,scaledwidth=25%] -## Persist Data using Amazon EKS Auto Mode +== Persist Data using Amazon EKS Auto Mode Now that the 2048 game is up and running on your Amazon EKS cluster, it's time to ensure that your game data is safely persisted using the block storage capability of Amazon EKS Auto Mode. @@ -186,7 +186,7 @@ parameters: type: gp3 encrypted: "true" ---- -. Apply the StorageClass: +. Apply the `StorageClass`: + [source,bash] ---- @@ -274,7 +274,7 @@ With these steps, your 2048 game on the cluster is now set up to persist data us If you liked this tutorial, let us know by providing feedback so we're able to provide you with more use case-specific quickstart tutorials like this one. -## Delete your cluster and nodes +== Delete your cluster and nodes After you've finished with the cluster that you created for this tutorial, you should clean up by deleting the cluster with the following command. If you want to do more with this cluster before you clean up, see Next steps. @@ -283,4 +283,3 @@ eksctl delete cluster -f ./cluster-config.yaml ``` EKS will automatically clean up any nodes it provisioned when the cluster is deleted. - diff --git a/latest/ug/related-projects.adoc b/latest/ug/related-projects.adoc index 7f68f7e77..66a5ed7a7 100644 --- a/latest/ug/related-projects.adoc +++ b/latest/ug/related-projects.adoc @@ -1,33 +1,40 @@ -//!!NODE_ROOT include::attributes.txt[] -[[related-projects,related-projects.title]] + +[#related-projects] = Extend Amazon EKS capabilities with open source projects -:doctype: book -:sectnums: -:toc: left -:icons: font -:experimental: -:idprefix: -:idseparator: - -:sourcedir: . -:info_doctype: chapter -:info_title: Extend Amazon EKS capabilities with open source projects :info_titleabbrev: Projects related to Amazon EKS -:info_abstract: Learn how to use open source projects to add features to Amazon EKS for management, networking, machine learning, auto-scaling, monitoring, and CI/CD. [abstract] -- Learn how to use open source projects to add features to Amazon EKS for management, networking, machine learning, auto-scaling, monitoring, and CI/CD. -- -These open-source projects extend the functionality of [.noloc]`Kubernetes` clusters running on or outside of {aws}, including clusters managed by Amazon EKS. +These open-source projects extend the functionality of Kubernetes clusters running on or outside of {aws}, including clusters managed by Amazon EKS. + +[#oss-scope] +== Support for software deployed to EKS + +When reviewing the Amazon EKS docs, you'll encounter references to various open-source tools and software throughout our procedures and examples. These tools include the https://github.com/kubernetes-sigs/metrics-server[Kubernetes Metrics Server] and https://cert-manager.io/[Cert Manager.] + +Please note that any third-party or open-source software you choose to deploy falls outside the scope of your {aws} Support Agreements. A benefit of using Kubernetes is the active open source community. We recommend working directly with the relevant open-source communities and project maintainers to establish appropriate support channels for such components. For more information, see the https://www.cncf.io/projects/[graduated and incubating projects] associated with the Cloud Native Computing Foundation (CNCF). + +The Kubernetes ecosystem includes numerous projects and components that come with different levels of community support, response times, and intended use cases. When implementing these technologies alongside EKS, ensure you understand the support matrix for each component. + +{aws} maintains the open-source components we integrate into the EKS control plane. This includes our comprehensive security pipeline covering build verification, vulnerability scanning, validation testing, and patch management for all container images and binaries we distribute. For example, {aws} is responsible for the https://kubernetes.io/docs/concepts/architecture/#kube-apiserver[Kubernetes API Server]. The Kubernetes API server is covered by link:eks/sla/["Amazon EKS Service Level Agreement",type="marketing"]. You can use your link:premiumsupport/plans/["Amazon Web Services Support Plan",type="marketing"] to resolve issues with the Kubernetes API server, or get general guidance. -[[related-management-tools,related-management-tools.title]] +You need to carefully review the support offered for various Amazon EKS Add-ons. {aws} add-ons are the only type of Amazon EKS add-on that are fully supported by {aws}. {aws} Marketplace add-ons are primarily supported by {aws} Partners. Community add-ons receive basic lifecycle support from {aws}. For more information, see xref:addon-support[add-on Support.] + +Every EKS add-ons, irrespective of the type, receives basic lifecycle support from EKS including Marketplace add-ons. Basic lifecycle support includes installing and uninstalling the add-on. For more information on the types of Amazon EKS Add-ons available and the associated levels of support, see xref:addon-support[Scope of Support for Amazon EKS add-ons.] To view add-ons fully supported by {aws}, see xref:workloads-add-ons-available-eks[Amazon Web Services add-ons.] + +* For more information about our security practices and support boundaries, see xref:security[Security in Amazon EKS.] +* For more information about community and {aws} marketplace add-ons available through Amazon EKS Add-ons, see xref:addon-support["EKS Add-ons Support",type="documentation"]. + +[#related-management-tools] == Management tools -Related management tools for Amazon EKS and [.noloc]`Kubernetes` clusters. +Related management tools for Amazon EKS and Kubernetes clusters. -[[related-eksctl,related-eksctl.title]] +[#related-eksctl] === eksctl `eksctl` is a simple CLI tool for creating clusters on Amazon EKS. @@ -39,10 +46,10 @@ Related management tools for Amazon EKS and [.noloc]`Kubernetes` clusters. * {aws} open source blog: link:opensource/eksctl-eks-cluster-one-command[eksctl: Amazon EKS cluster with one command,type="blog"] -[[related-aws-controllers,related-aws-controllers.title]] -=== {aws} controllers for [.noloc]`Kubernetes` +[#related-aws-controllers] +=== {aws} controllers for Kubernetes -With {aws} Controllers for [.noloc]`Kubernetes`, you can create and manage {aws} resources directly from your [.noloc]`Kubernetes` cluster. +With {aws} Controllers for Kubernetes, you can create and manage {aws} resources directly from your Kubernetes cluster. @@ -50,10 +57,10 @@ With {aws} Controllers for [.noloc]`Kubernetes`, you can create and manage {aws} * {aws} open source blog: link:opensource/aws-service-operator-kubernetes-available[{aws} service operator for Kubernetes now available,type="blog"] -[[related-flux-cd,related-flux-cd.title]] +[#related-flux-cd] === Flux CD -Flux is a tool that you can use to manage your cluster configuration using Git. It uses an operator in the cluster to trigger deployments inside of [.noloc]`Kubernetes`. For more information about operators, see https://operatorhub.io/[OperatorHub.io] on [.noloc]`GitHub`. +Flux is a tool that you can use to manage your cluster configuration using Git. It uses an operator in the cluster to trigger deployments inside of Kubernetes. For more information about operators, see https://operatorhub.io/[OperatorHub.io] on GitHub. @@ -61,10 +68,10 @@ Flux is a tool that you can use to manage your cluster configuration using Git. * https://docs.fluxcd.io/[Project documentation] -[[related-cdk,related-cdk.title]] -=== CDK for [.noloc]`Kubernetes` +[#related-cdk] +=== CDK for Kubernetes -With the CDK for [.noloc]`Kubernetes` (cdk8s), you can define [.noloc]`Kubernetes` apps and components using familiar programming languages. cdk8s apps synthesize into standard [.noloc]`Kubernetes` manifests, which can be applied to any [.noloc]`Kubernetes` cluster. +With the CDK for Kubernetes (cdk8s), you can define Kubernetes apps and components using familiar programming languages. cdk8s apps synthesize into standard Kubernetes manifests, which can be applied to any Kubernetes cluster. @@ -73,15 +80,15 @@ With the CDK for [.noloc]`Kubernetes` (cdk8s), you can define [.noloc]`Kubernete * {aws} containers blog: link:containers/introducing-cdk8s-intent-driven-apis-for-kubernetes-objects[Introducing cdk8s+: Intent-driven APIs for Kubernetes objects,type="blog"] -[[related-networking,related-networking.title]] +[#related-networking] == Networking -Related networking projects for Amazon EKS and [.noloc]`Kubernetes` clusters. +Related networking projects for Amazon EKS and Kubernetes clusters. -[[related-vpc-cni-k8s,related-vpc-cni-k8s.title]] -=== [.noloc]`Amazon VPC CNI plugin for Kubernetes` +[#related-vpc-cni-k8s] +=== Amazon VPC CNI plugin for Kubernetes -Amazon EKS supports native VPC networking through the [.noloc]`Amazon VPC CNI plugin for Kubernetes`. The plugin assigns an IP address from your VPC to each [.noloc]`Pod`. +Amazon EKS supports native VPC networking through the Amazon VPC CNI plugin for Kubernetes. The plugin assigns an IP address from your VPC to each Pod. @@ -89,10 +96,10 @@ Amazon EKS supports native VPC networking through the [.noloc]`Amazon VPC CNI pl * https://github.com/aws/amazon-vpc-cni-k8s/blob/master/README.md[Project documentation] -[[related-alb-ingress-controller,related-alb-ingress-controller.title]] -=== [.noloc]`{aws} Load Balancer Controller` for [.noloc]`Kubernetes` +[#related-alb-ingress-controller] +=== {aws} Load Balancer Controller for Kubernetes -The [.noloc]`{aws} Load Balancer Controller` helps manage {aws} Elastic Load Balancers for a [.noloc]`Kubernetes` cluster. It satisfies [.noloc]`Kubernetes` Ingress resources by provisioning {aws} Application Load Balancers. It satisfies [.noloc]`Kubernetes` service resources by provisioning {aws} Network Load Balancers. +The {aws} Load Balancer Controller helps manage {aws} Elastic Load Balancers for a Kubernetes cluster. It satisfies Kubernetes Ingress resources by provisioning {aws} Application Load Balancers. It satisfies Kubernetes service resources by provisioning {aws} Network Load Balancers. @@ -100,10 +107,10 @@ The [.noloc]`{aws} Load Balancer Controller` helps manage {aws} Elastic Load Bal * https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/[Project documentation] -[[related-externaldns,related-externaldns.title]] +[#related-externaldns] === ExternalDNS -ExternalDNS synchronizes exposed [.noloc]`Kubernetes` services and ingresses with DNS providers including Amazon Route 53 and {aws} Service Discovery. +ExternalDNS synchronizes exposed Kubernetes services and ingresses with DNS providers including Amazon Route 53 and {aws} Service Discovery. @@ -111,15 +118,15 @@ ExternalDNS synchronizes exposed [.noloc]`Kubernetes` services and ingresses wit * https://github.com/kubernetes-incubator/external-dns/blob/master/docs/tutorials/aws.md[Project documentation] -[[related-machine-learning,related-machine-learning.title]] +[#related-machine-learning] == Machine learning -Related machine learning projects for Amazon EKS and [.noloc]`Kubernetes` clusters. +Related machine learning projects for Amazon EKS and Kubernetes clusters. -[[related-kubeflow,related-kubeflow.title]] +[#related-kubeflow] === Kubeflow -A machine learning toolkit for [.noloc]`Kubernetes`. +A machine learning toolkit for Kubernetes. @@ -128,15 +135,15 @@ A machine learning toolkit for [.noloc]`Kubernetes`. * {aws} open source blog: link:opensource/kubeflow-amazon-eks[Kubeflow on Amazon EKS,type="blog"] -[[related-auto-scaling,related-auto-scaling.title]] +[#related-auto-scaling] == Auto Scaling -Related auto scaling projects for Amazon EKS and [.noloc]`Kubernetes` clusters. +Related auto scaling projects for Amazon EKS and Kubernetes clusters. -[[related-cluster-autoscaler,related-cluster-autoscaler.title]] +[#related-cluster-autoscaler] === Cluster autoscaler -Cluster Autoscaler is a tool that automatically adjusts the size of the [.noloc]`Kubernetes` cluster based on CPU and memory pressure. +Cluster Autoscaler is a tool that automatically adjusts the size of the Kubernetes cluster based on CPU and memory pressure. @@ -145,10 +152,10 @@ Cluster Autoscaler is a tool that automatically adjusts the size of the [.noloc] * Amazon EKS workshop: https://www.eksworkshop.com/docs/autoscaling/compute/cluster-autoscaler/[Cluster Autoscaler] -[[related-karpenter,related-karpenter.title]] +[#related-karpenter] === Karpenter -Karpenter is a [.noloc]`Kubernetes` Node Autoscaler built for flexibility, performance, and simplicity. +Karpenter is a Kubernetes Node Autoscaler built for flexibility, performance, and simplicity. @@ -157,10 +164,10 @@ Karpenter is a [.noloc]`Kubernetes` Node Autoscaler built for flexibility, perfo * Amazon EKS workshop: https://www.eksworkshop.com/docs/autoscaling/compute/karpenter/[Karpenter] -[[related-escalator,related-escalator.title]] +[#related-escalator] === Escalator -Escalator is a batch or job optimized horizontal autoscaler for [.noloc]`Kubernetes`. +Escalator is a batch or job optimized horizontal autoscaler for Kubernetes. @@ -168,15 +175,15 @@ Escalator is a batch or job optimized horizontal autoscaler for [.noloc]`Kuberne * https://github.com/atlassian/escalator/blob/master/docs/README.md[Project documentation] -[[related-monitoring,related-monitoring.title]] +[#related-monitoring] == Monitoring -Related monitoring projects for Amazon EKS and [.noloc]`Kubernetes` clusters. +Related monitoring projects for Amazon EKS and Kubernetes clusters. -[[related-prometheus,related-prometheus.title]] -=== [.noloc]`Prometheus` +[#related-prometheus] +=== Prometheus -[.noloc]`Prometheus` is an open-source systems monitoring and alerting toolkit. +Prometheus is an open-source systems monitoring and alerting toolkit. @@ -185,20 +192,17 @@ Related monitoring projects for Amazon EKS and [.noloc]`Kubernetes` clusters. * Amazon EKS workshop: https://eksworkshop.com/intermediate/240_monitoring/[https://eksworkshop.com/intermediate/240_monitoring/] -[[related-cicd,related-cicd.title]] +[#related-cicd] == Continuous integration / continuous deployment -Related CI/CD projects for Amazon EKS and [.noloc]`Kubernetes` clusters. +Related CI/CD projects for Amazon EKS and Kubernetes clusters. -[[related-jenkinsx,related-jenkinsx.title]] +[#related-jenkinsx] === Jenkins X -CI/CD solution for modern cloud applications on Amazon EKS and [.noloc]`Kubernetes` clusters. +CI/CD solution for modern cloud applications on Amazon EKS and Kubernetes clusters. * https://jenkins-x.io/[Project URL] -* https://jenkins-x.io/docs/[Project documentation] - - -📝 https://github.com/search?q=repo:awsdocs/amazon-eks-user-guide+[[related-projects,&type=code[Edit this page on GitHub] \ No newline at end of file +* https://jenkins-x.io/docs/[Project documentation] \ No newline at end of file diff --git a/latest/ug/roadmap.adoc b/latest/ug/roadmap.adoc index 227222e7b..0ab1ac917 100644 --- a/latest/ug/roadmap.adoc +++ b/latest/ug/roadmap.adoc @@ -1,24 +1,13 @@ -//!!NODE_ROOT include::attributes.txt[] + [.topic] -[[roadmap,roadmap.title]] +[#roadmap] = Learn about Amazon EKS new features and roadmap -:doctype: book -:sectnums: -:toc: left -:icons: font -:experimental: -:idprefix: -:idseparator: - -:sourcedir: . -:info_doctype: chapter -:info_title: Learn about Amazon EKS new features and roadmap :info_titleabbrev: New features and roadmap -:info_abstract: Follow the What's New feed and public roadmap to help plan for new and future Amazon EKS developments. [abstract] -- Follow the What's New feed and public roadmap to help plan for new and future Amazon EKS developments. -- -You can learn about new Amazon EKS features by scrolling to the What's New feed on the link:new/?whats-new-content-all.sort-by=item.additionalFields.postDateTime&whats-new-content-all.sort-order=desc&awsf.whats-new-compute=*all&awsf.whats-new-containers=general-products%23amazon-eks[What's New with {aws},type="marketing"] page. You can also review the https://github.com/aws/containers-roadmap/projects/1?card_filter_query=eks[roadmap] on [.noloc]`GitHub`, which lets you know about upcoming features and priorities so that you can plan how you want to use Amazon EKS in the future. You can provide direct feedback to us about the roadmap priorities. +You can learn about new Amazon EKS features by scrolling to the What's New feed on the link:new/?whats-new-content-all.sort-by=item.additionalFields.postDateTime&whats-new-content-all.sort-order=desc&awsf.whats-new-compute=*all&awsf.whats-new-containers=general-products%23amazon-eks[What's New with {aws},type="marketing"] page. You can also review the https://github.com/aws/containers-roadmap/projects/1?card_filter_query=eks[roadmap] on GitHub, which lets you know about upcoming features and priorities so that you can plan how you want to use Amazon EKS in the future. You can provide direct feedback to us about the roadmap priorities. \ No newline at end of file diff --git a/latest/ug/security/auto-security.adoc b/latest/ug/security/auto-security.adoc index ec163d415..832de7e61 100644 --- a/latest/ug/security/auto-security.adoc +++ b/latest/ug/security/auto-security.adoc @@ -1,59 +1,57 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[auto-security,auto-security.title]] +[#auto-security] = Security considerations for Amazon EKS Auto Mode -:info_doctype: section :info_titleabbrev: Considerations for EKS Auto -include::../attributes.txt[] - This topic describes the security architecture, controls, and best practices for Amazon EKS Auto Mode. As organizations deploy containerized applications at scale, maintaining a strong security posture becomes increasingly complex. EKS Auto Mode implements automated security controls and integrates with {aws} security services to help you protect your cluster infrastructure, workloads, and data. Through built-in security features like enforced node lifecycle management and automated patch deployment, EKS Auto Mode helps you maintain security best practices while reducing operational overhead. Before proceeding with this topic, make sure that you're familiar with basic EKS Auto Mode concepts and have reviewed the prerequisites for enabling EKS Auto Mode on your clusters. For general information about Amazon EKS security, see <>. Amazon EKS Auto Mode builds upon the existing security foundations of Amazon EKS while introducing additional automated security controls for EC2 managed instances. -## API security and authentication +== API security and authentication Amazon EKS Auto Mode uses {aws} platform security mechanisms to secure and authenticate calls to the Amazon EKS API. * Access to the Kubernetes API is secured through EKS access entries, which integrate with {aws} IAM identities. -** For more information, see link:eks/latest/userguide/access-entries.html["Grant IAM users access to Kubernetes with EKS access entries",type="documentation"]. +** For more information, see <>. * Customers can implement fine-grained access control to the Kubernetes API endpoint through configuration of EKS access entries. -## Network security +== Network security Amazon EKS Auto Mode supports multiple layers of network security: -* **VPC integration** +* *VPC integration* ** Operates within your Amazon Virtual Private Cloud (VPC) ** Supports custom VPC configurations and subnet layouts ** Enables private networking between cluster components ** For more information, see link:vpc/latest/userguide/security.html["Managing security responsibilities for Amazon Virtual Private Cloud",type="documentation"] -* **Network Policies** +* *Network Policies* ** Native support for Kubernetes Network Policies ** Ability to define granular network traffic rules -** For more information, see link:eks/latest/userguide/cni-network-policy.html["Limit pod traffic with Kubernetes network policies",type="documentation"] +** For more information, see <> -## EC2 managed instance security +== EC2 managed instance security Amazon EKS Auto Mode operates EC2 managed instances with the following security controls: -### EC2 security +=== EC2 security * EC2 managed instances maintain the security features of Amazon EC2. * For more information about EC2 managed instances, see link:AWSEC2/latest/UserGuide/ec2-security.html["Security in Amazon EC2",type="documentation"]. -### Instance lifecycle management +=== Instance lifecycle management EC2 managed instances operated by EKS Auto Mode have maximum lifetime of 21 days. Amazon EKS Auto Mode automatically terminates instances exceeding this lifetime. This lifecycle limit helps prevent configuration drift and maintains security posture. -### Data protection +=== Data protection * Amazon EC2 Instance Storage is encrypted, this is storage directly attached to the instance. For more information, see link:AWSEC2/latest/UserGuide/data-protection.html["Data protection in Amazon EC2",type="documentation"]. * EKS Auto Mode manages the volumes attached to EC2 instances at creation time, including root and data volumes. EKS Auto Mode does not fully manage EBS volumes created using Kubernetes persistent storage features. -### Patch management +=== Patch management * Amazon EKS Auto Mode automatically applies patches to managed instances. * Patches include: @@ -67,38 +65,38 @@ Customers retain responsibility for securing and updating workloads running on t ==== -### Access controls +=== Access controls * Direct instance access is restricted: ** SSH access is not available. ** {aws} Systems Manager Session Manager (SSM) access is not available. * Management operations are performed through the Amazon EKS API and Kubernetes API. -## Automated resource management +== Automated resource management Amazon EKS Auto Mode does not fully manage Amazon Elastic Block Store (Amazon EBS) Volumes created using Kubernetes persistent storage features. EKS Auto Mode also does not manage Elastic Load Balancers (ELB). Amazon EKS Auto Mode automates routine tasks for these resources. -### Storage security +=== Storage security -* {aws} recommends that you enable encryption for EBS Volumes provisionsed by Kubernetes persistent storage features. For more information, see <>. +* {aws} recommends that you enable encryption for EBS Volumes provisioned by Kubernetes persistent storage features. For more information, see <>. * Encryption at rest using {aws} KMS * You can configure your {aws} account to enforce the encryption of the new EBS volumes and snapshot copies that you create. For more information, see link:ebs/latest/userguide/encryption-by-default.html["Enable Amazon EBS encryption by default",type="documentation"] in the Amazon EBS User Guide. * For more information, see link:ebs/latest/userguide/security.html["Security in Amazon EBS",type="documentation"]. -### Load balancer security +=== Load balancer security * Automated configuration of Elastic Load Balancers * SSL/TLS certificate management through {aws} Certificate Manager integration * Security group automation for load balancer access control * For more information, see link:elasticloadbalancing/latest/userguide/security.html["Security in Elastic Load Balancing",type="documentation"]. -[[auto-security-bp,auto-security-bp.title]] -## Security best practices +[#auto-security-bp] +== Security best practices The following section describes security best practices for Amazon EKS Auto Mode. * Regularly review {aws} IAM policies and EKS access entries. * Implement least privilege access patterns for workloads. -* Monitor cluster activity through {aws} CloudTrail and Amazon CloudWatch. For more information, see link:eks/latest/userguide/logging-using-cloudtrail.html["Log API calls as CloudTrail events",type="documentation"] and link:eks/latest/userguide/cloudwatch.html["Monitor cluster data with Amazon CloudWatch",type="documentation"]. +* Monitor cluster activity through {aws} CloudTrail and Amazon CloudWatch. For more information, see <> and <>. * Use {aws} Security Hub for security posture assessment. * Implement pod security standards appropriate for your workloads. diff --git a/latest/ug/security/cert-signing.adoc b/latest/ug/security/cert-signing.adoc new file mode 100644 index 000000000..744d30da0 --- /dev/null +++ b/latest/ug/security/cert-signing.adoc @@ -0,0 +1,102 @@ +include::../attributes.txt[] + +[.topic] +[#cert-signing] += Secure workloads with Kubernetes certificates +:info_titleabbrev: Certificate signing + +[abstract] +-- +Learn how to request and obtain X.509 certificates from the Certificate Authority (CA) using Certificate Signing Requests (CSRs) in Amazon EKS, including details on migrating from legacy signers, generating CSRs, approving requests, and handling certificate signing considerations. +-- + +The Kubernetes Certificates API automates https://www.itu.int/rec/T-REC-X.509[X.509] credential provisioning. The API features a command line interface for Kubernetes API clients to request and obtain https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/[X.509 certificates] from a Certificate Authority (CA). You can use the `CertificateSigningRequest` (CSR) resource to request that a denoted signer sign the certificate. Your requests are either approved or denied before they're signed. Kubernetes supports both built-in signers and custom signers with well-defined behaviors. This way, clients can predict what happens to their CSRs. To learn more about certificate signing, see https://kubernetes.io/docs/reference/access-authn-authz/certificate-signing-requests/[signing requests]. + +One of the built-in signers is `kubernetes.io/legacy-unknown`. The `v1beta1` API of CSR resource honored this legacy-unknown signer. However, the stable `v1` API of CSR doesn't allow the `signerName` to be set to `kubernetes.io/legacy-unknown`. + +If you want to use Amazon EKS CA for generating certificates on your clusters, you must use a custom signer. To use the CSR `v1` API version and generate a new certificate, you must migrate any existing manifests and API clients. Existing certificates that were created with the existing `v1beta1` API are valid and function until the certificate expires. This includes the following: + +* Trust distribution: None. There's no standard trust or distribution for this signer in a Kubernetes cluster. +* Permitted subjects: Any +* Permitted x509 extensions: Honors subjectAltName and key usage extensions and discards other extensions +* Permitted key usages: Must not include usages beyond ["key encipherment", "digital signature", "server auth"] ++ +NOTE: Client certificate signing is not supported. +* Expiration/certificate lifetime: 1 year (default and maximum) +* CA bit allowed/disallowed: Not allowed + + +[#csr-example] +== Example CSR generation with signerName + +These steps shows how to generate a serving certificate for DNS name `myserver.default.svc` using `signerName: beta.eks.amazonaws.com/app-serving`. Use this as a guide for your own environment. + +. Run the `openssl genrsa -out myserver.key 2048` command to generate an RSA private key. ++ +[source,bash,subs="verbatim,attributes"] +---- +openssl genrsa -out myserver.key 2048 +---- +. Run the following command to generate a certificate request. ++ +[source,bash,subs="verbatim,attributes"] +---- +openssl req -new -key myserver.key -out myserver.csr -subj "/CN=myserver.default.svc" +---- +. Generate a `base64` value for the CSR request and store it in a variable for use in a later step. ++ +[source,bash,subs="verbatim,attributes"] +---- +base_64=$(cat myserver.csr | base64 -w 0 | tr -d " +") +---- +. Run the following command to create a file named `mycsr.yaml`. In the following example, `beta.eks.amazonaws.com/app-serving` is the `signerName`. ++ +[source,yaml,subs="verbatim,attributes"] +---- +cat >mycsr.yaml < myserver.crt +---- diff --git a/latest/ug/security/compliance.adoc b/latest/ug/security/compliance.adoc index e8be4ab3e..23748cfb4 100644 --- a/latest/ug/security/compliance.adoc +++ b/latest/ug/security/compliance.adoc @@ -1,37 +1,28 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[compliance,compliance.title]] +[#compliance] = Compliance validation for Amazon EKS clusters -:info_doctype: section -:info_title: Compliance validation for Amazon EKS clusters :info_titleabbrev: Validate compliance -:info_abstract: Discover compliance resources and services for Amazon Elastic Kubernetes Service \ - to help secure your {aws} workloads, meet regulatory requirements like HIPAA, and \ - validate adherence to security standards like NIST, PCI, and ISO using {aws} Config, \ - Security Hub, GuardDuty, and Audit Manager. - -include::../attributes.txt[] [abstract] -- -Discover compliance resources and services for Amazon Elastic Kubernetes Service to help secure your {aws} workloads, meet regulatory requirements like HIPAA, and validate adherence to security standards like NIST, PCI, and ISO using {aws} Config, Security Hub, GuardDuty, and Audit Manager. +Discover compliance resources to help secure your {aws} workloads, meet regulatory requirements like HIPAA, and validate adherence to security standards. -- +// Entire topic last refreshed to match the ComplianceResources shared entity on 2025-04-15. + To learn whether an {aws} service is within the scope of specific compliance programs, see link:compliance/services-in-scope/[{aws} services in Scope by Compliance Program,type="marketing"] and choose the compliance program that you are interested in. For general information, see link:compliance/programs/[{aws} Compliance Programs,type="marketing"]. You can download third-party audit reports using {aws} Artifact. For more information, see link:artifact/latest/ug/downloading-documents.html[Downloading Reports in {aws} Artifact,type="documentation"]. Your compliance responsibility when using {aws} services is determined by the sensitivity of your data, your company's compliance objectives, and applicable laws and regulations. {aws} provides the following resources to help with compliance: - - -* link:quickstart/?awsf.filter-tech-category=tech-category%23security-identity-compliance[Security and Compliance Quick Start Guides,type="marketing"] – These deployment guides discuss architectural considerations and provide steps for deploying baseline environments on {aws} that are security and compliance focused. -* link:whitepapers/latest/architecting-hipaa-security-and-compliance-on-aws/architecting-hipaa-security-and-compliance-on-aws.html[Architecting for HIPAA Security and Compliance on Amazon Web Services,type="documentation"] – This whitepaper describes how companies can use {aws} to create HIPAA-eligible applications. -+ -NOTE: Not all {aws} services are HIPAA eligible. For more information, see the link:compliance/hipaa-eligible-services-reference/[HIPAA Eligible Services Reference,type="marketing"]. +* link:solutions/security/security-compliance-governance/[Security Compliance & Governance,type="marketing"] – These solution implementation guides discuss architectural considerations and provide steps for deploying security and compliance features. +* link:compliance/hipaa-eligible-services-reference/[HIPAA Eligible Services Reference,type="marketing"] – Lists HIPAA eligible services. Not all {aws} services are HIPAA eligible. * link:compliance/resources/[{aws} Compliance Resources,type="marketing"] – This collection of workbooks and guides might apply to your industry and location. * https://d1.awsstatic.com/whitepapers/compliance/AWS_Customer_Compliance_Guides.pdf[{aws} Customer Compliance Guides] – Understand the shared responsibility model through the lens of compliance. The guides summarize the best practices for securing {aws} services and map the guidance to security controls across multiple frameworks (including National Institute of Standards and Technology (NIST), Payment Card Industry Security Standards Council (PCI), and International Organization for Standardization (ISO)). * link:config/latest/developerguide/evaluate-config.html[Evaluating Resources with Rules,type="documentation"] in the _{aws} Config Developer Guide_ – The {aws} Config service assesses how well your resource configurations comply with internal practices, industry guidelines, and regulations. * link:securityhub/latest/userguide/what-is-securityhub.html[{aws} Security Hub,type="documentation"] – This {aws} service provides a comprehensive view of your security state within {aws}. Security Hub uses security controls to evaluate your {aws} resources and to check your compliance against security industry standards and best practices. For a list of supported services and controls, see link:securityhub/latest/userguide/securityhub-controls-reference.html[Security Hub controls reference,type="documentation"]. * link:guardduty/latest/ug/what-is-guardduty.html[Amazon GuardDuty,type="documentation"] – This {aws} service detects potential threats to your {aws} accounts, workloads, containers, and data by monitoring your environment for suspicious and malicious activities. GuardDuty can help you address various compliance requirements, like PCI DSS, by meeting intrusion detection requirements mandated by certain compliance frameworks. -* link:audit-manager/latest/userguide/what-is.html[{aws} Audit Manager,type="documentation"] – This {aws} service helps you continuously audit your {aws} usage to simplify how you manage risk and compliance with regulations and industry standards. +* link:audit-manager/latest/userguide/what-is.html[{aws} Audit Manager,type="documentation"] – This {aws} service helps you continuously audit your {aws} usage to simplify how you manage risk and compliance with regulations and industry standards. \ No newline at end of file diff --git a/latest/ug/security/configuration-vulnerability-analysis.adoc b/latest/ug/security/configuration-vulnerability-analysis.adoc index 7ee9138e4..93b82271d 100644 --- a/latest/ug/security/configuration-vulnerability-analysis.adoc +++ b/latest/ug/security/configuration-vulnerability-analysis.adoc @@ -1,67 +1,59 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[configuration-vulnerability-analysis,configuration-vulnerability-analysis.title]] +[#configuration-vulnerability-analysis] = Analyze vulnerabilities in Amazon EKS -:info_doctype: section -:info_title: Analyze vulnerabilities \ - in Amazon EKS :info_titleabbrev: Analyze vulnerabilities -:info_abstract: Learn how to analyze the security configuration and vulnerabilities of your Amazon EKS \ - clusters and resources using tools like the CIS EKS Benchmark, platform \ - versions, vulnerability lists, Amazon Inspector, and Amazon GuardDuty for \ - comprehensive threat detection and protection. - -include::../attributes.txt[] [abstract] -- Learn how to analyze the security configuration and vulnerabilities of your Amazon EKS clusters and resources using tools like the CIS EKS Benchmark, platform versions, vulnerability lists, Amazon Inspector, and Amazon GuardDuty for comprehensive threat detection and protection. -- -Security is a critical consideration for configuring and maintaining [.noloc]`Kubernetes` clusters and applications. The following lists resources for you to analyze the security configuration of your EKS clusters, resources for you to check for vulnerabilities, and integrations with {aws} services that can do that analysis for you. +Security is a critical consideration for configuring and maintaining Kubernetes clusters and applications. The following lists resources for you to analyze the security configuration of your EKS clusters, resources for you to check for vulnerabilities, and integrations with {aws} services that can do that analysis for you. -[[configuration-vulnerability-analysis-cis,configuration-vulnerability-analysis-cis.title]] +[#configuration-vulnerability-analysis-cis] == The Center for Internet Security (CIS) benchmark for Amazon EKS The https://www.cisecurity.org/benchmark/kubernetes/[Center for Internet Security (CIS) Kubernetes Benchmark] provides guidance for Amazon EKS security configurations. The benchmark: - - -* Is applicable to Amazon EC2 nodes (both managed and self-managed) where you are responsible for security configurations of [.noloc]`Kubernetes` components. -* Provides a standard, community-approved way to ensure that you have configured your [.noloc]`Kubernetes` cluster and nodes securely when using Amazon EKS. +* Is applicable to Amazon EC2 nodes (both managed and self-managed) where you are responsible for security configurations of Kubernetes components. +* Provides a standard, community-approved way to ensure that you have configured your Kubernetes cluster and nodes securely when using Amazon EKS. * Consists of four sections; control plane logging configuration, node security configurations, policies, and managed services. -* Supports all of the [.noloc]`Kubernetes` versions currently available in Amazon EKS and can be run using https://github.com/aquasecurity/kube-bench[kube-bench], a standard open source tool for checking configuration using the CIS benchmark on [.noloc]`Kubernetes` clusters. +* Supports all of the Kubernetes versions currently available in Amazon EKS and can be run using https://github.com/aquasecurity/kube-bench[kube-bench], a standard open source tool for checking configuration using the CIS benchmark on Kubernetes clusters. To learn more, see link:containers/introducing-cis-amazon-eks-benchmark[Introducing The CIS Amazon EKS Benchmark,type="blog"]. -[[configuration-vulnerability-analysis-pv,configuration-vulnerability-analysis-pv.title]] +For an automated `aws-sample` pipeline to update your node group with a CIS benchmarked AMI, see https://github.com/aws-samples/pipeline-for-hardening-eks-nodes-and-automating-updates[EKS-Optimized AMI Hardening Pipeline]. + +[#configuration-vulnerability-analysis-pv] == Amazon EKS platform versions -Amazon EKS _platform versions_ represent the capabilities of the cluster control plane, including which [.noloc]`Kubernetes` API server flags are enabled and the current [.noloc]`Kubernetes` patch version. New clusters are deployed with the latest platform version. For details, see <>. +Amazon EKS _platform versions_ represent the capabilities of the cluster control plane, including which Kubernetes API server flags are enabled and the current Kubernetes patch version. New clusters are deployed with the latest platform version. For details, see link:eks/latest/userguide/platform-versions.html[EKS platform-versions,type="documentation"]. -You can <> to newer [.noloc]`Kubernetes` versions. As new [.noloc]`Kubernetes` versions become available in Amazon EKS, we recommend that you proactively update your clusters to use the latest available version. For more information about [.noloc]`Kubernetes` versions in EKS, see <>. +You can <> to newer Kubernetes versions. As new Kubernetes versions become available in Amazon EKS, we recommend that you proactively update your clusters to use the latest available version. For more information about Kubernetes versions in EKS, see link:eks/latest/userguide/kubernetes-versions.html[Amazon EKS supported versions,type="documentation"]. -[[configuration-vulnerability-analysis-os,configuration-vulnerability-analysis-os.title]] +[#configuration-vulnerability-analysis-os] == Operating system vulnerability list -[[configuration-vulnerability-analysis-al2023,configuration-vulnerability-analysis-al2023.title]] +[#configuration-vulnerability-analysis-al2023] === AL2023 vulnerability list Track security or privacy events for Amazon Linux 2023 at the https://alas.aws.amazon.com/alas2023.html[Amazon Linux Security Center] or subscribe to the associated https://alas.aws.amazon.com/AL2023/alas.rss[RSS feed]. Security and privacy events include an overview of the issue affected, packages, and instructions for updating your instances to correct the issue. -[[configuration-vulnerability-analysis-al2,configuration-vulnerability-analysis-al2.title]] +[#configuration-vulnerability-analysis-al2] === Amazon Linux 2 vulnerability list Track security or privacy events for Amazon Linux 2 at the https://alas.aws.amazon.com/alas2.html[Amazon Linux Security Center] or subscribe to the associated https://alas.aws.amazon.com/AL2/alas.rss[RSS feed]. Security and privacy events include an overview of the issue affected, packages, and instructions for updating your instances to correct the issue. -[[configuration-vulnerability-analysis-inspector,configuration-vulnerability-analysis-inspector.title]] +[#configuration-vulnerability-analysis-inspector] == Node detection with Amazon Inspector -You can use link:inspector/latest/userguide/inspector_introduction.html[Amazon Inspector,type="documentation"] to check for unintended network accessibility of your nodes and for vulnerabilities on those Amazon EC2 instances. +You can use link:inspector/latest/userguide/inspector_introduction.html[Amazon Inspector,type="documentation"] to check for unintended network accessibility of your nodes and for vulnerabilities on those Amazon EC2 instances. -[[configuration-vulnerability-analysis-guardduty,configuration-vulnerability-analysis-guardduty.title]] +[#configuration-vulnerability-analysis-guardduty] == Cluster and node detection with Amazon GuardDuty -Amazon GuardDuty threat detection service that helps protect your accounts, containers, workloads, and the data within your {aws} environment. Among other features, GuardDuty offers the following two features that detect potential threats to your EKS clusters: _EKS Protection_ and _Runtime Monitoring_. +Amazon GuardDuty threat detection service that helps protect your accounts, containers, workloads, and the data within your {aws} environment. Among other features, GuardDuty offers the following two features that detect potential threats to your EKS clusters: _EKS Protection_ and _Runtime Monitoring_. -For more information, see <>. +For more information, see <>. \ No newline at end of file diff --git a/latest/ug/security/cross-service-confused-deputy-prevention.adoc b/latest/ug/security/cross-service-confused-deputy-prevention.adoc new file mode 100644 index 000000000..4c41ebc4e --- /dev/null +++ b/latest/ug/security/cross-service-confused-deputy-prevention.adoc @@ -0,0 +1,40 @@ +include::../attributes.txt[] + +[.topic] +[#cross-service-confused-deputy-prevention] += Cross-service confused deputy prevention in Amazon EKS +:info_titleabbrev: Cross-service confused deputy prevention + +The confused deputy problem is a security issue where an entity that doesn't have permission to perform an action can coerce a more-privileged entity to perform the action. In {aws}, cross-service impersonation can result in the confused deputy problem. Cross-service impersonation can occur when one service (the _calling service_) calls another service (the _called service_). The calling service can be manipulated to use its permissions to act on another customer's resources in a way it should not otherwise have permission to access. To prevent this, {aws} provides tools that help you protect your data for all services with service principals that have been given access to resources in your account. + +We recommend using the link:IAM/latest/UserGuide/reference_policies_condition-keys.html#condition-keys-sourcearn[`aws:SourceArn`, type="documentation"], link:IAM/latest/UserGuide/reference_policies_condition-keys.html#condition-keys-sourceaccount[`aws:SourceAccount`, type="documentation"] global condition context keys in resource policies to limit the permissions that Amazon Elastic Kubernetes Service (Amazon EKS) gives another service to the resource. + +`aws:SourceArn`:: +Use `aws:SourceArn` to associate only one resource with cross-service access. + +`aws:SourceAccount`:: +Use `aws:SourceAccount` to let any resource in that account be associated with the cross-service use. + +The most effective way to protect against the confused deputy problem is to use the `aws:SourceArn` global condition context key with the full ARN of the resource. If you don't know the full ARN of the resource or if you are specifying multiple resources, use the `aws:SourceArn` global context condition key with wildcard characters (+*+) for the unknown portions of the ARN. For example, `{arn-aws}:*:<123456789012>:*`. + +If the `aws:SourceArn` value does not contain the account ID, such as an Amazon S3 bucket ARN, you must use both `aws:SourceAccount` and `aws:SourceArn` to limit permissions. + +[#cross-service-confused-deputy-cluster-role] +== Amazon EKS cluster role cross-service confused deputy prevention + +An Amazon EKS cluster IAM role is required for each cluster. Kubernetes clusters managed by Amazon EKS use this role to manage nodes and the link:https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/guide/service/annotations/#legacy-cloud-provider[legacy Cloud Provider] uses this role to create load balancers with Elastic Load Balancing for services. +These cluster actions can only affect the same account, so we recommend that you limit each cluster role to that cluster and account. +This is a specific application of the {aws} recommendation to follow the _principle of least privilege_ in your account. + +*Source ARN format* + +The value of `aws:SourceArn` must be the ARN of an EKS cluster in the format `{arn-aws}eks:[.replaceable]``region``:[.replaceable]``account``:cluster/[.replaceable]``cluster-name```. For example, `{arn-aws}eks:us-west-2:123456789012:cluster/my-cluster` . + +*Trust policy format for EKS cluster roles* + +The following example shows how you can use the `aws:SourceArn` and `aws:SourceAccount` global condition context keys in Amazon EKS to prevent the confused deputy problem. + +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:4715ec25-2bee-4c62-90c4-4418d4931464[] +---- diff --git a/latest/ug/security/default-roles-users.adoc b/latest/ug/security/default-roles-users.adoc new file mode 100644 index 000000000..89122f1bf --- /dev/null +++ b/latest/ug/security/default-roles-users.adoc @@ -0,0 +1,193 @@ +include::../attributes.txt[] + +[.topic] +[#default-roles-users] += Understand Amazon EKS created RBAC roles and users +:info_titleabbrev: Default roles and users + +[abstract] +-- +Learn about the Kubernetes roles and users that Amazon EKS creates for cluster components and add-ons. Amazon EKS uses these role-based authorization control (RBAC) identities to operate the cluster. +-- + +When you create a Kubernetes cluster, several default Kubernetes identities are created on that cluster for the proper functioning of Kubernetes. Amazon EKS creates Kubernetes identities for each of its default components. The identities provide Kubernetes role-based authorization control (RBAC) for the cluster components. For more information, see https://kubernetes.io/docs/reference/access-authn-authz/rbac/[Using RBAC Authorization] in the Kubernetes documentation. + +When you install optional <> to your cluster, additional Kubernetes identities might be added to your cluster. For more information about identities not addressed by this topic, see the documentation for the add-on. + +You can view the list of Amazon EKS created Kubernetes identities on your cluster using the {aws-management-console} or `kubectl` command line tool. All of the user identities appear in the `kube` audit logs available to you through Amazon CloudWatch. + +[#default-role-users-console] +== {aws-management-console} + +=== Prerequisite + +The link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] that you use must have the permissions described in <>. + +=== To view Amazon EKS created identities using the {aws-management-console} + +. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. +. In the *Clusters* list, choose the cluster that contains the identities that you want to view. +. Choose the *Resources* tab. +. Under *Resource types*, choose *Authorization*. +. Choose, *ClusterRoles*, *ClusterRoleBindings*, *Roles*, or *RoleBindings*. All resources prefaced with *eks* are created by Amazon EKS. Additional Amazon EKS created identity resources are: ++ +* The *ClusterRole* and *ClusterRoleBinding* named *aws-node*. The *aws-node* resources support the <>, which Amazon EKS installs on all clusters. +* A *ClusterRole* named *vpc-resource-controller-role* and a *ClusterRoleBinding* named *vpc-resource-controller-rolebinding*. These resources support the https://github.com/aws/amazon-vpc-resource-controller-k8s[Amazon VPC resource controller], which Amazon EKS installs on all clusters. + ++ +In addition to the resources that you see in the console, the following special user identities exist on your cluster, though they're not visible in the cluster's configuration: + +* *`eks:cluster-bootstrap`* – Used for `kubectl` operations during cluster bootstrap. +* *`eks:support-engineer`* – Used for cluster management operations. + +. Choose a specific resource to view details about it. By default, you're shown information in *Structured view*. In the top-right corner of the details page you can choose *Raw view* to see all information for the resource. + +[#default-role-users-kubectl] +== Kubectl + +=== Prerequisite + +The entity that you use ({aws} Identity and Access Management (IAM) or OpenID Connect (OIDC)) to list the Kubernetes resources on the cluster must be authenticated by IAM or your OIDC identity provider. The entity must be granted permissions to use the Kubernetes `get` and `list` verbs for the `Role`, `ClusterRole`, `RoleBinding`, and `ClusterRoleBinding` resources on your cluster that you want the entity to work with. For more information about granting IAM entities access to your cluster, see <>. For more information about granting entities authenticated by your own OIDC provider access to your cluster, see <>. + +=== To view Amazon EKS created identities using `kubectl` +Run the command for the type of resource that you want to see. All returned resources that are prefaced with *eks* are created by Amazon EKS. In addition to the resources returned in the output from the commands, the following special user identities exist on your cluster, though they're not visible in the cluster's configuration: + +* *`eks:cluster-bootstrap`* – Used for `kubectl` operations during cluster bootstrap. +* *`eks:support-engineer`* – Used for cluster management operations. + +*ClusterRoles* – `ClusterRoles` are scoped to your cluster, so any permission granted to a role applies to resources in any Kubernetes namespace on the cluster. + +The following command returns all of the Amazon EKS created Kubernetes `ClusterRoles` on your cluster. + +[source,bash,subs="verbatim,attributes"] +---- +kubectl get clusterroles | grep eks +---- + +In addition to the `ClusterRoles` returned in the output that are prefaced with, the following `ClusterRoles` exist. + +* *`aws-node`* – This `ClusterRole` supports the <>, which Amazon EKS installs on all clusters. +* *`vpc-resource-controller-role`* – This `ClusterRole` supports the https://github.com/aws/amazon-vpc-resource-controller-k8s[Amazon VPC resource controller], which Amazon EKS installs on all clusters. + +To see the specification for a `ClusterRole`, replace [.replaceable]`eks:k8s-metrics` in the following command with a `ClusterRole` returned in the output of the previous command. The following example returns the specification for the [.replaceable]`eks:k8s-metrics` `ClusterRole`. + +[source,bash,subs="verbatim,attributes"] +---- +kubectl describe clusterrole eks:k8s-metrics +---- + +An example output is as follows. + +[source,bash,subs="verbatim,attributes"] +---- +Name: eks:k8s-metrics +Labels: +Annotations: +PolicyRule: + Resources Non-Resource URLs Resource Names Verbs + --------- ----------------- -------------- ----- + [/metrics] [] [get] + endpoints [] [] [list] + nodes [] [] [list] + pods [] [] [list] + deployments.apps [] [] [list] +---- + +*ClusterRoleBindings* – `ClusterRoleBindings` are scoped to your cluster. + +The following command returns all of the Amazon EKS created Kubernetes `ClusterRoleBindings` on your cluster. + +[source,bash,subs="verbatim,attributes"] +---- +kubectl get clusterrolebindings | grep eks +---- + +In addition to the `ClusterRoleBindings` returned in the output, the following `ClusterRoleBindings` exist. + +* *`aws-node`* – This `ClusterRoleBinding` supports the <>, which Amazon EKS installs on all clusters. +* *`vpc-resource-controller-rolebinding`* – This `ClusterRoleBinding` supports the https://github.com/aws/amazon-vpc-resource-controller-k8s[Amazon VPC resource controller], which Amazon EKS installs on all clusters. + +To see the specification for a `ClusterRoleBinding`, replace [.replaceable]`eks:k8s-metrics` in the following command with a `ClusterRoleBinding` returned in the output of the previous command. The following example returns the specification for the [.replaceable]`eks:k8s-metrics` `ClusterRoleBinding`. + +[source,bash,subs="verbatim,attributes"] +---- +kubectl describe clusterrolebinding eks:k8s-metrics +---- + +An example output is as follows. + +[source,bash,subs="verbatim,attributes"] +---- +Name: eks:k8s-metrics +Labels: +Annotations: +Role: + Kind: ClusterRole + Name: eks:k8s-metrics +Subjects: + Kind Name Namespace + ---- ---- --------- + User eks:k8s-metrics +---- + +*Roles* – `Roles` are scoped to a Kubernetes namespace. All Amazon EKS created `Roles` are scoped to the `kube-system` namespace. + +The following command returns all of the Amazon EKS created Kubernetes `Roles` on your cluster. + +[source,bash,subs="verbatim,attributes"] +---- +kubectl get roles -n kube-system | grep eks +---- + +To see the specification for a `Role`, replace [.replaceable]`eks:k8s-metrics` in the following command with the name of a `Role` returned in the output of the previous command. The following example returns the specification for the [.replaceable]`eks:k8s-metrics` `Role`. + +[source,bash,subs="verbatim,attributes"] +---- +kubectl describe role eks:k8s-metrics -n kube-system +---- + +An example output is as follows. + +[source,bash,subs="verbatim,attributes"] +---- +Name: eks:k8s-metrics +Labels: +Annotations: +PolicyRule: + Resources Non-Resource URLs Resource Names Verbs + --------- ----------------- -------------- ----- + daemonsets.apps [] [aws-node] [get] + deployments.apps [] [vpc-resource-controller] [get] +---- + +*RoleBindings* – `RoleBindings` are scoped to a Kubernetes namespace. All Amazon EKS created `RoleBindings` are scoped to the `kube-system` namespace. + +The following command returns all of the Amazon EKS created Kubernetes `RoleBindings` on your cluster. + +[source,bash,subs="verbatim,attributes"] +---- +kubectl get rolebindings -n kube-system | grep eks +---- + +To see the specification for a `RoleBinding`, replace [.replaceable]`eks:k8s-metrics` in the following command with a `RoleBinding` returned in the output of the previous command. The following example returns the specification for the [.replaceable]`eks:k8s-metrics` `RoleBinding`. + +[source,bash,subs="verbatim,attributes"] +---- +kubectl describe rolebinding eks:k8s-metrics -n kube-system +---- + +An example output is as follows. + +[source,bash,subs="verbatim,attributes"] +---- +Name: eks:k8s-metrics +Labels: +Annotations: +Role: + Kind: Role + Name: eks:k8s-metrics +Subjects: + Kind Name Namespace + ---- ---- --------- + User eks:k8s-metrics +---- \ No newline at end of file diff --git a/latest/ug/security/disaster-recovery-resiliency.adoc b/latest/ug/security/disaster-recovery-resiliency.adoc new file mode 100644 index 000000000..7bc742c28 --- /dev/null +++ b/latest/ug/security/disaster-recovery-resiliency.adoc @@ -0,0 +1,25 @@ +include::../attributes.txt[] + +[.topic] +[#disaster-recovery-resiliency] += Understand resilience in Amazon EKS clusters +:info_titleabbrev: Resilience + +[abstract] +-- +Learn how Amazon EKS ensures high availability, data resilience, and fault tolerance for your Kubernetes control plane by leveraging {aws} infrastructure across multiple Availability Zones . +-- + +The {aws} global infrastructure is built around {aws} Regions and Availability Zones. {aws} Regions provide multiple physically separated and isolated Availability Zones, which are connected with low-latency, high-throughput, and highly redundant networking. With Availability Zones, you can design and operate applications and databases that automatically fail over between Availability Zones without interruption. Availability Zones are more highly available, fault tolerant, and scalable than traditional single or multiple data center infrastructures. + +Amazon EKS runs and scales the Kubernetes control plane across multiple {aws} Availability Zones to ensure high availability. Amazon EKS automatically scales control plane instances based on load, detects and replaces unhealthy control plane instances, and automatically patches the control plane. After you initiate a version update, Amazon EKS updates your control plane for you, maintaining high availability of the control plane during the update. + +This control plane consists of at least two API server instances and three `etcd` instances that run across three Availability Zones within an {aws} Region. Amazon EKS: + + + +* Actively monitors the load on control plane instances and automatically scales them to ensure high performance. +* Automatically detects and replaces unhealthy control plane instances, restarting them across the Availability Zones within the {aws} Region as needed. +* Leverages the architecture of {aws} Regions in order to maintain high availability. Because of this, Amazon EKS is able to offer an link:eks/sla[SLA for API server endpoint availability,type="marketing"]. + +For more information about {aws} Regions and Availability Zones, see link:about-aws/global-infrastructure/[{aws} global infrastructure,type="marketing"]. \ No newline at end of file diff --git a/latest/ug/security/enable-kms.adoc b/latest/ug/security/enable-kms.adoc new file mode 100644 index 000000000..cee97366b --- /dev/null +++ b/latest/ug/security/enable-kms.adoc @@ -0,0 +1,194 @@ +include::../attributes.txt[] + +[.topic] +[#enable-kms] += Encrypt Kubernetes secrets with KMS on existing clusters +:info_titleabbrev: Enable secret encryption + +[abstract] +-- +Learn how to enable Kubernetes secrets encryption with {aws} KMS on an existing Amazon EKS cluster, ensuring secure storage of sensitive data. +-- + +IMPORTANT: This procedure only applies to EKS clusters running Kubernetes version 1.27 or lower. If you are running Kubernetes version 1.28 or higher, your Kubernetes secrets are protected with envelope encryption by default. For more information, see <>. + +If you enable https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/[secrets encryption], the Kubernetes secrets are encrypted using the {aws} KMS key that you select. The KMS key must meet the following conditions: + +* Symmetric +* Can encrypt and decrypt data +* Created in the same {aws} Region as the cluster +* If the KMS key was created in a different account, the link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] must have access to the KMS key. + +For more information, see link:kms/latest/developerguide/key-policy-modifying-external-accounts.html[Allowing IAM principals in other accounts to use a KMS key,type="documentation"] in the _link:kms/latest/developerguide/[{aws} Key Management Service Developer Guide,type="documentation"]_. + +[WARNING] +==== + +You can't disable secrets encryption after enabling it. This action is irreversible. + +==== + +eksctl :: + +This procedure only applies to EKS clusters running Kubernetes version 1.27 or lower. For more information, see <>. + +You can enable encryption in two ways: + +** Add encryption to your cluster with a single command. ++ +To automatically re-encrypt your secrets, run the following command. ++ +[source,bash,subs="verbatim,attributes"] +---- +eksctl utils enable-secrets-encryption \ + --cluster my-cluster \ + --key-arn {arn-aws}kms:region-code:account:key/key +---- ++ +To opt-out of automatically re-encrypting your secrets, run the following command. ++ +[source,bash,subs="verbatim,attributes"] +---- +eksctl utils enable-secrets-encryption + --cluster my-cluster \ + --key-arn {arn-aws}kms:region-code:account:key/key \ + --encrypt-existing-secrets=false +---- +** Add encryption to your cluster with a `kms-cluster.yaml` file. ++ +[source,yaml,subs="verbatim,attributes"] +---- +apiVersion: eksctl.io/v1alpha5 +kind: ClusterConfig + +metadata: + name: my-cluster + region: region-code + +secretsEncryption: + keyARN: {arn-aws}kms:region-code:account:key/key +---- ++ +To have your secrets re-encrypt automatically, run the following command. ++ +[source,bash,subs="verbatim,attributes"] +---- +eksctl utils enable-secrets-encryption -f kms-cluster.yaml +---- ++ +To opt out of automatically re-encrypting your secrets, run the following command. ++ +[source,bash,subs="verbatim,attributes"] +---- +eksctl utils enable-secrets-encryption -f kms-cluster.yaml --encrypt-existing-secrets=false +---- + + +{aws-management-console}:: +.. This procedure only applies to EKS clusters running Kubernetes version 1.27 or lower. For more information, see <>. +.. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. +.. Choose the cluster that you want to add KMS encryption to. +.. Choose the *Overview* tab (this is selected by default). +.. Scroll down to the *Secrets encryption* section and choose *Enable*. +.. Select a key from the dropdown list and choose the *Enable* button. If no keys are listed, you must create one first. For more information, see link:kms/latest/developerguide/create-keys.html[Creating keys,type="documentation"] +.. Choose the *Confirm* button to use the chosen key. + + +{aws} CLI:: +.. This procedure only applies to EKS clusters running Kubernetes version 1.27 or lower. For more information, see <>. +.. Associate the https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/[secrets encryption] configuration with your cluster using the following {aws} CLI command. Replace the example values with your own. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks associate-encryption-config \ + --cluster-name my-cluster \ + --encryption-config '[{"resources":["secrets"],"provider":{"keyArn":"{arn-aws}kms:region-code:account:key/key"}}]' +---- ++ +An example output is as follows. ++ +[source,json,subs="verbatim,attributes",role="nocopy"] +---- +{ +  "update": { +    "id": "3141b835-8103-423a-8e68-12c2521ffa4d", +    "status": "InProgress", +    "type": "AssociateEncryptionConfig", +    "params": [ +      { +        "type": "EncryptionConfig", +        "value": "[{\"resources\":[\"secrets\"],\"provider\":{\"keyArn\":\"{arn-aws}kms:region-code:account:key/key\"}}]" +      } +    ], +    "createdAt": 1613754188.734, +    "errors": [] +  } +} +---- +.. You can monitor the status of your encryption update with the following command. Use the specific `cluster name` and `update ID` that was returned in the previous output. When a `Successful` status is displayed, the update is complete. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks describe-update \ + --region region-code \ + --name my-cluster \ + --update-id 3141b835-8103-423a-8e68-12c2521ffa4d +---- ++ +An example output is as follows. ++ +[source,json,subs="verbatim,attributes",role="nocopy"] +---- +{ +  "update": { +    "id": "3141b835-8103-423a-8e68-12c2521ffa4d", +    "status": "Successful", +    "type": "AssociateEncryptionConfig", +    "params": [ +      { +        "type": "EncryptionConfig", +        "value": "[{\"resources\":[\"secrets\"],\"provider\":{\"keyArn\":\"{arn-aws}kms:region-code:account:key/key\"}}]" +      } +    ], +    "createdAt": 1613754188.734>, +    "errors": [] +  } +} +---- +.. To verify that encryption is enabled in your cluster, run the `describe-cluster` command. The response contains an `EncryptionConfig` string. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks describe-cluster --region region-code --name my-cluster +---- + +After you enabled encryption on your cluster, you must encrypt all existing secrets with the new key: + +[NOTE] +==== + +If you use `eksctl`, running the following command is necessary only if you opt out of re-encrypting your secrets automatically. + +==== + +[source,bash,subs="verbatim,attributes"] +---- +kubectl get secrets --all-namespaces -o json | kubectl annotate --overwrite -f - kms-encryption-timestamp="time value" +---- + +[WARNING] +==== + +If you enable https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/[secrets encryption] for an existing cluster and the KMS key that you use is ever deleted, then there's no way to recover the cluster. If you delete the KMS key, you permanently put the cluster in a degraded state. For more information, see link:kms/latest/developerguide/deleting-keys.html[Deleting {aws} KMS keys,type="documentation"]. + +==== + +[NOTE] +==== + +By default, the `create-key` command creates a link:kms/latest/developerguide/symmetric-asymmetric.html[symmetric encryption KMS key,type="documentation"] with a key policy that gives the account root admin access on {aws} KMS actions and resources. If you want to scope down the permissions, make sure that the `kms:DescribeKey` and `kms:CreateGrant` actions are permitted on the policy for the principal that calls the `create-cluster` API. + + +For clusters using KMS Envelope Encryption, `kms:CreateGrant` permissions are required. The condition `kms:GrantIsForAWSResource` is not supported for the CreateCluster action, and should not be used in KMS policies to control `kms:CreateGrant` permissions for users performing CreateCluster. + +==== \ No newline at end of file diff --git a/latest/ug/security/envelope-encryption.adoc b/latest/ug/security/envelope-encryption.adoc new file mode 100644 index 000000000..7709b9283 --- /dev/null +++ b/latest/ug/security/envelope-encryption.adoc @@ -0,0 +1,154 @@ +include::../attributes.txt[] + +[.topic] +[#envelope-encryption] += Default envelope encryption for all Kubernetes API Data + +//GDC +//Insert Images +//Existing clusters and new clusters + +Amazon Elastic Kubernetes Service (Amazon EKS) provides default envelope encryption for all Kubernetes API data in EKS clusters running Kubernetes version 1.28 or higher. + +Envelope encryption protects the data you store with the Kubernetes API server. For example, envelope encryption applies to the configuration of your Kubernetes cluster, such as `ConfigMaps`. Envelope encryption does not apply to data on nodes or EBS volumes. EKS previously supported encrypting Kubernetes secrets, and now this envelope encryption extends to all Kubernetes API data. + +This provides a managed, default experience that implements defense-in-depth for your Kubernetes applications and doesn't require any action on your part. + +Amazon EKS uses {aws} link:kms/latest/developerguide/overview.html["Key Management Service (KMS)",type="documentation"] with https://kubernetes.io/docs/tasks/administer-cluster/kms-provider/#configuring-the-kms-provider-kms-v2[Kubernetes KMS provider v2] for this additional layer of security with an link:kms/latest/developerguide/concepts.html#aws-owned-cmk["Amazon Web Services owned key",type="documentation"], and the option for you to bring your own link:kms/latest/developerguide/concepts.html#customer-cmk["customer managed key",type="documentation"] (CMK) from {aws} KMS. + +== Understanding envelope encryption + +Envelope encryption is the process of encrypting plain text data with a data encryption key (DEK) before it's sent to the datastore (etcd), and then encrypting the DEK with a root KMS key that is stored in a remote, centrally managed KMS system ({aws} KMS). This is a defense-in-depth strategy because it protects the data with an encryption key (DEK), and then adds another security layer by protecting that DEK with a separate, securely stored encryption key called a key encryption key (KEK). + +== How Amazon EKS enables default envelope encryption with KMS v2 and {aws} KMS + +// Do we want to expose this detail? + +Amazon EKS uses https://kubernetes.io/docs/tasks/administer-cluster/kms-provider/#kms-v2[KMS v2] to implement default envelope encryption for all API data in the managed Kubernetes control plane before it's persisted in the https://etcd.io/docs/v3.5/faq/[etcd] database. At startup, the cluster API server generates a data encryption key (DEK) from a secret seed combined with randomly generated data. Also at startup, the API server makes a call to the KMS plugin to encrypt the DEK seed using a remote key encryption key (KEK) from {aws} KMS. This is a one-time call executed at startup of the API server and on KEK rotation. The API server then caches the encrypted DEK seed. After this, the API server uses the cached DEK seed to generate other single use DEKs based on a Key Derivation Function (KDF). Each of these generated DEKs is then used only once to encrypt a single Kubernetes resource before it's stored in etcd. With the use of an encrypted cached DEK seed in KMS v2, the process of encrypting Kubernetes resources in the API server is both more performant and cost effective. + +*By default, this KEK is owned by {aws}, but you can optionally bring your own from {aws} KMS.* + +The diagram below depicts the generation and encryption of a DEK at the startup of the API server. + +image::images/security-generate-dek.png[The diagram depicts the generation and encryption of a DEK at the startup of the API server] + +The high-level diagram below depicts the encryption of a Kubernetes resource before it's stored in etcd. + +image::images/security-encrypt-request.png[The high-level diagram depicts the encryption of a Kubernetes resource before it's stored in etcd. ] + +== Frequently asked questions + +=== How does default envelope encryption improve the security posture of my EKS cluster? + +// This feature gives you defense-in-depth by default with every EKS cluster running Kubernetes version 1.28 or higher, and heightens the level of security for all Kubernetes data before it's stored in etcd, including customer content. + +This feature reduces the surface area and period of time in which metadata and customer content are un-encrypted. With default envelope encryption, metadata and customer content are only ever in a temporarily un-encrypted state in the kube-apiserver's memory before being stored in etcd. The kube-apiserver's memory is secured through the link:whitepapers/latest/security-design-of-aws-nitro-system/the-components-of-the-nitro-system.html["Nitro system",type="documentation"]. Amazon EKS only uses link:whitepapers/latest/security-design-of-aws-nitro-system/security-design-of-aws-nitro-system.html["Nitro-based EC2 instances",type="documentation"] for the managed Kubernetes control plane. These instances have security control designs that prevent any system or person from accessing their memory. + +=== Which version of Kubernetes do I need to run in order to have this feature? + +For default envelope encryption to be enabled, your Amazon EKS cluster has to be running Kubernetes version 1.28 or later. + +=== Is my data still secure if I'm running a Kubernetes cluster version that doesn't support this feature? + +Yes. At {aws}, link:security/["security is our highest priority",type="marketing"]. We base all our digital transformation and innovation on the highest security operational practices, and stay committed to raising that bar. + +All of the data stored in the etcd are encrypted at the disk level for every EKS cluster, irrespective of the Kubernetes version being run. EKS uses root keys that generate volume encryption keys which are managed by the EKS service. Additionally, every Amazon EKS cluster is run in an isolated VPC using cluster-specific virtual machines. Because of this architecture, and our practices around operational security, Amazon EKS has link:eks/latest/userguide/compliance.html["achieved multiple compliance ratings and standards",type="documentation"] including SOC 1,2,3, PCI-DSS, ISO, and HIPAA eligibility. These compliance ratings and standards are maintained for all EKS clusters with or without default envelope encryption. + +=== How does envelope encryption work in Amazon EKS? + +// Do we want to expose this detail? + +At startup, the cluster API server generates a data encryption key (DEK) from a secret seed combined with randomly generated data. Also at startup, the API server makes a call to the KMS plugin to encrypt the DEK using a remote key encryption key (KEK) from {aws} KMS. This is a one-time call executed at startup of the API server and on KEK rotation. The API server then caches the encrypted DEK seed. After this, the API server uses the cached DEK seed to generate other single use DEKs based on a Key Derivation Function (KDF). Each of these generated DEKs is then used only once to encrypt a single Kubernetes resource before it's stored in etcd. + +It's important to note that there are additional calls made from the API server to verify the health and normal functionality of the {aws} KMS integration. These additional health checks are visible in your {aws} CloudTrail. + + +=== Do I have to do anything or change any permissions for this feature to work in my EKS cluster? + +No, you don't have to take any action. Envelope encryption in Amazon EKS is now a default configuration that is enabled in all clusters running Kubernetes version 1.28 or higher. The {aws} KMS integration is established by the Kubernetes API server managed by {aws}. This means you do not need to configure any permissions to start using KMS encryption for your cluster. + +=== How can I know if default envelope encryption is enabled on my cluster? + +If you migrate to use your own CMK, then you will see the ARN of the KMS key associated with your cluster. Additionally, you can view the {aws} CloudTrail event logs associated with the use of your cluster's CMK. + +If your cluster uses an {aws} owned key, then this will be detailed in the EKS console (excluding the ARN of the key). + + +=== Can {aws} access the {aws} owned key used for default envelope encryption in Amazon EKS? + +No. {aws} has stringent security controls in Amazon EKS that prevent any person from accessing any plaintext encryption keys used for securing data in the etcd database. These security measures are also applied to the {aws} owned KMS key. + +=== Is default envelope encryption enabled in my existing EKS cluster? + +// Get clarity on this + +If you are running an Amazon EKS cluster with Kubernetes version 1.28 or higher, then envelope encryption of all Kubernetes API data is enabled. For existing clusters, Amazon EKS uses the `eks:kms-storage-migrator` RBAC ClusterRole to migrate data that was previously not envelope encrypted in etcd to this new encryption state. + +=== What does this mean if I already enabled envelope encryption for Secrets in my EKS cluster? + +If you have an existing customer managed key (CMK) in KMS that was used to envelope encrypt your Kubernetes Secrets, that same key will be used as the KEK for envelope encryption of all Kubernetes API data types in your cluster. + +=== Is there any additional cost to running an EKS cluster with default envelope encryption? + +There is no additional cost associated with the managed Kubernetes control plane if you are using an link:kms/latest/developerguide/concepts.html#aws-owned-cmk["Amazon Web Services owned key",type="documentation"] for the default envelope encryption. By default, every EKS cluster running Kubernetes version 1.28 or later uses an link:kms/latest/developerguide/concepts.html#aws-owned-cmk["Amazon Web Service owned key",type="documentation"]. However, if you use your own {aws} KMS key, normal link:kms/pricing/["KMS pricing",type="marketing"] will apply. + +=== How much does it cost to use my own {aws} KMS key to encrypt Kubernetes API data in my cluster? + +You pay $1 per month to store any custom key that you create or import to KMS. KMS charges for encryption and decryption requests. There is a free tier of 20,000 requests per month per account and you pay $0.03 per 10,000 requests above the free tier per month. This applies across all KMS usage for an account, so the cost of using your own {aws} KMS key on your cluster will be impacted by the usage of this key on other clusters or {aws} resources within your account. + +=== Will my KMS charges be higher now that my customer managed key (CMK) is being used to envelope encrypt all Kubernetes API data and not just Secrets? + +No. Our implementation with KMS v2 significantly reduces the number of calls made to {aws} KMS. This will in turn reduce the costs associated with your CMK irrespective of the additional Kubernetes data being encrypted or decrypted in your EKS cluster. + +As detailed above, the generated DEK seed used for encryption of Kubernetes resources is stored locally in the Kubernetes API server's cache after it has been encrypted with the remote KEK. If the encrypted DEK seed is not in the API server's cache, the API server will call {aws} KMS to encrypt the DEK seed. The API server then caches the encrypted DEK seed for future use in the cluster without calling KMS. Similarly, for decrypt requests, the API server will call {aws} KMS for the first decrypt request, after which the decrypted DEK seed will be cached and used for future decrypt operations. + +For more information, see https://github.com/kubernetes/enhancements/tree/master/keps/sig-auth/3299-kms-v2-improvements[KEP-3299: KMS v2 Improvements] in the Kubernetes Enhancements on GitHub. + +=== Can I use the same CMK key for multiple Amazon EKS clusters? + +Yes. To use a key again, you can link it to a cluster in the same region by associating the ARN with the cluster during creation. However, if you are using the same CMK for multiple EKS clusters, you should put the requisite measures in place to prevent arbitrary disablement of the CMK. Otherwise, a disabled CMK associated with multiple EKS clusters will have a wider scope of impact on the clusters depending on that key. + +=== What happens to my EKS cluster if my CMK becomes unavailable after default envelope encryption is enabled? + +If you disable a KMS key, it cannot be used in any link:kms/latest/developerguide/kms-cryptography.html#cryptographic-operations["cryptographic operation",type="documentation"]. Without access to an existing CMK, the API server will be unable to encrypt and persist any newly created Kubernetes objects, as well as decrypt any previously encrypted Kubernetes objects stored in etcd. If the CMK is disabled, the cluster will be immediately placed in an unhealthy/degraded state at which point we will be unable to fulfill our link:eks/sla/["Service Commitment",type="marketing"] until you re-enable the associated CMK. + +When a CMK is disabled, you will receive notifications about the degraded health of your EKS cluster and the need to re-enable your CMK within 30 days of disabling it to ensure successful restoration of your Kubernetes control plane resources. + +=== How can I protect my EKS cluster from the impact of a disabled/deleted CMK? + +To protect your EKS clusters from such an occurrence, your key administrators should manage access to KMS key operations using IAM policies with a least privilege principle to reduce the risk of any arbitrary disablement or deletion of keys associated with EKS clusters. Additionally, you can set a link:kms/latest/developerguide/deleting-keys-creating-cloudwatch-alarm.html["CloudWatch alarm",type="documentation"] to be notified about the state of your CMK. + +=== Will my EKS cluster be restored if I re-enable the CMK? + +To ensure successful restoration of your EKS cluster, we strongly recommend re-enabling your CMK within the first 30 days of it being disabled. However, the successful restoration of your EKS cluster will also depend on whether or not it undergoes any API breaking changes due to an automatic Kubernetes upgrade that may take place while the cluster is in an unhealthy/degraded state. + +=== Why is my EKS cluster placed in an unhealthy/degraded state after disabling the CMK? + +// remove, duplicative + +The EKS control plane's API server uses a DEK key which is encrypted and cached in the API server's memory to encrypt all the objects during create/update operations before they're stored in etcd. When an existing object is being retrieved from etcd, the API server uses the same cached DEK key and decrypts the Kubernetes resource object. If you disable the CMK, the API server will not see any immediate impact because of the cached DEK key in the API server's memory. However, when the API server instance is restarted, it won't have a cached DEK and will need to call {aws} KMS for encrypt and decrypt operations. Without a CMK, this process will fail with a KMS_KEY_DISABLED error code, preventing the API server from booting successfully. + +=== What happens to my EKS cluster if I delete my CMK? + +Deleting the CMK key associated with your EKS cluster will degrade its health beyond recovery. Without your cluster's CMK, the API server will no longer be able to encrypt and persist any new Kubernetes objects, as well as decrypt any previously encrypted Kubernetes objects stored in the etcd database. You should only proceed with deleting a CMK key for your EKS cluster when you are sure that you don't need to use the EKS cluster anymore. + +Please note that if the CMK is not found (KMS_KEY_NOT_FOUND) or the grants for the CMK associated with your cluster are revoked (KMS_GRANT_REVOKED), your cluster will not be recoverable. For more information about about cluster health and error codes, see link:eks/latest/userguide/troubleshooting.html#cluster-health-status["Cluster health FAQs and error codes with resolution paths",type="documentation"]. + +=== Will I still be charged for a degraded/unhealthy EKS cluster because I disabled or deleted my CMK? + +Yes. Although the EKS control plane will not be usable in the event of a disabled CMK, {aws} will still be running dedicated infrastructure resources allocated to the EKS cluster until it is deleted by the customer. Additionally, our link:eks/sla/["Service Commitment",type="marketing"] will not apply in such a circumstance because it will be a voluntary action or inaction by the customer that prevents the normal health and operation of your EKS cluster. + +=== Can my EKS cluster be automatically upgraded when it's in an unhealthy/degraded state because of a disabled CMK? + +Yes. However, if your cluster has a disabled CMK, you will have a 30 day period to re-enable it. In this 30 day period, your Kubernetes cluster will not be automatically upgraded. However, if this period lapses and you have not re-enabled the CMK, the cluster will be automatically upgraded to the next version (n+1) that is in standard support, following the Kubernetes version lifecycle in EKS. + +We strongly recommend quickly re-enabling a disabled CMK when you become aware of an impacted cluster. It's important to note, that although EKS will automatically upgrade these impacted clusters, there's no guarantee that they will recover successfully, especially if the cluster undergoes multiple automatic upgrades since this may include changes to the Kubernetes API and unexpected behavior in the API server's bootstrap process. + + +=== Can I use a KMS key alias? + +Yes. Amazon EKS link:eks/latest/APIReference/API_EncryptionConfig.html#API_EncryptionConfig_Contents["supports using KMS key aliases",type="documentation"]. An alias is a friendly name for a link:kms/latest/developerguide/concepts.html#kms_keys["Amazon Web Service KMS key",type="documentation"]. For example, an alias lets you refer to a KMS key as *my-key* instead of *`1234abcd-12ab-34cd-56ef-1234567890ab`*. + +=== Can I still backup and restore my cluster resources using my own Kubernetes backup solution? + +Yes. You can use a Kubernetes backup solution (like https://velero.io/[Velero]) for Kubernetes cluster disaster recovery, data migration, and data protection. If you run a Kubernetes backup solution that accesses the cluster resources through the API server, any data that the application retrieves will be decrypted before reaching the client. This will allow you to recover the cluster resources in another Kubernetes cluster. diff --git a/latest/ug/security/iam-reference/auto-cluster-iam-role.adoc b/latest/ug/security/iam-reference/auto-cluster-iam-role.adoc index 74cf036af..321ce6f82 100644 --- a/latest/ug/security/iam-reference/auto-cluster-iam-role.adoc +++ b/latest/ug/security/iam-reference/auto-cluster-iam-role.adoc @@ -1,26 +1,26 @@ -//!!NODE_ROOT
+include::../../attributes.txt[] + [.topic] -[[auto-cluster-iam-role,auto-cluster-iam-role.title]] +[#auto-cluster-iam-role] = Amazon EKS Auto Mode cluster IAM role -:idprefix: wip_ - -include::../../attributes.txt[] +:info_titleabbrev: Auto Mode cluster IAM role [abstract] -- Learn how to create and configure the required {aws} Identity and Access Management role for Amazon EKS Auto Mode clusters to automate routine tasks for storage, networking, and compute autoscaling. -- -An Amazon EKS cluster IAM role is required for each cluster. [.noloc]`Kubernetes` clusters managed by Amazon EKS use this role to automate routine tasks for storage, networking, and compute autoscaling. +An Amazon EKS cluster IAM role is required for each cluster. Kubernetes clusters managed by Amazon EKS use this role to automate routine tasks for storage, networking, and compute autoscaling. Before you can create Amazon EKS clusters, you must create an IAM role with the policies required for EKS Auto Mode. You can either attach the suggested {aws} IAM managed policies, or create custom polices with equivalent permissions. -* xref:security-iam-awsmanpol-AmazonEKSComputePolicy[AmazonEKSComputePolicy] -* xref:security-iam-awsmanpol-AmazonEKSBlockStoragePolicy[AmazonEKSBlockStoragePolicy] -* xref:security-iam-awsmanpol-AmazonEKSLoadBalancingPolicy[AmazonEKSLoadBalancingPolicy] -* xref:security-iam-awsmanpol-AmazonEKSNetworkingPolicy[AmazonEKSNetworkingPolicy] -* xref:security-iam-awsmanpol-amazoneksclusterpolicy[AmazonEKSClusterPolicy] +* <> +* <> +* <> +* <> +* <> +[#auto-cluster-iam-role-check] == Check for an existing cluster role You can use the following procedure to check and see if your account already has the Amazon EKS cluster role. @@ -30,26 +30,12 @@ You can use the following procedure to check and see if your account already has . Search the list of roles for `AmazonEKSAutoClusterRole`. If a role that includes `AmazonEKSAutoClusterRole` doesn't exist, then see the instructions in the next section to create the role. If a role that includes `AmazonEKSAutoClusterRole` does exist, then select the role to view the attached policies. . Choose *Permissions*. . Ensure that the *AmazonEKSClusterPolicy* managed policy is attached to the role. If the policy is attached, your Amazon EKS cluster role is properly configured. -. Choose *Trust relationships*, and then choose *Edit trust policy*. -. Verify that the trust relationship contains the following policy. If the trust relationship matches the following policy, choose *Cancel*. If the trust relationship doesn't match, copy the policy into the *Edit trust policy* window and choose *Update policy*. +. Choose *Trust relationships*, and then choose *Edit trust policy*. +. Verify that the trust relationship contains the following policy. If the trust relationship matches the following policy, choose *Cancel*. If the trust relationship doesn't match, copy the policy into the *Edit trust policy* window and choose *Update policy*. + [source,json,subs="verbatim,attributes"] ---- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Service": "eks.amazonaws.com" - }, - "Action": [ - "sts:AssumeRole", - "sts:TagSession" - ] - } - ] -} +include::iam_snippet:4f04e1d1-8756-441d-aabf-98b754503967[] ---- [NOTE] @@ -57,50 +43,37 @@ You can use the following procedure to check and see if your account already has {aws} does not require the name `AmazonEKSAutoClusterRole` for this role. ==== +[#auto-cluster-iam-role-create] == Creating the Amazon EKS cluster role You can use the {aws-management-console} or the {aws} CLI to create the cluster role. - +[#auto-cluster-iam-role-console] === {aws-management-console} . Open the IAM console at https://console.aws.amazon.com/iam/. -. Choose *Roles*, then *Create role*. -. Under *Trusted entity type*, select *{aws} service*. -. From the *Use cases for other {aws} services* dropdown list, choose *EKS*. -. Choose *EKS - Cluster* for your use case, and then choose *Next*. -. On the *Add permissions* tab, select the policies and then choose *Next*. -** xref:security-iam-awsmanpol-AmazonEKSComputePolicy[AmazonEKSComputePolicy] -** xref:security-iam-awsmanpol-AmazonEKSBlockStoragePolicy[AmazonEKSBlockStoragePolicy] -** xref:security-iam-awsmanpol-AmazonEKSLoadBalancingPolicy[AmazonEKSLoadBalancingPolicy] -** xref:security-iam-awsmanpol-AmazonEKSNetworkingPolicy[AmazonEKSNetworkingPolicy] -** xref:security-iam-awsmanpol-amazoneksclusterpolicy[AmazonEKSClusterPolicy] +. Choose *Roles*, then *Create role*. +. Under *Trusted entity type*, select *{aws} service*. +. From the *Use cases for other {aws} services* dropdown list, choose *EKS*. +. Choose *EKS - Cluster* for your use case, and then choose *Next*. +. On the *Add permissions* tab, select the policies and then choose *Next*. +** <> +** <> +** <> +** <> +** <> . For *Role name*, enter a unique name for your role, such as `AmazonEKSAutoClusterRole`. . For *Description*, enter descriptive text such as `Amazon EKS - Cluster role`. . Choose *Create role*. - +[#auto-cluster-iam-role-cli] === {aws} CLI . Copy the following contents to a file named [.replaceable]`cluster-trust-policy.json`. + [source,json,subs="verbatim,attributes"] ---- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Service": "eks.amazonaws.com" - }, - "Action": [ - "sts:AssumeRole", - "sts:TagSession" - ] - } - ] -} +include::iam_snippet:7ec38f13-ea46-440f-9a4e-009e8855e11b[] ---- . Create the role. You can replace [.replaceable]`AmazonEKSAutoClusterRole` with any name that you choose. + @@ -112,42 +85,47 @@ aws iam create-role \ ---- . Attach the required IAM policies to the role: -**AmazonEKSClusterPolicy**: +*AmazonEKSClusterPolicy*: -``` +[source,bash,subs="verbatim,attributes"] +---- aws iam attach-role-policy \ --role-name AmazonEKSAutoClusterRole \ - --policy-arn arn:aws:iam::aws:policy/AmazonEKSClusterPolicy -``` + --policy-arn {arn-aws}iam::aws:policy/AmazonEKSClusterPolicy +---- -**AmazonEKSComputePolicy**: +*AmazonEKSComputePolicy*: -``` +[source,bash,subs="verbatim,attributes"] +---- aws iam attach-role-policy \ --role-name AmazonEKSAutoClusterRole \ - --policy-arn arn:aws:iam::aws:policy/AmazonEKSComputePolicy -``` + --policy-arn {arn-aws}iam::aws:policy/AmazonEKSComputePolicy +---- -**AmazonEKSBlockStoragePolicy**: +*AmazonEKSBlockStoragePolicy*: -``` +[source,bash,subs="verbatim,attributes"] +---- aws iam attach-role-policy \ --role-name AmazonEKSAutoClusterRole \ - --policy-arn arn:aws:iam::aws:policy/AmazonEKSBlockStoragePolicy -``` + --policy-arn {arn-aws}iam::aws:policy/AmazonEKSBlockStoragePolicy +---- -**AmazonEKSLoadBalancingPolicy**: +*AmazonEKSLoadBalancingPolicy*: -``` +[source,bash,subs="verbatim,attributes"] +---- aws iam attach-role-policy \ --role-name AmazonEKSAutoClusterRole \ - --policy-arn arn:aws:iam::aws:policy/AmazonEKSLoadBalancingPolicy -``` + --policy-arn {arn-aws}iam::aws:policy/AmazonEKSLoadBalancingPolicy +---- -**AmazonEKSNetworkingPolicy**: +*AmazonEKSNetworkingPolicy*: -``` +[source,bash,subs="verbatim,attributes"] +---- aws iam attach-role-policy \ --role-name AmazonEKSAutoClusterRole \ - --policy-arn arn:aws:iam::aws:policy/AmazonEKSNetworkingPolicy -``` + --policy-arn {arn-aws}iam::aws:policy/AmazonEKSNetworkingPolicy +---- \ No newline at end of file diff --git a/latest/ug/security/iam-reference/auto-create-node-role.adoc b/latest/ug/security/iam-reference/auto-create-node-role.adoc index 24231ede9..e77c2dc4e 100644 --- a/latest/ug/security/iam-reference/auto-create-node-role.adoc +++ b/latest/ug/security/iam-reference/auto-create-node-role.adoc @@ -1,10 +1,9 @@ -//!!NODE_ROOT
+include::../../attributes.txt[] + [.topic] -[[auto-create-node-role,auto-create-node-role.title]] +[#auto-create-node-role] = Amazon EKS Auto Mode node IAM role -:idprefix: id_ - -include::../../attributes.txt[] +:info_titleabbrev: Auto Mode node IAM role // write short desc @@ -17,10 +16,10 @@ You can't use the same role that is used to create any clusters. Before you create nodes, you must create an IAM role with the following policies, or equivalent permissions: -* xref:security-iam-awsmanpol-AmazonEKSWorkerNodeMinimalPolicy[AmazonEKSWorkerNodeMinimalPolicy] +* <> * link:AmazonECR/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-AmazonEC2ContainerRegistryPullOnly["AmazonEC2ContainerRegistryPullOnly",type="documentation"] - +[#auto-create-node-role-check] == Check for an existing node role You can use the following procedure to check and see if your account already has the Amazon EKS node role. @@ -30,52 +29,41 @@ You can use the following procedure to check and see if your account already has . Search the list of roles for `AmazonEKSAutoNodeRole`. If a role with one of those names doesn't exist, then see instructions in the next section to create the role. If a role that contains `AmazonEKSAutoNodeRole` does exist, then select the role to view the attached policies. . Choose *Permissions*. . Ensure that the required policies above are attached, or equivalent custom policies. -. Choose *Trust relationships*, and then choose *Edit trust policy*. -. Verify that the trust relationship contains the following policy. If the trust relationship matches the following policy, choose *Cancel*. If the trust relationship doesn't match, copy the policy into the *Edit trust policy* window and choose *Update policy*. +. Choose *Trust relationships*, and then choose *Edit trust policy*. +. Verify that the trust relationship contains the following policy. If the trust relationship matches the following policy, choose *Cancel*. If the trust relationship doesn't match, copy the policy into the *Edit trust policy* window and choose *Update policy*. + [source,json,subs="verbatim,attributes"] ---- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Service": "ec2.amazonaws.com" - }, - "Action": "sts:AssumeRole" - } - ] -} +include::iam_snippet:584c629e-58a0-41b7-a7b6-8bf95de49c53[] ---- - +[#auto-create-node-role-iam] == Creating the Amazon EKS node IAM role You can create the node IAM role with the {aws-management-console} or the {aws} CLI. - +[#auto-create-node-role-console] === {aws-management-console} . Open the IAM console at https://console.aws.amazon.com/iam/. . In the left navigation pane, choose *Roles*. -. On the *Roles* page, choose *Create role*. +. On the *Roles* page, choose *Create role*. . On the *Select trusted entity* page, do the following: + -.. In the *Trusted entity type* section, choose *{aws} service*. -.. Under *Use case*, choose *EC2*. +.. In the *Trusted entity type* section, choose *{aws} service*. +.. Under *Use case*, choose *EC2*. .. Choose *Next*. . On the *Add permissions* page, attach the following policies: -** xref:security-iam-awsmanpol-AmazonEKSWorkerNodeMinimalPolicy[AmazonEKSWorkerNodeMinimalPolicy] +** <> ** link:AmazonECR/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-AmazonEC2ContainerRegistryPullOnly["AmazonEC2ContainerRegistryPullOnly",type="documentation"] . On the *Name, review, and create* page, do the following: + .. For *Role name*, enter a unique name for your role, such as `AmazonEKSAutoNodeRole`. .. For *Description*, replace the current text with descriptive text such as `Amazon EKS - Node role`. -.. Under *Add tags (Optional)*, add metadata to the role by attaching tags as key-value pairs. For more information about using tags in IAM, see link:IAM/latest/UserGuide/id_tags.html[Tagging IAM resources,type="documentation"] in the _IAM User Guide_. +.. Under *Add tags (Optional)*, add metadata to the role by attaching tags as key-value pairs. For more information about using tags in IAM, see link:IAM/latest/UserGuide/id_tags.html[Tagging IAM resources,type="documentation"] in the _IAM User Guide_. .. Choose *Create role*. - +[#auto-create-node-role-cli] === {aws} CLI *Create the Node IAM Role* @@ -102,16 +90,18 @@ Attach the following {aws} managed policies to the Node IAM Role to provide the To attach AmazonEKSWorkerNodeMinimalPolicy: -``` +[source,bash,subs="verbatim,attributes"] +---- aws iam attach-role-policy \ --role-name AmazonEKSAutoNodeRole \ - --policy-arn arn:aws:iam::aws:policy/AmazonEKSWorkerNodeMinimalPolicy -``` + --policy-arn {arn-aws}iam::aws:policy/AmazonEKSWorkerNodeMinimalPolicy +---- To attach AmazonEC2ContainerRegistryPullOnly: -``` +[source,bash,subs="verbatim,attributes"] +---- aws iam attach-role-policy \ --role-name AmazonEKSAutoNodeRole \ - --policy-arn arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPullOnly -``` + --policy-arn {arn-aws}iam::aws:policy/AmazonEC2ContainerRegistryPullOnly +---- \ No newline at end of file diff --git a/latest/ug/security/iam-reference/cluster-iam-role.adoc b/latest/ug/security/iam-reference/cluster-iam-role.adoc index fe9ba478c..4c11600f1 100644 --- a/latest/ug/security/iam-reference/cluster-iam-role.adoc +++ b/latest/ug/security/iam-reference/cluster-iam-role.adoc @@ -1,56 +1,27 @@ -//!!NODE_ROOT
+include::../../attributes.txt[] + [.topic] -[[cluster-iam-role,cluster-iam-role.title]] +[#cluster-iam-role] = Amazon EKS cluster IAM role - -include::../../attributes.txt[] +:info_titleabbrev: Cluster IAM role [abstract] -- Learn how to create and configure the required {aws} Identity and Access Management role for Amazon EKS clusters to manage nodes and load balancers using managed or custom IAM policies. -- -An Amazon EKS cluster IAM role is required for each cluster. [.noloc]`Kubernetes` clusters managed by Amazon EKS use this role to manage nodes and the https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/service/annotations/#legacy-cloud-provider[legacy Cloud Provider] uses this role to create load balancers with Elastic Load Balancing for services. +An Amazon EKS cluster IAM role is required for each cluster. Kubernetes clusters managed by Amazon EKS use this role to manage nodes and the https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/guide/service/annotations/#legacy-cloud-provider[legacy Cloud Provider] uses this role to create load balancers with Elastic Load Balancing for services. Before you can create Amazon EKS clusters, you must create an IAM role with either of the following IAM policies: * link:aws-managed-policy/latest/reference/AmazonEKSClusterPolicy.html[AmazonEKSClusterPolicy,type="documentation"] -* A custom IAM policy. The minimal permissions that follow allows the [.noloc]`Kubernetes` cluster to manage nodes, but doesn't allow the https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/service/annotations/#legacy-cloud-provider[legacy Cloud Provider] to create load balancers with Elastic Load Balancing. Your custom IAM policy must have at least the following permissions: +* A custom IAM policy. The minimal permissions that follow allows the Kubernetes cluster to manage nodes, but doesn't allow the https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/guide/service/annotations/#legacy-cloud-provider[legacy Cloud Provider] to create load balancers with Elastic Load Balancing. Your custom IAM policy must have at least the following permissions: + [source,json,subs="verbatim,attributes"] ---- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "ec2:CreateTags" - ], - "Resource": "{arn-aws}ec2:*:*:instance/*", - "Condition": { - "ForAnyValue:StringLike": { - "aws:TagKeys": "kubernetes.io/cluster/*" - } - } - }, - { - "Effect": "Allow", - "Action": [ - "ec2:DescribeInstances", - "ec2:DescribeNetworkInterfaces", - "ec2:DescribeVpcs", - "ec2:DescribeDhcpOptions", - "ec2:DescribeAvailabilityZones", - "ec2:DescribeInstanceTopology", - "kms:DescribeKey" - ], - "Resource": "*" - } - ] -} +include::iam_snippet:de80ec81-9dde-4bf1-83df-de287477358d[] ---- @@ -63,7 +34,7 @@ Prior to April 16, 2020, link:aws-managed-policy/latest/reference/AmazonEKSServi ==== -[[check-service-role,check-service-role.title]] +[#check-service-role] == Check for an existing cluster role You can use the following procedure to check and see if your account already has the Amazon EKS cluster role. @@ -73,27 +44,16 @@ You can use the following procedure to check and see if your account already has . Search the list of roles for `eksClusterRole`. If a role that includes `eksClusterRole` doesn't exist, then see <> to create the role. If a role that includes `eksClusterRole` does exist, then select the role to view the attached policies. . Choose *Permissions*. . Ensure that the *AmazonEKSClusterPolicy* managed policy is attached to the role. If the policy is attached, your Amazon EKS cluster role is properly configured. -. Choose *Trust relationships*, and then choose *Edit trust policy*. -. Verify that the trust relationship contains the following policy. If the trust relationship matches the following policy, choose *Cancel*. If the trust relationship doesn't match, copy the policy into the *Edit trust policy* window and choose *Update policy*. +. Choose *Trust relationships*, and then choose *Edit trust policy*. +. Verify that the trust relationship contains the following policy. If the trust relationship matches the following policy, choose *Cancel*. If the trust relationship doesn't match, copy the policy into the *Edit trust policy* window and choose *Update policy*. + [source,json,subs="verbatim,attributes"] ---- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Service": "eks.amazonaws.com" - }, - "Action": "sts:AssumeRole" - } - ] -} +include::iam_snippet:c83fb40d-63b7-4444-992f-a9a3d76e2396[] ---- -[[create-service-role,create-service-role.title]] +[#create-service-role] == Creating the Amazon EKS cluster role You can use the {aws-management-console} or the {aws} CLI to create the cluster role. @@ -102,11 +62,11 @@ You can use the {aws-management-console} or the {aws} CLI to create the cluster {aws-management-console}:: .. Open the IAM console at https://console.aws.amazon.com/iam/. -.. Choose *Roles*, then *Create role*. -.. Under *Trusted entity type*, select *{aws} service*. -.. From the *Use cases for other {aws} services* dropdown list, choose *EKS*. -.. Choose *EKS - Cluster* for your use case, and then choose *Next*. -.. On the *Add permissions* tab, choose *Next*. +.. Choose *Roles*, then *Create role*. +.. Under *Trusted entity type*, select *{aws} service*. +.. From the *Use cases for other {aws} services* dropdown list, choose *EKS*. +.. Choose *EKS - Cluster* for your use case, and then choose *Next*. +.. On the *Add permissions* tab, choose *Next*. .. For *Role name*, enter a unique name for your role, such as `eksClusterRole`. .. For *Description*, enter descriptive text such as `Amazon EKS - Cluster role`. .. Choose *Create role*. @@ -117,18 +77,7 @@ You can use the {aws-management-console} or the {aws} CLI to create the cluster + [source,json,subs="verbatim,attributes"] ---- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Service": "eks.amazonaws.com" - }, - "Action": "sts:AssumeRole" - } - ] -} +include::iam_snippet:cd984bb8-7fc7-4d7b-8cb6-38a2bb0a48fc[] ---- .. Create the role. You can replace [.replaceable]`eksClusterRole` with any name that you choose. + @@ -145,4 +94,4 @@ aws iam create-role \ aws iam attach-role-policy \ --policy-arn {arn-aws}iam::aws:policy/AmazonEKSClusterPolicy \ --role-name eksClusterRole ----- +---- \ No newline at end of file diff --git a/latest/ug/security/iam-reference/connector-iam-role.adoc b/latest/ug/security/iam-reference/connector-iam-role.adoc new file mode 100644 index 000000000..47a90b5c0 --- /dev/null +++ b/latest/ug/security/iam-reference/connector-iam-role.adoc @@ -0,0 +1,121 @@ +include::../../attributes.txt[] + +[.topic] +[#connector-iam-role] += Amazon EKS connector IAM role +:info_titleabbrev: Connector IAM role + +You can connect Kubernetes clusters to view them in your {aws-management-console}. To connect to a Kubernetes cluster, create an IAM role. + +[#check-connector-role] +== Check for an existing EKS connector role + +You can use the following procedure to check and see if your account already has the Amazon EKS connector role. + +. Open the IAM console at https://console.aws.amazon.com/iam/. +. In the left navigation pane, choose *Roles*. +. Search the list of roles for `AmazonEKSConnectorAgentRole`. If a role that includes `AmazonEKSConnectorAgentRole` doesn't exist, then see <> to create the role. If a role that includes `AmazonEKSConnectorAgentRole` does exist, then select the role to view the attached policies. +. Choose *Permissions*. +. Ensure that the *AmazonEKSConnectorAgentPolicy* managed policy is attached to the role. If the policy is attached, your Amazon EKS connector role is properly configured. +. Choose *Trust relationships*, and then choose *Edit trust policy*. +. Verify that the trust relationship contains the following policy. If the trust relationship matches the following policy, choose *Cancel*. If the trust relationship doesn't match, copy the policy into the *Edit trust policy* window and choose *Update policy*. ++ +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:8a891a66-8698-44f2-a057-bbae3d155c47[] +---- + + +[#create-connector-role] +== Creating the Amazon EKS connector agent role + +You can use the {aws-management-console} or {aws} CloudFormation to create the connector agent role. + +{aws} CLI:: +.. Create a file named `eks-connector-agent-trust-policy.json` that contains the following JSON to use for the IAM role. ++ +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:2e26040d-2a2b-47d2-927c-2f51254195af[] +---- +.. Create a file named `eks-connector-agent-policy.json` that contains the following JSON to use for the IAM role. ++ +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:7e131b38-c620-4f45-9c4d-cec5c9b40408[] +---- +.. Create the Amazon EKS Connector agent role using the trust policy and policy you created in the previous list items. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam create-role \ + --role-name AmazonEKSConnectorAgentRole \ + --assume-role-policy-document file://eks-connector-agent-trust-policy.json +---- +.. Attach the policy to your Amazon EKS Connector agent role. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam put-role-policy \ + --role-name AmazonEKSConnectorAgentRole \ + --policy-name AmazonEKSConnectorAgentPolicy \ + --policy-document file://eks-connector-agent-policy.json +---- + + +{aws} CloudFormation:: +.. Save the following {aws} CloudFormation template to a text file on your local system. ++ +NOTE: This template also creates the service-linked role that would otherwise be created when the `registerCluster` API is called. See <> for details. ++ +[source,yaml,subs="verbatim,attributes"] +---- +--- +AWSTemplateFormatVersion: '2010-09-09' +Description: 'Provisions necessary resources needed to register clusters in EKS' +Parameters: {} +Resources: + EKSConnectorSLR: + Type: {aws}::IAM::ServiceLinkedRole + Properties: + AWSServiceName: eks-connector.amazonaws.com + + EKSConnectorAgentRole: + Type: {aws}::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: [ 'sts:AssumeRole' ] + Principal: + Service: 'ssm.amazonaws.com' + + EKSConnectorAgentPolicy: + Type: {aws}::IAM::Policy + Properties: + PolicyName: EKSConnectorAgentPolicy + Roles: + - {Ref: 'EKSConnectorAgentRole'} + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: 'Allow' + Action: [ 'ssmmessages:CreateControlChannel' ] + Resource: + - Fn::Sub: 'arn:${{aws}::Partition}:eks:*:*:cluster/*' + - Effect: 'Allow' + Action: [ 'ssmmessages:CreateDataChannel', 'ssmmessages:OpenDataChannel', 'ssmmessages:OpenControlChannel' ] + Resource: "*" +Outputs: + EKSConnectorAgentRoleArn: + Description: The agent role that EKS connector uses to communicate with {aws} services. + Value: !GetAtt EKSConnectorAgentRole.Arn +---- +.. Open the link:cloudformation/[{aws} CloudFormation console,type="console"]. +.. Choose *Create stack* with new resources (standard). +.. For *Specify template*, select *Upload a template file*, and then choose *Choose file*. +.. Choose the file you created earlier, and then choose *Next*. +.. For *Stack name*, enter a name for your role, such as `eksConnectorAgentRole`, and then choose *Next*. +.. On the *Configure stack options* page, choose *Next*. +.. On the *Review* page, review your information, acknowledge that the stack might create IAM resources, and then choose *Create stack*. \ No newline at end of file diff --git a/latest/ug/security/iam-reference/create-node-role.adoc b/latest/ug/security/iam-reference/create-node-role.adoc index abf05173c..29c890ca7 100644 --- a/latest/ug/security/iam-reference/create-node-role.adoc +++ b/latest/ug/security/iam-reference/create-node-role.adoc @@ -1,9 +1,9 @@ -//!!NODE_ROOT
+include::../../attributes.txt[] + [.topic] -[[create-node-role,create-node-role.title]] +[#create-node-role] = Amazon EKS node IAM role - -include::../../attributes.txt[] +:info_titleabbrev: Node IAM role The Amazon EKS node `kubelet` daemon makes calls to {aws} APIs on your behalf. Nodes receive permissions for these API calls through an IAM instance profile and associated policies. Before you can launch nodes and register them into a cluster, you must create an IAM role for those nodes to use when they are launched. This requirement applies to nodes launched with the Amazon EKS optimized AMI provided by Amazon, or with any other node AMIs that you intend to use. Additionally, this requirement applies to both managed node groups and self-managed nodes. @@ -18,8 +18,8 @@ Before you create nodes, you must create an IAM role with the following permissi * Permissions for the `kubelet` to describe Amazon EC2 resources in the VPC, such as provided by the link:aws-managed-policy/latest/reference/AmazonEKSWorkerNodePolicy.html[AmazonEKSWorkerNodePolicy,type="documentation"] policy. This policy also provides the permissions for the Amazon EKS Pod Identity Agent. * Permissions for the `kubelet` to use container images from Amazon Elastic Container Registry (Amazon ECR), such as provided by the link:aws-managed-policy/latest/reference/AmazonEC2ContainerRegistryPullOnly.html[AmazonEC2ContainerRegistryPullOnly,type="documentation"] policy. The permissions to use container images from Amazon Elastic Container Registry (Amazon ECR) are required because the built-in add-ons for networking run pods that use container images from Amazon ECR. -* (Optional) Permissions for the Amazon EKS Pod Identity Agent to use the `eks-auth:AssumeRoleForPodIdentity` action to retrieve credentials for pods. If you don't use the link:aws-managed-policy/latest/reference/AmazonEKSWorkerNodePolicy.html[AmazonEKSWorkerNodePolicy,type="documentation"], then you must provide this permission in addition to the EC2 permissions to use EKS Pod Identity.` -* (Optional) If you don't use IRSA or EKS Pod Identity to give permissions to the VPC CNI pods, then you must provide permissions for the VPC CNI on the instance role. You can use either the ` link:aws-managed-policy/latest/reference/AmazonEKS_CNI_Policy.html[AmazonEKS_CNI_Policy,type="documentation"]` managed policy (if you created your cluster with the IPv4` family) or an <> (if you created your cluster with the `IPv6` family). Rather than attaching the policy to this role however, we recommend that you attach the policy to a separate role used specifically for the Amazon VPC CNI add-on. For more information about creating a separate role for the Amazon VPC CNI add-on, see <>. +* (Optional) Permissions for the Amazon EKS Pod Identity Agent to use the `eks-auth:AssumeRoleForPodIdentity` action to retrieve credentials for pods. If you don't use the link:aws-managed-policy/latest/reference/AmazonEKSWorkerNodePolicy.html[AmazonEKSWorkerNodePolicy,type="documentation"], then you must provide this permission in addition to the EC2 permissions to use EKS Pod Identity. +* (Optional) If you don't use IRSA or EKS Pod Identity to give permissions to the VPC CNI pods, then you must provide permissions for the VPC CNI on the instance role. You can use either the link:aws-managed-policy/latest/reference/AmazonEKS_CNI_Policy.html[`AmazonEKS_CNI_Policy`,type="documentation"] managed policy (if you created your cluster with the `IPv4` family) or an <> (if you created your cluster with the `IPv6` family). Rather than attaching the policy to this role however, we recommend that you attach the policy to a separate role used specifically for the Amazon VPC CNI add-on. For more information about creating a separate role for the Amazon VPC CNI add-on, see <>. [NOTE] @@ -27,7 +27,7 @@ Before you create nodes, you must create an IAM role with the following permissi The Amazon EC2 node groups must have a different IAM role than the Fargate profile. For more information, see <>. ==== -[[check-worker-node-role,check-worker-node-role.title]] +[#check-worker-node-role] == Check for an existing node role You can use the following procedure to check and see if your account already has the Amazon EKS node role. @@ -36,34 +36,19 @@ You can use the following procedure to check and see if your account already has . In the left navigation pane, choose *Roles*. . Search the list of roles for `eksNodeRole`, `AmazonEKSNodeRole`, or `NodeInstanceRole`. If a role with one of those names doesn't exist, then see <> to create the role. If a role that contains `eksNodeRole`, `AmazonEKSNodeRole`, or `NodeInstanceRole` does exist, then select the role to view the attached policies. . Choose *Permissions*. -. Ensure that the *AmazonEKSWorkerNodePolicy* and *AmazonEC2ContainerRegistryPullOnly* managed policies are attached to the role or a custom policy is attached with the minimal permissions. +. Ensure that the *AmazonEKSWorkerNodePolicy* and *AmazonEC2ContainerRegistryPullOnly* managed policies are attached to the role or a custom policy is attached with the minimal permissions. + -NOTE: If the *AmazonEKS_CNI_Policy* policy is attached to the role, we recommend removing it and attaching it to an IAM role that is mapped to the `aws-node` [.noloc]`Kubernetes` service account instead. For more information, see <>. -. Choose *Trust relationships*, and then choose *Edit trust policy*. -. Verify that the trust relationship contains the following policy. If the trust relationship matches the following policy, choose *Cancel*. If the trust relationship doesn't match, copy the policy into the *Edit trust policy* window and choose *Update policy*. +NOTE: If the *AmazonEKS_CNI_Policy* policy is attached to the role, we recommend removing it and attaching it to an IAM role that is mapped to the `aws-node` Kubernetes service account instead. For more information, see <>. +. Choose *Trust relationships*, and then choose *Edit trust policy*. +. Verify that the trust relationship contains the following policy. If the trust relationship matches the following policy, choose *Cancel*. If the trust relationship doesn't match, copy the policy into the *Edit trust policy* window and choose *Update policy*. + [source,json,subs="verbatim,attributes"] ---- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "sts:AssumeRole" - ], - "Principal": { - "Service": [ - "ec2.amazonaws.com" - ] - } - } - ] -} +include::iam_snippet:fdc51e78-5f5f-470a-adfa-cf09e1f9c0ee[] ---- -[[create-worker-node-role,create-worker-node-role.title]] +[#create-worker-node-role] == Creating the Amazon EKS node IAM role You can create the node IAM role with the {aws-management-console} or the {aws} CLI. @@ -73,11 +58,11 @@ You can create the node IAM role with the {aws-management-console} or the {aws} {aws-management-console}:: .. Open the IAM console at https://console.aws.amazon.com/iam/. .. In the left navigation pane, choose *Roles*. -.. On the *Roles* page, choose *Create role*. +.. On the *Roles* page, choose *Create role*. .. On the *Select trusted entity* page, do the following: + -... In the *Trusted entity type* section, choose *{aws} service*. -... Under *Use case*, choose *EC2*. +... In the *Trusted entity type* section, choose *{aws} service*. +... Under *Use case*, choose *EC2*. ... Choose *Next*. .. On the *Add permissions* page, attach a custom policy or do the following: + @@ -87,13 +72,13 @@ You can create the node IAM role with the {aws-management-console} or the {aws} ... In the *Filter policies* box, enter `AmazonEC2ContainerRegistryPullOnly`. ... Select the check box to the left of *AmazonEC2ContainerRegistryPullOnly* in the search results. + -Either the *AmazonEKS_CNI_Policy* managed policy, or an <> that you create must also be attached to either this role or to a different role that's mapped to the `aws-node` [.noloc]`Kubernetes` service account. We recommend assigning the policy to the role associated to the [.noloc]`Kubernetes` service account instead of assigning it to this role. For more information, see <>. +Either the *AmazonEKS_CNI_Policy* managed policy, or an <> that you create must also be attached to either this role or to a different role that's mapped to the `aws-node` Kubernetes service account. We recommend assigning the policy to the role associated to the Kubernetes service account instead of assigning it to this role. For more information, see <>. ... Choose *Next*. .. On the *Name, review, and create* page, do the following: + ... For *Role name*, enter a unique name for your role, such as `AmazonEKSNodeRole`. ... For *Description*, replace the current text with descriptive text such as `Amazon EKS - Node role`. -... Under *Add tags (Optional)*, add metadata to the role by attaching tags as key-value pairs. For more information about using tags in IAM, see link:IAM/latest/UserGuide/id_tags.html[Tagging IAM resources,type="documentation"] in the _IAM User Guide_. +... Under *Add tags (Optional)*, add metadata to the role by attaching tags as key-value pairs. For more information about using tags in IAM, see link:IAM/latest/UserGuide/id_tags.html[Tagging IAM resources,type="documentation"] in the _IAM User Guide_. ... Choose *Create role*. @@ -102,24 +87,7 @@ Either the *AmazonEKS_CNI_Policy* managed policy, or an <node-role-trust-relationship.json <>. +.. Attach one of the following IAM policies to the IAM role depending on which IP family you created your cluster with. The policy must be attached to this role or to a role associated to the Kubernetes `aws-node` service account that's used for the Amazon VPC CNI plugin for Kubernetes. We recommend assigning the policy to the role associated to the Kubernetes service account. To assign the policy to the role associated to the Kubernetes service account, see <>. + *** IPv4 + @@ -156,31 +124,7 @@ aws iam attach-role-policy \ + [source,json,subs="verbatim,attributes"] ---- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "ec2:AssignIpv6Addresses", - "ec2:DescribeInstances", - "ec2:DescribeTags", - "ec2:DescribeNetworkInterfaces", - "ec2:DescribeInstanceTypes" - ], - "Resource": "*" - }, - { - "Effect": "Allow", - "Action": [ - "ec2:CreateTags" - ], - "Resource": [ - "{arn-aws}ec2:*:*:network-interface/*" - ] - } - ] -} +include::iam_snippet:92698140-4266-4007-b17c-7293f46f0036[] ---- .... Create the IAM policy. + @@ -195,4 +139,4 @@ aws iam create-policy --policy-name AmazonEKS_CNI_IPv6_Policy --policy-document aws iam attach-role-policy \ --policy-arn {arn-aws}iam::111122223333:policy/AmazonEKS_CNI_IPv6_Policy \ --role-name AmazonEKSNodeRole ----- +---- \ No newline at end of file diff --git a/latest/ug/security/iam-reference/pod-execution-role.adoc b/latest/ug/security/iam-reference/pod-execution-role.adoc new file mode 100644 index 000000000..4eed69242 --- /dev/null +++ b/latest/ug/security/iam-reference/pod-execution-role.adoc @@ -0,0 +1,121 @@ +include::../../attributes.txt[] + +[.topic] +[#pod-execution-role] += Amazon EKS Pod execution IAM role +:info_titleabbrev: Pod execution IAM role + +The Amazon EKS Pod execution role is required to run Pods on {aws} Fargate infrastructure. + +When your cluster creates Pods on {aws} Fargate infrastructure, the components running on the Fargate infrastructure must make calls to {aws} APIs on your behalf. This is so that they can do actions such as pull container images from Amazon ECR or route logs to other {aws} services. The Amazon EKS Pod execution role provides the IAM permissions to do this. + +When you create a Fargate profile, you must specify a Pod execution role for the Amazon EKS components that run on the Fargate infrastructure using the profile. This role is added to the cluster's Kubernetes https://kubernetes.io/docs/reference/access-authn-authz/rbac/[Role based access control] (RBAC) for authorization. This allows the `kubelet` that's running on the Fargate infrastructure to register with your Amazon EKS cluster so that it can appear in your cluster as a node. + +[NOTE] +==== + +The Fargate profile must have a different IAM role than Amazon EC2 node groups. + +==== + +[IMPORTANT] +==== + +The containers running in the Fargate Pod can't assume the IAM permissions associated with a Pod execution role. To give the containers in your Fargate Pod permissions to access other {aws} services, you must use <>. + +==== + +Before you create a Fargate profile, you must create an IAM role with the link:aws-managed-policy/latest/reference/AmazonEKSFargatePodExecutionRolePolicy.html[AmazonEKSFargatePodExecutionRolePolicy,type="documentation"]. +[#check-pod-execution-role] +== Check for a correctly configured existing Pod execution role + +You can use the following procedure to check and see if your account already has a correctly configured Amazon EKS Pod execution role. To avoid a confused deputy security problem, it's important that the role restricts access based on `SourceArn`. You can modify the execution role as needed to include support for Fargate profiles on other clusters. + +. Open the IAM console at https://console.aws.amazon.com/iam/. +. In the left navigation pane, choose *Roles*. +. On the *Roles* page, search the list of roles for *AmazonEKSFargatePodExecutionRole*. If the role doesn't exist, see <> to create the role. If the role does exist, choose the role. +. On the *AmazonEKSFargatePodExecutionRole* page, do the following: ++ +.. Choose *Permissions*. +.. Ensure that the *AmazonEKSFargatePodExecutionRolePolicy* Amazon managed policy is attached to the role. +.. Choose *Trust relationships*. +.. Choose *Edit trust policy*. +. On the *Edit trust policy* page, verify that the trust relationship contains the following policy and has a line for Fargate profiles on your cluster. If so, choose *Cancel*. ++ +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:98e67c07-8932-4f31-9b4c-145bf8c2613c[] +---- ++ +If the policy matches but doesn't have a line specifying the Fargate profiles on your cluster, you can add the following line at the top of the `ArnLike` object. Replace [.replaceable]`region-code` with the {aws} Region that your cluster is in, [.replaceable]`111122223333` with your account ID, and [.replaceable]`my-cluster` with the name of your cluster. ++ +[source,json,subs="verbatim,attributes"] +---- +"aws:SourceArn": "{arn-aws}eks:region-code:111122223333:fargateprofile/my-cluster/*", +---- ++ +If the policy doesn't match, copy the full previous policy into the form and choose *Update policy*. Replace [.replaceable]`region-code` with the {aws} Region that your cluster is in. If you want to use the same role in all {aws} Regions in your account, replace [.replaceable]`region-code` with `{asterisk}`. Replace [.replaceable]`111122223333` with your account ID and [.replaceable]`my-cluster` with the name of your cluster. If you want to use the same role for all clusters in your account, replace [.replaceable]`my-cluster` with `{asterisk}`. + + +[#create-pod-execution-role] +== Creating the Amazon EKS Pod execution role + +If you don't already have the Amazon EKS Pod execution role for your cluster, you can use the {aws-management-console} or the {aws} CLI to create it. + + + +{aws-management-console}:: +.. Open the IAM console at https://console.aws.amazon.com/iam/. +.. In the left navigation pane, choose *Roles*. +.. On the *Roles* page, choose *Create role*. +.. On the *Select trusted entity* page, do the following: ++ +... In the *Trusted entity type* section, choose *{aws} service*. +... From the *Use cases for other {aws} services* dropdown list, choose *EKS*. +... Choose *EKS - Fargate Pod*. +... Choose *Next*. +.. On the *Add permissions* page, choose *Next*. +.. On the *Name, review, and create* page, do the following: ++ +... For *Role name*, enter a unique name for your role, such as `AmazonEKSFargatePodExecutionRole`. +... Under *Add tags (Optional)*, add metadata to the role by attaching tags as key-value pairs. For more information about using tags in IAM, see link:IAM/latest/UserGuide/id_tags.html[Tagging IAM resources,type="documentation"] in the _IAM User Guide_. +... Choose *Create role*. +.. On the *Roles* page, search the list of roles for *AmazonEKSFargatePodExecutionRole*. Choose the role. +.. On the *AmazonEKSFargatePodExecutionRole* page, do the following: ++ +... Choose *Trust relationships*. +... Choose *Edit trust policy*. +.. On the *Edit trust policy* page, do the following: ++ +... Copy and paste the following contents into the *Edit trust policy* form. Replace [.replaceable]`region-code` with the {aws} Region that your cluster is in. If you want to use the same role in all {aws} Regions in your account, replace [.replaceable]`region-code` with `{asterisk}`. Replace [.replaceable]`111122223333` with your account ID and [.replaceable]`my-cluster` with the name of your cluster. If you want to use the same role for all clusters in your account, replace [.replaceable]`my-cluster` with `{asterisk}`. ++ +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:44cd2230-767e-4bc2-a43c-fe1860ad8774[] +---- +... Choose *Update policy*. + + +{aws} CLI:: +.. Copy and paste the following contents to a file named `pod-execution-role-trust-policy.json`. Replace [.replaceable]`region-code` with the {aws} Region that your cluster is in. If you want to use the same role in all {aws} Regions in your account, replace [.replaceable]`region-code` with `{asterisk}`. Replace [.replaceable]`111122223333` with your account ID and [.replaceable]`my-cluster` with the name of your cluster. If you want to use the same role for all clusters in your account, replace [.replaceable]`my-cluster` with `{asterisk}`. ++ +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:7a7df467-ae7f-495d-982f-d4b325abe470[] +---- +.. Create a Pod execution IAM role. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam create-role \ + --role-name AmazonEKSFargatePodExecutionRole \ + --assume-role-policy-document file://"pod-execution-role-trust-policy.json" +---- +.. Attach the required Amazon EKS managed IAM policy to the role. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam attach-role-policy \ + --policy-arn {arn-aws}iam::aws:policy/AmazonEKSFargatePodExecutionRolePolicy \ + --role-name AmazonEKSFargatePodExecutionRole +---- \ No newline at end of file diff --git a/latest/ug/security/iam-reference/security-iam-awsmanpol.adoc b/latest/ug/security/iam-reference/security-iam-awsmanpol.adoc index b853d34d6..f23f1e29e 100644 --- a/latest/ug/security/iam-reference/security-iam-awsmanpol.adoc +++ b/latest/ug/security/iam-reference/security-iam-awsmanpol.adoc @@ -1,7 +1,7 @@ -//!!NODE_ROOT
include::../../attributes.txt[] + [.topic] -[[security-iam-awsmanpol,security-iam-awsmanpol.title]] +[#security-iam-awsmanpol] = {aws} managed policies for Amazon Elastic Kubernetes Service :info_titleabbrev: {aws} managed policies @@ -12,50 +12,69 @@ Learn about {aws} managed policies for Amazon EKS and recent changes to those po An {aws} managed policy is a standalone policy that is created and administered by {aws}. {aws} managed policies are designed to provide permissions for many common use cases so that you can start assigning permissions to users, groups, and roles. -Keep in mind that {aws} managed policies might not grant least-privilege permissions for your specific use cases because they're available for all {aws} customers to use. We recommend that you reduce permissions further by defining link:IAM/latest/UserGuide/access_policies_managed-vs-inline.html#customer-managed-policies[customer managed policies,type="documentation"] that are specific to your use cases. +Keep in mind that {aws} managed policies might not grant least-privilege permissions for your specific use cases because they're available for all {aws} customers to use. We recommend that you reduce permissions further by defining link:IAM/latest/UserGuide/access_policies_managed-vs-inline.html#customer-managed-policies[customer managed policies,type="documentation"] that are specific to your use cases. You cannot change the permissions defined in {aws} managed policies. If {aws} updates the permissions defined in an {aws} managed policy, the update affects all principal identities (users, groups, and roles) that the policy is attached to. {aws} is most likely to update an {aws} managed policy when a new {aws} service is launched or new API operations become available for existing services. -For more information, see link:IAM/latest/UserGuide/access_policies_managed-vs-inline.html#aws-managed-policies[{aws} managed policies,type="documentation"] in the _IAM User Guide_. +For more information, see link:IAM/latest/UserGuide/access_policies_managed-vs-inline.html#aws-managed-policies[{aws} managed policies,type="documentation"] in the _IAM User Guide_. -[[security-iam-awsmanpol-amazoneks-cni-policy,security-iam-awsmanpol-amazoneks-cni-policy.title]] +[#security-iam-awsmanpol-amazoneks-cni-policy] == {aws} managed policy: AmazonEKS_CNI_Policy :info_titleabbrev: AmazonEKS_CNI_Policy -You can attach the `AmazonEKS_CNI_Policy` to your IAM entities. Before you create an Amazon EC2 node group, this policy must be attached to either the <>, or to an IAM role that's used specifically by the [.noloc]`Amazon VPC CNI plugin for Kubernetes`. This is so that it can perform actions on your behalf. We recommend that you attach the policy to a role that's used only by the plugin. For more information, see <> and <>. +You can attach the `AmazonEKS_CNI_Policy` to your IAM entities. Before you create an Amazon EC2 node group, this policy must be attached to either the <>, or to an IAM role that's used specifically by the Amazon VPC CNI plugin for Kubernetes. This is so that it can perform actions on your behalf. We recommend that you attach the policy to a role that's used only by the plugin. For more information, see <> and <>. *Permissions details* This policy includes the following permissions that allow Amazon EKS to complete the following tasks: -* *`ec2:*NetworkInterface` and `ec2:*PrivateIpAddresses`* – Allows the Amazon VPC CNI plugin to perform actions such as provisioning Elastic Network Interfaces and IP addresses for [.noloc]`Pods` to provide networking for applications that run in Amazon EKS. +* *`ec2:*NetworkInterface` and `ec2:*PrivateIpAddresses`* – Allows the Amazon VPC CNI plugin to perform actions such as provisioning Elastic Network Interfaces and IP addresses for Pods to provide networking for applications that run in Amazon EKS. * *`ec2` read actions* – Allows the Amazon VPC CNI plugin to perform actions such as describe instances and subnets to see the amount of free IP addresses in your Amazon VPC subnets. The VPC CNI can use the free IP addresses in each subnet to pick the subnets with the most free IP addresses to use when creating an elastic network interface. -To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEKS_CNI_Policy.html#AmazonEKS_CNI_Policy-json[AmazonEKS_CNI_Policy,type="documentation"] in the {aws} Managed Policy Reference Guide. +To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEKS_CNI_Policy.html#AmazonEKS_CNI_Policy-json[AmazonEKS_CNI_Policy,type="documentation"] in the {aws} Managed Policy Reference Guide. -[[security-iam-awsmanpol-amazoneksclusterpolicy,security-iam-awsmanpol-amazoneksclusterpolicy.title]] +[#security-iam-awsmanpol-amazoneksclusterpolicy] == {aws} managed policy: AmazonEKSClusterPolicy :info_titleabbrev: AmazonEKSClusterPolicy -You can attach `AmazonEKSClusterPolicy` to your IAM entities. Before creating a cluster, you must have a <> with this policy attached. [.noloc]`Kubernetes` clusters that are managed by Amazon EKS make calls to other {aws} services on your behalf. They do this to manage the resources that you use with the service. +You can attach `AmazonEKSClusterPolicy` to your IAM entities. Before creating a cluster, you must have a <> with this policy attached. Kubernetes clusters that are managed by Amazon EKS make calls to other {aws} services on your behalf. They do this to manage the resources that you use with the service. This policy includes the following permissions that allow Amazon EKS to complete the following tasks: * *`autoscaling`* – Read and update the configuration of an Auto Scaling group. These permissions aren't used by Amazon EKS but remain in the policy for backwards compatibility. -* *`ec2`* – Work with volumes and network resources that are associated to Amazon EC2 nodes. This is required so that the [.noloc]`Kubernetes` control plane can join instances to a cluster and dynamically provision and manage Amazon EBS volumes that are requested by [.noloc]`Kubernetes` persistent volumes. -* *`elasticloadbalancing`* – Work with Elastic Load Balancers and add nodes to them as targets. This is required so that the [.noloc]`Kubernetes` control plane can dynamically provision Elastic Load Balancers requested by [.noloc]`Kubernetes` services. -* *`iam`* – Create a service-linked role. This is required so that the [.noloc]`Kubernetes` control plane can dynamically provision Elastic Load Balancers that are requested by [.noloc]`Kubernetes` services. -* *`kms`* – Read a key from {aws} KMS. This is required for the [.noloc]`Kubernetes` control plane to support https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/[secrets encryption] of [.noloc]`Kubernetes` secrets stored in `etcd`. +* *`ec2`* – Work with volumes and network resources that are associated to Amazon EC2 nodes. This is required so that the Kubernetes control plane can join instances to a cluster and dynamically provision and manage Amazon EBS volumes that are requested by Kubernetes persistent volumes. +* *`ec2`* - Delete elastic network interfaces that are created by the VPC CNI. This is required so that EKS can clean up elastic network interfaces that are left behind if the VPC CNI quits unexpectedly. +* *`elasticloadbalancing`* – Work with Elastic Load Balancers and add nodes to them as targets. This is required so that the Kubernetes control plane can dynamically provision Elastic Load Balancers requested by Kubernetes services. +* *`iam`* – Create a service-linked role. This is required so that the Kubernetes control plane can dynamically provision Elastic Load Balancers that are requested by Kubernetes services. +* *`kms`* – Read a key from {aws} KMS. This is required for the Kubernetes control plane to support https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/[secrets encryption] of Kubernetes secrets stored in `etcd`. + +To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEKSClusterPolicy.html#AmazonEKSClusterPolicy-json[AmazonEKSClusterPolicy,type="documentation"] in the {aws} Managed Policy Reference Guide. + -To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEKSClusterPolicy.html#AmazonEKSClusterPolicy-json[AmazonEKSClusterPolicy,type="documentation"] in the {aws} Managed Policy Reference Guide. +[#security-iam-awsmanpol-amazoneksdashboardconsolereadonly] +== {aws} managed policy: AmazonEKSDashboardConsoleReadOnly +:info_titleabbrev: AmazonEKSDashboardConsoleReadOnly + +You can attach `AmazonEKSDashboardConsoleReadOnly` to your IAM entities. + +This policy includes the following permissions that allow Amazon EKS to complete the following tasks: -[[security-iam-awsmanpol-amazoneksfargatepodexecutionrolepolicy,security-iam-awsmanpol-amazoneksfargatepodexecutionrolepolicy.title]] +* *`eks`* - Read-only access to EKS dashboard data, resources, and cluster versions information. This allows viewing EKS-related metrics and cluster configuration details. + +* *`organizations`* - Read-only access to {aws} Organizations information, including: +** Viewing organization details and service access +** Listing organizational roots, accounts, and organizational units +** Viewing organization structure + +To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEKSDashboardConsoleReadOnly.html#AmazonEKSDashboardConsoleReadOnly-json[AmazonEKSDashboardConsoleReadOnly,type="documentation"] in the {aws} Managed Policy Reference Guide. + +[#security-iam-awsmanpol-amazoneksfargatepodexecutionrolepolicy] == {aws} managed policy: AmazonEKSFargatePodExecutionRolePolicy :info_titleabbrev: AmazonEKSFargatePodExecutionRolePolicy -You can attach `AmazonEKSFargatePodExecutionRolePolicy` to your IAM entities. Before you can create a Fargate profile, you must create a Fargate [.noloc]`Pod` execution role and attach this policy to it. For more information, see <> and <>. +You can attach `AmazonEKSFargatePodExecutionRolePolicy` to your IAM entities. Before you can create a Fargate profile, you must create a Fargate Pod execution role and attach this policy to it. For more information, see <> and <>. -This policy grants the role the permissions that provide access to other {aws} service resources that are required to run Amazon EKS [.noloc]`Pods` on Fargate. +This policy grants the role the permissions that provide access to other {aws} service resources that are required to run Amazon EKS Pods on Fargate. *Permissions details* @@ -63,9 +82,31 @@ This policy includes the following permissions that allow Amazon EKS to complete * *`ecr`* – Allows Pods that are running on Fargate to pull container images that are stored in Amazon ECR. -To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEKSFargatePodExecutionRolePolicy.html#AmazonEKSFargatePodExecutionRolePolicy-json[AmazonEKSFargatePodExecutionRolePolicy,type="documentation"] in the {aws} Managed Policy Reference Guide. +To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEKSFargatePodExecutionRolePolicy.html#AmazonEKSFargatePodExecutionRolePolicy-json[AmazonEKSFargatePodExecutionRolePolicy,type="documentation"] in the {aws} Managed Policy Reference Guide. + +[#security-iam-awsmanpol-AmazonEKSConnectorServiceRolePolicy] +== {aws} managed policy: AmazonEKSConnectorServiceRolePolicy +:info_titleabbrev: AmazonEKSConnectorServiceRolePolicy + +You can't attach `AmazonEKSConnectorServiceRolePolicy` to your IAM entities. This policy is attached to a service-linked role that allows Amazon EKS to perform actions on your behalf. For more information, see <>. + +The role allows Amazon EKS to connect Kubernetes clusters. The attached policies allow the role to manage necessary resources to connect to your registered Kubernetes cluster. -[[security-iam-awsmanpol-amazoneksforfargateservicerolepolicy,security-iam-awsmanpol-amazoneksforfargateservicerolepolicy.title]] +*Permissions details* + +This policy includes the following permissions that allow Amazon EKS to complete the following tasks. + +* *`SSM Management`* – Create, describe, and delete SSM activations, and deregister managed instances. This allows basic Systems Manager operations. + +* *`Session Management`* – Start SSM sessions specifically for EKS clusters and execute non-interactive commands using the AmazonEKS document. + +* *`IAM Role Passing`* – Pass IAM roles specifically to the SSM service, controlled by a condition that restricts the passed roles to `ssm.amazonaws.com`. + +* *`EventBridge Rules`* – Create EventBridge rules and targets, but only when managed by `eks-connector.amazonaws.com`. Rules are specifically limited to {aws} SSM as the event source. + +To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEKSConnectorServiceRolePolicy.html[AmazonEKSConnectorServiceRolePolicy,type="documentation"] in the {aws} Managed Policy Reference Guide. + +[#security-iam-awsmanpol-amazoneksforfargateservicerolepolicy] == {aws} managed policy: AmazonEKSForFargateServiceRolePolicy :info_titleabbrev: AmazonEKSForFargateServiceRolePolicy @@ -79,15 +120,15 @@ This policy includes the following permissions that allow Amazon EKS to complete * *`ec2`* – Create and delete Elastic Network Interfaces and describe Elastic Network Interfaces and resources. This is required so that the Amazon EKS Fargate service can configure the VPC networking that's required for Fargate Pods. -To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEKSForFargateServiceRolePolicy.html#AmazonEKSForFargateServiceRolePolicy-json[AmazonEKSForFargateServiceRolePolicy,type="documentation"] in the {aws} Managed Policy Reference Guide. +To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEKSForFargateServiceRolePolicy.html#AmazonEKSForFargateServiceRolePolicy-json[AmazonEKSForFargateServiceRolePolicy,type="documentation"] in the {aws} Managed Policy Reference Guide. -[[security-iam-awsmanpol-AmazonEKSComputePolicy,security-iam-awsmanpol-AmazonEKSComputePolicy.title]] +[#security-iam-awsmanpol-AmazonEKSComputePolicy] == {aws} managed policy: AmazonEKSComputePolicy :info_titleabbrev: AmazonEKSComputePolicy You can attach `AmazonEKSComputePolicy` to your IAM entities. You may attach this policy to your <> to expand the resources EKS can manage in your account. -This policy grants the permissions required for Amazon EKS to create and manage EC2 instances for the EKS cluster, as well as the necessary IAM permissions to configure EC2. +This policy grants the permissions required for Amazon EKS to create and manage EC2 instances for the EKS cluster, and the necessary IAM permissions to configure EC2. Also, this policy grants the permissions for Amazon EKS to create the EC2 Spot service-linked role on your behalf. === Permissions details @@ -103,9 +144,9 @@ This policy includes the following permissions that allow Amazon EKS to complete - `iam:AddRoleToInstanceProfile` - Allows adding an IAM role to the EKS compute instance profile. - `iam:PassRole` - Allows passing the necessary IAM roles to the EC2 service. -To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEKSComputePolicy.html#AmazonEKSComputePolicy-json[AmazonEKSComputePolicy,type="documentation"] in the {aws} Managed Policy Reference Guide. +To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEKSComputePolicy.html#AmazonEKSComputePolicy-json[AmazonEKSComputePolicy,type="documentation"] in the {aws} Managed Policy Reference Guide. -[[security-iam-awsmanpol-AmazonEKSNetworkingPolicy,security-iam-awsmanpol-AmazonEKSNetworkingPolicy.title]] +[#security-iam-awsmanpol-AmazonEKSNetworkingPolicy] == {aws} managed policy: AmazonEKSNetworkingPolicy :info_titleabbrev: AmazonEKSNetworkingPolicy @@ -129,11 +170,11 @@ This policy grants the following permissions to allow Amazon EKS to manage netwo -To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEKSNetworkingPolicy.html#AmazonEKSNetworkingPolicy-json[AmazonEKSNetworkingPolicy,type="documentation"] in the {aws} Managed Policy Reference Guide. +To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEKSNetworkingPolicy.html#AmazonEKSNetworkingPolicy-json[AmazonEKSNetworkingPolicy,type="documentation"] in the {aws} Managed Policy Reference Guide. -[[security-iam-awsmanpol-AmazonEKSBlockStoragePolicy,security-iam-awsmanpol-AmazonEKSBlockStoragePolicy.title]] +[#security-iam-awsmanpol-AmazonEKSBlockStoragePolicy] == {aws} managed policy: AmazonEKSBlockStoragePolicy :info_titleabbrev: AmazonEKSBlockStoragePolicy @@ -158,10 +199,10 @@ This IAM policy grants the following permissions to allow Amazon EKS to manage E -To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEKSBlockStoragePolicy.html#AmazonEKSBlockStoragePolicy-json[AmazonEKSBlockStoragePolicy,type="documentation"] in the {aws} Managed Policy Reference Guide. +To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEKSBlockStoragePolicy.html#AmazonEKSBlockStoragePolicy-json[AmazonEKSBlockStoragePolicy,type="documentation"] in the {aws} Managed Policy Reference Guide. -[[security-iam-awsmanpol-AmazonEKSLoadBalancingPolicy,security-iam-awsmanpol-AmazonEKSLoadBalancingPolicy.title]] +[#security-iam-awsmanpol-AmazonEKSLoadBalancingPolicy] == {aws} managed policy: AmazonEKSLoadBalancingPolicy :info_titleabbrev: AmazonEKSLoadBalancingPolicy @@ -187,9 +228,9 @@ The key permissions granted by this policy are: The policy also includes several condition checks to ensure that the permissions are scoped to the specific EKS cluster being managed, using the `eks:eks-cluster-name` tag. -To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEKSLoadBalancingPolicy.html#AmazonEKSLoadBalancingPolicy-json[AmazonEKSLoadBalancingPolicy,type="documentation"] in the {aws} Managed Policy Reference Guide. +To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEKSLoadBalancingPolicy.html#AmazonEKSLoadBalancingPolicy-json[AmazonEKSLoadBalancingPolicy,type="documentation"] in the {aws} Managed Policy Reference Guide. -[[security-iam-awsmanpol-amazoneksservicepolicy,security-iam-awsmanpol-amazoneksservicepolicy.title]] +[#security-iam-awsmanpol-amazoneksservicepolicy] == {aws} managed policy: AmazonEKSServicePolicy :info_titleabbrev: AmazonEKSServicePolicy @@ -201,15 +242,15 @@ This policy allows Amazon EKS to create and manage the necessary resources to op This policy includes the following permissions that allow Amazon EKS to complete the following tasks. -* *`eks`* – Update the [.noloc]`Kubernetes` version of your cluster after you initiate an update. This permission isn't used by Amazon EKS but remains in the policy for backwards compatibility. -* *`ec2`* – Work with Elastic Network Interfaces and other network resources and tags. This is required by Amazon EKS to configure networking that facilitates communication between nodes and the [.noloc]`Kubernetes` control plane. Read information about security groups. Update tags on security groups. -* *`route53`* – Associate a VPC with a hosted zone. This is required by Amazon EKS to enable private endpoint networking for your [.noloc]`Kubernetes` cluster API server. -* *`logs`* – Log events. This is required so that Amazon EKS can ship [.noloc]`Kubernetes` control plane logs to CloudWatch. -* *`iam`* – Create a service-linked role. This is required so that Amazon EKS can create the <> service-linked role on your behalf. +* *`eks`* – Update the Kubernetes version of your cluster after you initiate an update. This permission isn't used by Amazon EKS but remains in the policy for backwards compatibility. +* *`ec2`* – Work with Elastic Network Interfaces and other network resources and tags. This is required by Amazon EKS to configure networking that facilitates communication between nodes and the Kubernetes control plane. Read information about security groups. Update tags on security groups. +* *`route53`* – Associate a VPC with a hosted zone. This is required by Amazon EKS to enable private endpoint networking for your Kubernetes cluster API server. +* *`logs`* – Log events. This is required so that Amazon EKS can ship Kubernetes control plane logs to CloudWatch. +* *`iam`* – Create a service-linked role. This is required so that Amazon EKS can create the <> service-linked role on your behalf. -To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEKSServicePolicy.html#AmazonEKSServicePolicy-json[AmazonEKSServicePolicy,type="documentation"] in the {aws} Managed Policy Reference Guide. +To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEKSServicePolicy.html#AmazonEKSServicePolicy-json[AmazonEKSServicePolicy,type="documentation"] in the {aws} Managed Policy Reference Guide. -[[security-iam-awsmanpol-amazoneksservicerolepolicy,security-iam-awsmanpol-amazoneksservicerolepolicy.title]] +[#security-iam-awsmanpol-amazoneksservicerolepolicy] == {aws} managed policy: AmazonEKSServiceRolePolicy :info_titleabbrev: AmazonEKSServiceRolePolicy @@ -221,11 +262,12 @@ This policy allows the service-linked role to call {aws} services on your behalf This policy includes the following permissions that allow Amazon EKS to complete the following tasks. -* *`ec2`* – Create and describe Elastic Network Interfaces and Amazon EC2 instances, the cluster security group, and VPC that are required to create a cluster. For more information, see <>. Read information about security groups. Update tags on security groups. +* *`ec2`* – Create and describe Elastic Network Interfaces and Amazon EC2 instances, the cluster security group, and VPC that are required to create a cluster. For more information, see <>. Read information about security groups. Update tags on security groups. Read information about On-Demand Capacity Reservations. Read VPC configuration including route tables and network ACLs to detect configuration issues as part of cluster insights. +* *`ec2` Auto Mode* – Terminate EC2 instances created by EKS Auto Mode. For more information, see <>. * *`iam`* – List all of the managed policies that attached to an IAM role. This is required so that Amazon EKS can list and validate all managed policies and permissions required to create a cluster. -* *Associate a VPC with a hosted zone* – This is required by Amazon EKS to enable private endpoint networking for your [.noloc]`Kubernetes` cluster API server. -* *Log event* – This is required so that Amazon EKS can ship [.noloc]`Kubernetes` control plane logs to CloudWatch. -* *Put metric* – This is required so that Amazon EKS can ship [.noloc]`Kubernetes` control plane logs to CloudWatch. +* *Associate a VPC with a hosted zone* – This is required by Amazon EKS to enable private endpoint networking for your Kubernetes cluster API server. +* *Log event* – This is required so that Amazon EKS can ship Kubernetes control plane logs to CloudWatch. +* *Put metric* – This is required so that Amazon EKS can ship Kubernetes control plane logs to CloudWatch. * *`eks`* - Manage cluster access entries and policies, allowing fine-grained control over who can access EKS resources and what actions they can perform. This includes associating standard access policies for compute, networking, load balancing, and storage operations. * *`elasticloadbalancing`* - Create, manage, and delete load balancers and their components (listeners, target groups, certificates) that are associated with EKS clusters. View load balancer attributes and health status. * *`events`* - Create and manage EventBridge rules for monitoring EC2 and {aws} Health events related to EKS clusters, enabling automated responses to infrastructure changes and health alerts. @@ -233,9 +275,9 @@ This policy includes the following permissions that allow Amazon EKS to complete * *`pricing`* & *`shield`* - Access {aws} pricing information and Shield protection status, enabling cost management and advanced security features for EKS resources. * **Resource cleanup** - Safely delete EKS-tagged resources including volumes, snapshots, launch templates, and network interfaces during cluster cleanup operations. -To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEKSServiceRolePolicy.html#AmazonEKSServiceRolePolicy-json[AmazonEKSServiceRolePolicy,type="documentation"] in the {aws} Managed Policy Reference Guide. +To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEKSServiceRolePolicy.html#AmazonEKSServiceRolePolicy-json[AmazonEKSServiceRolePolicy,type="documentation"] in the {aws} Managed Policy Reference Guide. -[[security-iam-awsmanpol-amazoneksvpcresourcecontroller,security-iam-awsmanpol-amazoneksvpcresourcecontroller.title]] +[#security-iam-awsmanpol-amazoneksvpcresourcecontroller] == {aws} managed policy: AmazonEKSVPCResourceController :info_titleabbrev: AmazonEKSVPCResourceController @@ -247,11 +289,11 @@ This policy grants the cluster role permissions to manage Elastic Network Interf This policy includes the following permissions that allow Amazon EKS to complete the following tasks: -* *`ec2`* – Manage Elastic Network Interfaces and IP addresses to support [.noloc]`Pod` security groups and [.noloc]`Windows` nodes. +* *`ec2`* – Manage Elastic Network Interfaces and IP addresses to support Pod security groups and Windows nodes. -To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEKSVPCResourceController.html#AmazonEKSVPCResourceController-json[AmazonEKSVPCResourceController,type="documentation"] in the {aws} Managed Policy Reference Guide. +To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEKSVPCResourceController.html#AmazonEKSVPCResourceController-json[AmazonEKSVPCResourceController,type="documentation"] in the {aws} Managed Policy Reference Guide. -[[security-iam-awsmanpol-amazoneksworkernodepolicy,security-iam-awsmanpol-amazoneksworkernodepolicy.title]] +[#security-iam-awsmanpol-amazoneksworkernodepolicy] == {aws} managed policy: AmazonEKSWorkerNodePolicy :info_titleabbrev: AmazonEKSWorkerNodePolicy @@ -263,14 +305,14 @@ This policy grants Amazon EKS Amazon EC2 nodes permissions to connect to Amazon This policy includes the following permissions that allow Amazon EKS to complete the following tasks: -* *`ec2`* – Read instance volume and network information. This is required so that [.noloc]`Kubernetes` nodes can describe information about Amazon EC2 resources that are required for the node to join the Amazon EKS cluster. +* *`ec2`* – Read instance volume and network information. This is required so that Kubernetes nodes can describe information about Amazon EC2 resources that are required for the node to join the Amazon EKS cluster. * *`eks`* – Optionally describe the cluster as part of node bootstrapping. * *`eks-auth:AssumeRoleForPodIdentity`* – Allow retrieving credentials for EKS workloads on the node. This is required for EKS Pod Identity to function properly. -To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEKSWorkerNodePolicy.html#AmazonEKSWorkerNodePolicy-json[AmazonEKSWorkerNodePolicy,type="documentation"] in the {aws} Managed Policy Reference Guide. +To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEKSWorkerNodePolicy.html#AmazonEKSWorkerNodePolicy-json[AmazonEKSWorkerNodePolicy,type="documentation"] in the {aws} Managed Policy Reference Guide. -[[security-iam-awsmanpol-AmazonEKSWorkerNodeMinimalPolicy,security-iam-awsmanpol-AmazonEKSWorkerNodeMinimalPolicy.title]] +[#security-iam-awsmanpol-AmazonEKSWorkerNodeMinimalPolicy] == {aws} managed policy: AmazonEKSWorkerNodeMinimalPolicy :info_titleabbrev: AmazonEKSWorkerNodeMinimalPolicy @@ -286,7 +328,7 @@ This policy includes the following permissions that allow Amazon EKS to complete To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEKSWorkerNodeMinimalPolicy.html#AmazonEKSWorkerNodeMinimalPolicy-json["AmazonEKSWorkerNodePolicy", type="documentation"] in the {aws} Managed Policy Reference Guide. -[[security-iam-awsmanpol-awsserviceroleforamazoneksnodegroup,security-iam-awsmanpol-awsserviceroleforamazoneksnodegroup.title]] +[#security-iam-awsmanpol-awsserviceroleforamazoneksnodegroup] == {aws} managed policy: AWSServiceRoleForAmazonEKSNodegroup :info_titleabbrev: AWSServiceRoleForAmazonEKSNodegroup @@ -300,31 +342,47 @@ This policy includes the following permissions that allow Amazon EKS to complete * *`ec2`* – Work with security groups, tags, capacity reservations, and launch templates. This is required for Amazon EKS managed node groups to enable remote access configuration and to describe capacity reservations that can be used in managed node groups. Additionally, Amazon EKS managed node groups create a launch template on your behalf. This is to configure the Amazon EC2 Auto Scaling group that backs each managed node group. * *`iam`* – Create a service-linked role and pass a role. This is required by Amazon EKS managed node groups to manage instance profiles for the role being passed when creating a managed node group. This instance profile is used by Amazon EC2 instances launched as part of a managed node group. Amazon EKS needs to create service-linked roles for other services such as Amazon EC2 Auto Scaling groups. These permissions are used in the creation of a managed node group. -* *`autoscaling`* – Work with security Auto Scaling groups. This is required by Amazon EKS managed node groups to manage the Amazon EC2 Auto Scaling group that backs each managed node group. It's also used to support functionality such as evicting [.noloc]`Pods` when nodes are terminated or recycled during node group updates. +* *`autoscaling`* – Work with security Auto Scaling groups. This is required by Amazon EKS managed node groups to manage the Amazon EC2 Auto Scaling group that backs each managed node group. It's also used to support functionality such as evicting Pods when nodes are terminated or recycled during node group updates. + +To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AWSServiceRoleForAmazonEKSNodegroup.html#AWSServiceRoleForAmazonEKSNodegroup-json[AWSServiceRoleForAmazonEKSNodegroup,type="documentation"] in the {aws} Managed Policy Reference Guide. + +[#security-iam-awsmanpol-AmazonEKSDashboardServiceRolePolicy] +== {aws} managed policy: AmazonEKSDashboardServiceRolePolicy +:info_titleabbrev: AmazonEKSDashboardServiceRolePolicy -To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AWSServiceRoleForAmazonEKSNodegroup.html#AWSServiceRoleForAmazonEKSNodegroup-json[AWSServiceRoleForAmazonEKSNodegroup,type="documentation"] in the {aws} Managed Policy Reference Guide. +You can't attach `AmazonEKSDashboardServiceRolePolicy` to your IAM entities. This policy is attached to a service-linked role that allows Amazon EKS to perform actions on your behalf. For more information, see <>. -[[security-iam-awsmanpol-amazonebscsidriverservicerolepolicy,security-iam-awsmanpol-amazonebscsidriverservicerolepolicy.title]] +This policy grants the `AWSServiceRoleForAmazonEKSDashboard` role permissions that allow it to create and manage Amazon EC2 node groups in your account. + +*Permissions details* + +This policy includes the following permissions that allow access to complete these tasks: + +* *`organizations`* – View information about your {aws} Organizations structure and accounts. This includes permissions to list accounts in your organization, view organizational units and roots, list delegated administrators, view services that have access to your organization, and retrieve detailed information about your organization and accounts. + +To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEKSDashboardServiceRolePolicy.html#AmazonEKSDashboardServiceRolePolicy-json[AmazonEKSDashboardServiceRolePolicy,type="documentation"] in the {aws} Managed Policy Reference Guide. + +[#security-iam-awsmanpol-amazonebscsidriverservicerolepolicy] == {aws} managed policy: AmazonEBSCSIDriverPolicy :info_titleabbrev: AmazonEBSCSIDriverPolicy The `AmazonEBSCSIDriverPolicy` policy allows the Amazon EBS Container Storage Interface (CSI) driver to create, modify, attach, detach, and delete volumes on your behalf. This includes modifying tags on existing volumes and enabling Fast Snapshot Restore (FSR) on EBS volumes. It also grants the EBS CSI driver permissions to create, restore, and delete snapshots, and to list your instances, volumes, and snapshots. -To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEBSCSIDriverPolicy.html#AmazonEBSCSIDriverPolicy-json[AmazonEBSCSIDriverServiceRolePolicy,type="documentation"] in the {aws} Managed Policy Reference Guide. +To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEBSCSIDriverPolicy.html#AmazonEBSCSIDriverPolicy-json[AmazonEBSCSIDriverServiceRolePolicy,type="documentation"] in the {aws} Managed Policy Reference Guide. -[[security-iam-awsmanpol-amazonefscsidriverservicerolepolicy,security-iam-awsmanpol-amazonefscsidriverservicerolepolicy.title]] +[#security-iam-awsmanpol-amazonefscsidriverservicerolepolicy] == {aws} managed policy: AmazonEFSCSIDriverPolicy :info_titleabbrev: AmazonEFSCSIDriverPolicy The `AmazonEFSCSIDriverPolicy` policy allows the Amazon EFS Container Storage Interface (CSI) to create and delete access points on your behalf. It also grants the Amazon EFS CSI driver permissions to list your access points file systems, mount targets, and Amazon EC2 availability zones. -To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEFSCSIDriverPolicy.html#AmazonEFSCSIDriverPolicy-json[AmazonEFSCSIDriverServiceRolePolicy,type="documentation"] in the {aws} Managed Policy Reference Guide. +To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEFSCSIDriverPolicy.html#AmazonEFSCSIDriverPolicy-json[AmazonEFSCSIDriverServiceRolePolicy,type="documentation"] in the {aws} Managed Policy Reference Guide. -[[security-iam-awsmanpol-amazonekslocaloutpostclusterpolicy,security-iam-awsmanpol-amazonekslocaloutpostclusterpolicy.title]] +[#security-iam-awsmanpol-amazonekslocaloutpostclusterpolicy] == {aws} managed policy: AmazonEKSLocalOutpostClusterPolicy :info_titleabbrev: AmazonEKSLocalOutpostClusterPolicy -You can attach this policy to IAM entities. Before creating a local cluster, you must attach this policy to your <>. [.noloc]`Kubernetes` clusters that are managed by Amazon EKS make calls to other {aws} services on your behalf. They do this to manage the resources that you use with the service. +You can attach this policy to IAM entities. Before creating a local cluster, you must attach this policy to your <>. Kubernetes clusters that are managed by Amazon EKS make calls to other {aws} services on your behalf. They do this to manage the resources that you use with the service. The `AmazonEKSLocalOutpostClusterPolicy` includes the following permissions: @@ -332,11 +390,11 @@ The `AmazonEKSLocalOutpostClusterPolicy` includes the following permissions: * *`ssm`* – Allows Amazon EC2 Systems Manager connection to the control plane instance, which is used by Amazon EKS to communicate and manage the local cluster in your account. * *`logs`* – Allows instances to push logs to Amazon CloudWatch. * *`secretsmanager`* – Allows instances to get and delete bootstrap data for the control plane instances securely from {aws} Secrets Manager. -* *`ecr`* – Allows [.noloc]`Pods` and containers that are running on the control plane instances to pull container images that are stored in Amazon Elastic Container Registry. +* *`ecr`* – Allows Pods and containers that are running on the control plane instances to pull container images that are stored in Amazon Elastic Container Registry. -To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEKSLocalOutpostClusterPolicy.html#AmazonEKSLocalOutpostClusterPolicy-json[AmazonEKSLocalOutpostClusterPolicy,type="documentation"] in the {aws} Managed Policy Reference Guide. +To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEKSLocalOutpostClusterPolicy.html#AmazonEKSLocalOutpostClusterPolicy-json[AmazonEKSLocalOutpostClusterPolicy,type="documentation"] in the {aws} Managed Policy Reference Guide. -[[security-iam-awsmanpol-amazonekslocaloutpostservicerolepolicy,security-iam-awsmanpol-amazonekslocaloutpostservicerolepolicy.title]] +[#security-iam-awsmanpol-amazonekslocaloutpostservicerolepolicy] == {aws} managed policy: AmazonEKSLocalOutpostServiceRolePolicy :info_titleabbrev: AmazonEKSLocalOutpostServiceRolePolicy @@ -345,32 +403,84 @@ You can't attach this policy to your IAM entities. When you create a cluster usi The `AmazonEKSLocalOutpostServiceRolePolicy` includes the following permissions: * *`ec2`* – Allows Amazon EKS to work with security, network, and other resources to successfully launch and manage control plane instances in your account. -* *`ssm`* – Allows Amazon EC2 Systems Manager connection to the control plane instances, which is used by Amazon EKS to communicate and manage the local cluster in your account. +* *`ssm`, `ssmmessages`* – Allows Amazon EC2 Systems Manager connection to the control plane instances, which is used by Amazon EKS to communicate and manage the local cluster in your account. * *`iam`* – Allows Amazon EKS to manage the instance profile associated with the control plane instances. * *`secretsmanager`* - Allows Amazon EKS to put bootstrap data for the control plane instances into {aws} Secrets Manager so it can be securely referenced during instance bootstrapping. * *`outposts`* – Allows Amazon EKS to get Outpost information from your account to successfully launch a local cluster in an Outpost. -To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEKSLocalOutpostServiceRolePolicy.html#AmazonEKSLocalOutpostServiceRolePolicy-json[AmazonEKSLocalOutpostServiceRolePolicy,type="documentation"] in the {aws} Managed Policy Reference Guide. +To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEKSLocalOutpostServiceRolePolicy.html#AmazonEKSLocalOutpostServiceRolePolicy-json[AmazonEKSLocalOutpostServiceRolePolicy,type="documentation"] in the {aws} Managed Policy Reference Guide. -[[security-iam-awsmanpol-updates,security-iam-awsmanpol-updates.title]] +[#security-iam-awsmanpol-updates] == Amazon EKS updates to {aws} managed policies :info_titleabbrev: Policy updates -View details about updates to {aws} managed policies for Amazon EKS since this service began tracking these changes. For automatic alerts about changes to this page, subscribe to the RSS feed on the Amazon EKS Document history page. +View details about updates to {aws} managed policies for Amazon EKS since this service began tracking these changes. -[cols="1,1,1", options="header"] +To receive notifications of all source file changes to this specific documentation page, you can subscribe to the following URL with an RSS reader: + +[source,none] +---- +https://github.com/awsdocs/amazon-eks-user-guide/commits/mainline/latest/ug/security/iam-reference/security-iam-awsmanpol.adoc.atom +---- + +[%header,cols="3"] |=== |Change |Description |Date +|Added permission to <>. +|Added `ec2:DescribeRouteTables` and `ec2:DescribeNetworkAcls` permissions to `AmazonEKSServiceRolePolicy`. This allows Amazon EKS to detect configuration issues with VPC route tables and network ACLs for hybrid nodes as part of cluster insights. +|Oct 22, 2025 + +|Added permission to <> +|Added `ssmmessages:OpenDataChannel` permission to `AmazonEKSConnectorServiceRolePolicy` +|October 15, 2025 + +|Added permission to <> +|This role can attach new access policy `AmazonEKSEventPolicy`. Restricted permissions for `ec2:DeleteLaunchTemplate` and `ec2:TerminateInstances`. +|August 26, 2025 + +|Added permission to <> +|Added `ssmmessages:OpenDataChannel` permission to `AmazonEKSLocalOutpostServiceRolePolicy`. +|June 26, 2025 + +| Added permission to <>. +| Updated resource permissions for the `ec2:RunInstances` and `ec2:CreateFleet` actions to include capacity reservations `{arn-aws}ec2:*:*:capacity-reservation/*`. This allows Amazon EKS Auto Mode to launch instances by using the EC2 On-Demand Capacity Reservations in your account. Added `iam:CreateServiceLinkedRole` to allow Amazon EKS Auto Mode to create the EC2 Spot service-linked role `AWSServiceRoleForEC2Spot` on your behalf. +| June 20, 2025 + +| Added permission to <>. +| Added `ec2:DescribeCapacityReservations` permission to allow Amazon EKS Auto Mode to launch instances by using the EC2 On-Demand Capacity Reservations in your account. +| June 20, 2025 + + +|Introduced <>. +|Introduced new `AmazonEKSDashboardConsoleReadOnly` policy. +|June 19, 2025 + +|Introduced <>. +|Introduced new `AmazonEKSDashboardServiceRolePolicy` policy. +|May 21, 2025 + +|Added permissions to <>. +|Added `ec2:DeleteNetworkInterfaces` permission to allow Amazon EKS to delete elastic network interfaces that are left behind if the VPC CNI quits unexpectedly. +|April 16, 2025 + +| Added permission to <>. +| Added `ec2:RevokeSecurityGroupEgress` and `ec2:AuthorizeSecurityGroupEgress` permissions to allow EKS AI/ML customers to add Security Group Egress rules to the default EKS Cluster SG that are compatible with EFA, as part of the EKS 1.33 version release. +| April 14, 2025 + +| Added permissions to <>. +| Added permission to terminate EC2 instances created by EKS Auto Mode. +| February 28, 2025 + |Added permissions to <>. |Added a new statement authorizing the EBS CSI Driver to restore all snapshots. This was previously allowed by the existing policy but a new explicit statement is required due to a change in the handling of IAM for `CreateVolume`. -Added the ability for the EBS CSI Driver to modify tags on existing volumes. The EBS CSI Driver can modify tags of existing volumes via a parameters in Kubernetes `VolumeAttributesClass`es. +Added the ability for the EBS CSI Driver to modify tags on existing volumes. The EBS CSI Driver can modify tags of existing volumes via a parameters in Kubernetes VolumeAttributesClasses. -Added the ability for the EBS CSI Driver to enable Fast Snapshot Restore (FSR) on EBS volumes. The EBS CSI Driver can enable FSR on new volumes via parameters in Kubernetes `StorageClass`es. +Added the ability for the EBS CSI Driver to enable Fast Snapshot Restore (FSR) on EBS volumes. The EBS CSI Driver can enable FSR on new volumes via parameters in Kubernetes storage classes. |January 13, 2025 |Added permissions to <>. @@ -425,7 +535,7 @@ Added the ability for the EBS CSI Driver to enable Fast Snapshot Restore (FSR) o |Added `ec2:GetSecurityGroupsForVpc` and associated tag permissions to allow EKS to read security group information and update related tags. |October 10, 2024 -|Introduced xref:security-iam-awsmanpol-AmazonEKSWorkerNodeMinimalPolicy[AmazonEKSWorkerNodeMinimalPolicy]. +|Introduced <>. |{aws} introduced the `AmazonEKSWorkerNodeMinimalPolicy`. |October 3, 2024 @@ -438,7 +548,7 @@ Added the ability for the EBS CSI Driver to enable Fast Snapshot Restore (FSR) o |June 27, 2024 |<> – Update to an existing policy -|Amazon EKS added new `ec2:DescribeSubnets` permissions to allow the [.noloc]`Amazon VPC CNI plugin for Kubernetes` to see the amount of free IP addresses in your Amazon VPC subnets. The VPC CNI can use the free IP addresses in each subnet to pick the subnets with the most free IP addresses to use when creating an elastic network interface. +|Amazon EKS added new `ec2:DescribeSubnets` permissions to allow the Amazon VPC CNI plugin for Kubernetes to see the amount of free IP addresses in your Amazon VPC subnets. The VPC CNI can use the free IP addresses in each subnet to pick the subnets with the most free IP addresses to use when creating an elastic network interface. |March 4, 2024 |<> – Update to an existing policy diff --git a/latest/ug/security/iam-reference/security-iam-id-based-policy-examples.adoc b/latest/ug/security/iam-reference/security-iam-id-based-policy-examples.adoc new file mode 100644 index 000000000..d61f62bdf --- /dev/null +++ b/latest/ug/security/iam-reference/security-iam-id-based-policy-examples.adoc @@ -0,0 +1,107 @@ +include::../../attributes.txt[] + +[.topic] +[#security-iam-id-based-policy-examples] += Amazon EKS identity-based policy examples +:info_titleabbrev: Identity-based policies + +By default, IAM users and roles don't have permission to create or modify Amazon EKS resources. They also can't perform tasks using the {aws-management-console}, {aws} CLI, or {aws} API. An IAM administrator must create IAM policies that grant users and roles permission to perform specific API operations on the specified resources they need. The administrator must then attach those policies to the IAM users or groups that require those permissions. + +To learn how to create an IAM identity-based policy using these example JSON policy documents, see link:IAM/latest/UserGuide/access_policies_create.html#access_policies_create-json-editor[Creating policies on the JSON tab,type="documentation"] in the _IAM User Guide_. + +When you create an Amazon EKS cluster, the link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] that creates the cluster is automatically granted `system:masters` permissions in the cluster's role-based access control (RBAC) configuration in the Amazon EKS control plane. This principal doesn't appear in any visible configuration, so make sure to keep track of which principal originally created the cluster. To grant additional IAM principals the ability to interact with your cluster, edit the `aws-auth ConfigMap` within Kubernetes and create a Kubernetes `rolebinding` or `clusterrolebinding` with the name of a `group` that you specify in the `aws-auth ConfigMap`. + +For more information about working with the ConfigMap, see <>. + +[.topiclist] +[[Topic List]] + +[#security-iam-service-with-iam-policy-best-practices] +== Policy best practices + +Identity-based policies determine whether someone can create, access, or delete Amazon EKS resources in your account. These actions can incur costs for your {aws} account. When you create or edit identity-based policies, follow these guidelines and recommendations: + + + +* *Get started with {aws} managed policies and move toward least-privilege permissions* – To get started granting permissions to your users and workloads, use the _{aws} managed policies_ that grant permissions for many common use cases. They are available in your {aws} account. We recommend that you reduce permissions further by defining {aws} customer managed policies that are specific to your use cases. For more information, see link:IAM/latest/UserGuide/access_policies_managed-vs-inline.html#aws-managed-policies[{aws} managed policies,type="documentation"] or link:IAM/latest/UserGuide/access_policies_job-functions.html[{aws} managed policies for job functions,type="documentation"] in the _IAM User Guide_. +* *Apply least-privilege permissions* – When you set permissions with IAM policies, grant only the permissions required to perform a task. You do this by defining the actions that can be taken on specific resources under specific conditions, also known as _least-privilege permissions_. For more information about using IAM to apply permissions, see link:IAM/latest/UserGuide/access_policies.html[Policies and permissions in IAM,type="documentation"] in the _IAM User Guide_. +* *Use conditions in IAM policies to further restrict access* – You can add a condition to your policies to limit access to actions and resources. For example, you can write a policy condition to specify that all requests must be sent using SSL. You can also use conditions to grant access to service actions if they are used through a specific {aws} service, such as {aws} CloudFormation. For more information, see link:IAM/latest/UserGuide/reference_policies_elements_condition.html[IAM JSON policy elements: Condition,type="documentation"] in the _IAM User Guide_. +* *Use IAM Access Analyzer to validate your IAM policies to ensure secure and functional permissions* – IAM Access Analyzer validates new and existing policies so that the policies adhere to the IAM policy language (JSON) and IAM best practices. IAM Access Analyzer provides more than 100 policy checks and actionable recommendations to help you author secure and functional policies. For more information, see link:IAM/latest/UserGuide/access-analyzer-policy-validation.html[IAM Access Analyzer policy validation,type="documentation"] in the _IAM User Guide_. +* *Require multi-factor authentication (MFA)* – If you have a scenario that requires IAM users or a root user in your {aws} account, turn on MFA for additional security. To require MFA when API operations are called, add MFA conditions to your policies. For more information, see link:IAM/latest/UserGuide/id_credentials_mfa_configure-api-require.html[Configuring MFA-protected API access,type="documentation"] in the _IAM User Guide_. + +For more information about best practices in IAM, see link:IAM/latest/UserGuide/best-practices.html[Security best practices in IAM,type="documentation"] in the _IAM User Guide_. + +[#security-iam-id-based-policy-examples-console] +== Using the Amazon EKS console + +To access the Amazon EKS console, an link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"], must have a minimum set of permissions. These permissions allow the principal to list and view details about the Amazon EKS resources in your {aws} account. If you create an identity-based policy that is more restrictive than the minimum required permissions, the console won't function as intended for principals with that policy attached to them. + +To ensure that your IAM principals can still use the Amazon EKS console, create a policy with your own unique name, such as `AmazonEKSAdminPolicy`. Attach the policy to the principals. For more information, see link:IAM/latest/UserGuide/access_policies_manage-attach-detach.html[Adding and removing IAM identity permissions,type="documentation"] in the _IAM User Guide_. + +[IMPORTANT] +==== + +The following example policy allows a principal to view information on the *Configuration* tab in the console. To view information on the *Overview* and *Resources* tabs in the {aws-management-console}, the principal also needs Kubernetes permissions. For more information, see <>. + +==== + +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:c90cfc65-d638-4a20-a06a-47e7d2f69e7e[] +---- + +You don't need to allow minimum console permissions for principals that are making calls only to the {aws} CLI or the {aws} API. Instead, allow access to only the actions that match the API operation that you're trying to perform. + +[#security-iam-id-based-policy-examples-view-own-permissions] +== Allow IAM users to view their own permissions + +This example shows how you might create a policy that allows IAM users to view the inline and managed policies that are attached to their user identity. This policy includes permissions to complete this action on the console or programmatically using the {aws} CLI or {aws} API. + +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:b1f909cd-2c64-4c9b-b5c5-186f78c6aa5a[] +---- + + +[#policy-create-cluster] +== Create a Kubernetes cluster on the {aws} Cloud + +This example policy includes the minimum permissions required to create an Amazon EKS cluster named [.replaceable]`my-cluster` in the [.replaceable]`us-west-2` {aws} Region. You can replace the {aws} Region with the {aws} Region that you want to create a cluster in. If you see a warning that says *The actions in your policy do not support resource-level permissions and require you to choose `All resources`* in the {aws-management-console}, it can be safely ignored. If your account already has the [.replaceable]`AWSServiceRoleForAmazonEKS` role, you can remove the `iam:CreateServiceLinkedRole` action from the policy. If you've ever created an Amazon EKS cluster in your account then this role already exists, unless you deleted it. + +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:4e925c83-85eb-4a95-8b72-1631224902bb[] +---- + + +[#policy-create-local-cluster] +== Create a local Kubernetes cluster on an Outpost + +This example policy includes the minimum permissions required to create an Amazon EKS local cluster named [.replaceable]`my-cluster` on an Outpost in the [.replaceable]`us-west-2` {aws} Region. You can replace the {aws} Region with the {aws} Region that you want to create a cluster in. If you see a warning that says *The actions in your policy do not support resource-level permissions and require you to choose `All resources`* in the {aws-management-console}, it can be safely ignored. If your account already has the `AWSServiceRoleForAmazonEKSLocalOutpost` role, you can remove the `iam:CreateServiceLinkedRole` action from the policy. If you've ever created an Amazon EKS local cluster on an Outpost in your account then this role already exists, unless you deleted it. + +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:b19597bc-301e-4c81-94b5-b8fe30558df0[] +---- + + +[#policy-example1] +== Update a Kubernetes cluster + +This example policy includes the minimum permission required to update a cluster named [.replaceable]`my-cluster` in the us-west-2 {aws} Region. + +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:8287e107-08f4-473b-99f4-5d1c39273a86[] +---- + + +[#policy-example2] +== List or describe all clusters + +This example policy includes the minimum permissions required to list and describe all clusters in your account. An link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] must be able to list and describe clusters to use the `update-kubeconfig` {aws} CLI command. + +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:dbfd52f1-1fc5-4ab5-a4ac-7d2c5fcfa7c2[] +---- \ No newline at end of file diff --git a/latest/ug/security/iam-reference/security-iam-service-with-iam.adoc b/latest/ug/security/iam-reference/security-iam-service-with-iam.adoc new file mode 100644 index 000000000..3a212c9ef --- /dev/null +++ b/latest/ug/security/iam-reference/security-iam-service-with-iam.adoc @@ -0,0 +1,152 @@ +include::../../attributes.txt[] + +[.topic] +[#security-iam-service-with-iam] += How Amazon EKS works with IAM +:info_titleabbrev: Amazon EKS and IAM + +Before you use IAM to manage access to Amazon EKS, you should understand what IAM features are available to use with Amazon EKS. To get a high-level view of how Amazon EKS and other {aws} services work with IAM, see link:IAM/latest/UserGuide/reference_aws-services-that-work-with-iam.html[{aws} services that work with IAM,type="documentation"] in the _IAM User Guide_. + +[.topiclist] +[[Topic List]] + +[#security-iam-service-with-iam-id-based-policies] +== Amazon EKS identity-based policies + +With IAM identity-based policies, you can specify allowed or denied actions and resources as well as the conditions under which actions are allowed or denied. Amazon EKS supports specific actions, resources, and condition keys. To learn about all of the elements that you use in a JSON policy, see link:IAM/latest/UserGuide/reference_policies_elements.html[IAM JSON policy elements reference,type="documentation"] in the _IAM User Guide_. + +[#security-iam-service-with-iam-id-based-policies-actions] +=== Actions + +Administrators can use {aws} JSON policies to specify who has access to what. That is, which *principal* can perform *actions* on what *resources*, and under what *conditions*. + +The `Action` element of a JSON policy describes the actions that you can use to allow or deny access in a policy. Policy actions usually have the same name as the associated {aws} API operation. There are some exceptions, such as _permission-only actions_ that don't have a matching API operation. There are also some operations that require multiple actions in a policy. These additional actions are called _dependent actions_. + +Include actions in a policy to grant permissions to perform the associated operation. + +Policy actions in Amazon EKS use the following prefix before the action: `eks:`. For example, to grant someone permission to get descriptive information about an Amazon EKS cluster, you include the `DescribeCluster` action in their policy. Policy statements must include either an `Action` or `NotAction` element. + +To specify multiple actions in a single statement, separate them with commas as follows: + +[source,json,subs="verbatim,attributes"] +---- +"Action": ["eks:action1", "eks:action2"] +---- + +You can specify multiple actions using wildcards (*). For example, to specify all actions that begin with the word `Describe`, include the following action: + +[source,json,subs="verbatim,attributes"] +---- +"Action": "eks:Describe*" +---- + + +To see a list of Amazon EKS actions, see link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html#amazonelastickubernetesservice-actions-as-permissions[Actions defined by Amazon Elastic Kubernetes Service,type="documentation"] in the _Service Authorization Reference_. + +[#security-iam-service-with-iam-id-based-policies-resources] +=== Resources + +Administrators can use {aws} JSON policies to specify who has access to what. That is, which *principal* can perform *actions* on what *resources*, and under what *conditions*. + +The `Resource` JSON policy element specifies the object or objects to which the action applies. Statements must include either a `Resource` or a `NotResource` element. As a best practice, specify a resource using its link:IAM/latest/UserGuide/reference-arns.html[Amazon Resource Name (ARN),type="documentation"]. You can do this for actions that support a specific resource type, known as _resource-level permissions_. + +For actions that don't support resource-level permissions, such as listing operations, use a wildcard (*) to indicate that the statement applies to all resources. + +[source] +---- +"Resource": "*" +---- + +The Amazon EKS cluster resource has the following ARN. + +[source,none,subs="verbatim,attributes"] +---- +{arn-aws}eks:region-code:account-id:cluster/cluster-name +---- + +For more information about the format of ARNs, see link:general/latest/gr/aws-arns-and-namespaces.html[Amazon resource names (ARNs) and {aws} service namespaces,type="documentation"]. + +For example, to specify the cluster with the name [.replaceable]`my-cluster` in your statement, use the following ARN: + +[source,json,subs="verbatim,attributes"] +---- +"Resource": "{arn-aws}eks:region-code:111122223333:cluster/my-cluster" +---- + +To specify all clusters that belong to a specific account and {aws} Region, use the wildcard (*): + +[source,json,subs="verbatim,attributes"] +---- +"Resource": "{arn-aws}eks:region-code:111122223333:cluster/*" +---- + +Some Amazon EKS actions, such as those for creating resources, can't be performed on a specific resource. In those cases, you must use the wildcard (*). + +[source,json,subs="verbatim,attributes"] +---- +"Resource": "*" +---- + +To see a list of Amazon EKS resource types and their ARNs, see link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html#amazonelastickubernetesservice-resources-for-iam-policies[Resources defined by Amazon Elastic Kubernetes Service,type="documentation"] in the _Service Authorization Reference_. To learn with which actions you can specify the ARN of each resource, see link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html#amazonelastickubernetesservice-actions-as-permissions[Actions defined by Amazon Elastic Kubernetes Service,type="documentation"]. + +[#security-iam-service-with-iam-id-based-policies-conditionkeys] +=== Condition keys + +Amazon EKS defines its own set of condition keys and also supports using some global condition keys. To see all {aws} global condition keys, see link:IAM/latest/UserGuide/reference_policies_condition-keys.html[{aws} Global Condition Context Keys,type="documentation"] in the _IAM User Guide_. + +You can set condition keys when associating an OpenID Connect provider to your cluster. For more information, see <>. + +All Amazon EC2 actions support the `aws:RequestedRegion` and `ec2:Region` condition keys. For more information, see link:AWSEC2/latest/UserGuide/ExamplePolicies_EC2.html#iam-example-region[Example: Restricting Access to a Specific {aws} Region,type="documentation"]. + +For a list of Amazon EKS condition keys, see link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html#amazonelastickubernetesservice-policy-keys[Conditions defined by Amazon Elastic Kubernetes Service,type="documentation"] in the _Service Authorization Reference_. To learn which actions and resources you can use a condition key with, see link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html#amazonelastickubernetesservice-actions-as-permissions[Actions defined by Amazon Elastic Kubernetes Service,type="documentation"]. + +[#security-iam-service-with-iam-id-based-policies-examples] +=== Examples + + +To view examples of Amazon EKS identity-based policies, see <>. + +When you create an Amazon EKS cluster, the link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] that creates the cluster is automatically granted `system:masters` permissions in the cluster's role-based access control (RBAC) configuration in the Amazon EKS control plane. This principal doesn't appear in any visible configuration, so make sure to keep track of which principal originally created the cluster. To grant additional IAM principals the ability to interact with your cluster, edit the `aws-auth ConfigMap` within Kubernetes and create a Kubernetes `rolebinding` or `clusterrolebinding` with the name of a `group` that you specify in the `aws-auth ConfigMap`. + +For more information about working with the ConfigMap, see <>. + +[#security-iam-service-with-iam-resource-based-policies] +== Amazon EKS resource-based policies + +Amazon EKS does not support resource-based policies. + +[#security-iam-service-with-iam-tags] +== Authorization based on Amazon EKS tags + +You can attach tags to Amazon EKS resources or pass tags in a request to Amazon EKS. To control access based on tags, you provide tag information in the link:IAM/latest/UserGuide/reference_policies_elements_condition.html[condition element,type="documentation"] of a policy using the `aws:ResourceTag/[.replaceable]``key-name```, `aws:RequestTag/[.replaceable]``key-name```, or `aws:TagKeys` condition keys. For more information about tagging Amazon EKS resources, see <>. For more information about which actions that you can use tags in condition keys with, see link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html#amazonelastickubernetesservice-actions-as-permissions[Actions defined by Amazon EKS,type="documentation"] in the link:service-authorization/latest/reference/reference.html[Service Authorization Reference,type="documentation"]. + +[#security-iam-service-with-iam-roles] +== Amazon EKS IAM roles + +An link:IAM/latest/UserGuide/id_roles.html[IAM role,type="documentation"] is an entity within your {aws} account that has specific permissions. + +[#security-iam-service-with-iam-roles-tempcreds] +=== Using temporary credentials with Amazon EKS + +You can use temporary credentials to sign in with federation, assume an IAM role, or to assume a cross-account role. You obtain temporary security credentials by calling {aws} STS API operations such as link:STS/latest/APIReference/API_AssumeRole.html[AssumeRole,type="documentation"] or link:STS/latest/APIReference/API_GetFederationToken.html[GetFederationToken,type="documentation"]. + +Amazon EKS supports using temporary credentials. + +[#security-iam-service-with-iam-roles-service-linked] +=== Service-linked roles + +link:IAM/latest/UserGuide/id_roles.html#iam-term-service-linked-role[Service-linked roles,type="documentation"] allow {aws} services to access resources in other services to complete an action on your behalf. Service-linked roles appear in your IAM account and are owned by the service. An administrator can view but can't edit the permissions for service-linked roles. + +Amazon EKS supports service-linked roles. For details about creating or managing Amazon EKS service-linked roles, see <>. + +[#security-iam-service-with-iam-roles-service] +=== Service roles + +This feature allows a service to assume a link:IAM/latest/UserGuide/id_roles.html#iam-term-service-role[service role,type="documentation"] on your behalf. This role allows the service to access resources in other services to complete an action on your behalf. Service roles appear in your IAM account and are owned by the account. This means that an IAM administrator can change the permissions for this role. However, doing so might break the functionality of the service. + +Amazon EKS supports service roles. For more information, see <> and <>. + +[#security-iam-service-with-iam-roles-choose] +=== Choosing an IAM role in Amazon EKS + +When you create a cluster resource in Amazon EKS, you must choose a role to allow Amazon EKS to access several other {aws} resources on your behalf. If you have previously created a service role, then Amazon EKS provides you with a list of roles to choose from. It's important to choose a role that has the Amazon EKS managed policies attached to it. For more information, see <> and <>. diff --git a/latest/ug/security/iam-reference/security-iam-troubleshoot.adoc b/latest/ug/security/iam-reference/security-iam-troubleshoot.adoc index 6087b2c7f..0aba962dd 100644 --- a/latest/ug/security/iam-reference/security-iam-troubleshoot.adoc +++ b/latest/ug/security/iam-reference/security-iam-troubleshoot.adoc @@ -1,17 +1,16 @@ -//!!NODE_ROOT
+include::../../attributes.txt[] + [.topic] -[[security-iam-troubleshoot,security-iam-troubleshoot.title]] +[#security-iam-troubleshoot] = Troubleshooting IAM :info_titleabbrev: Troubleshooting -include::../../attributes.txt[] - This topic covers some common errors that you may see while using Amazon EKS with IAM and how to work around them. -[[iam-error,iam-error.title]] +[#iam-error] == AccessDeniedException -If you receive an `AccessDeniedException` when calling an {aws} API operation, then the link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] credentials that you're using don't have the required permissions to make that call. +If you receive an `AccessDeniedException` when calling an {aws} API operation, then the link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] credentials that you're using don't have the required permissions to make that call. [source,bash,subs="verbatim,attributes"] ---- @@ -22,20 +21,20 @@ eks:DescribeCluster on resource: {arn-aws}eks:region:111122223333:cluster/my-clu In the previous example message, the user does not have permissions to call the Amazon EKS `DescribeCluster` API operation. To provide Amazon EKS admin permissions to an IAM principal, see <>. -For more general information about IAM, see link:IAM/latest/UserGuide/access_controlling.html[Controlling access using policies,type="documentation"] in the _IAM User Guide_. +For more general information about IAM, see link:IAM/latest/UserGuide/access_controlling.html[Controlling access using policies,type="documentation"] in the _IAM User Guide_. -[[security-iam-troubleshoot-cannot-view-nodes-or-workloads,security-iam-troubleshoot-cannot-view-nodes-or-workloads.title]] -== Can't see *Nodes* on the *Compute* tab or anything on the *Resources* tab and you receive an error in the {aws-management-console} +[#security-iam-troubleshoot-cannot-view-nodes-or-workloads] +== Can't see *Nodes* on the *Compute* tab or anything on the *Resources* tab and you receive an error in the {aws-management-console} :info_titleabbrev: Can't see anything on Nodes or Compute tabs in console -You may see a console error message that says `Your current user or role does not have access to Kubernetes objects on this EKS cluster`. Make sure that the link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] user that you're using the {aws-management-console} with has the necessary permissions. For more information, see <>. +You may see a console error message that says `Your current user or role does not have access to Kubernetes objects on this EKS cluster`. Make sure that the link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] user that you're using the {aws-management-console} with has the necessary permissions. For more information, see <>. -[[security-iam-troubleshoot-configmap,security-iam-troubleshoot-configmap.title]] +[#security-iam-troubleshoot-configmap] == aws-auth `ConfigMap` does not grant access to the cluster The https://github.com/kubernetes-sigs/aws-iam-authenticator[{aws} IAM Authenticator] doesn't permit a path in the role ARN used in the `ConfigMap`. Therefore, before you specify `rolearn`, remove the path. For example, change `{arn-aws}iam::[.replaceable]``111122223333``:role/[.replaceable]``team``/[.replaceable]``developers``/[.replaceable]``eks-admin``` to `{arn-aws}iam::[.replaceable]``111122223333``:role/[.replaceable]``eks-admin```. -[[security-iam-troubleshoot-passrole,security-iam-troubleshoot-passrole.title]] +[#security-iam-troubleshoot-passrole] == I am not authorized to perform iam:PassRole If you receive an error that you're not authorized to perform the `iam:PassRole` action, your policies must be updated to allow you to pass a role to Amazon EKS. @@ -53,7 +52,7 @@ In this case, Mary's policies must be updated to allow her to perform the `iam:P If you need help, contact your {aws} administrator. Your administrator is the person who provided you with your sign-in credentials. -[[security-iam-troubleshoot-cross-account-access,security-iam-troubleshoot-cross-account-access.title]] +[#security-iam-troubleshoot-cross-account-access] == I want to allow people outside of my {aws} account to access my Amazon EKS resources :info_titleabbrev: Allow external IAM principals to access resources @@ -62,17 +61,17 @@ You can create a role that users in other accounts or people outside of your org To learn more, consult the following: * To learn whether Amazon EKS supports these features, see <>. -* To learn how to provide access to your resources across {aws} accounts that you own, see link:IAM/latest/UserGuide/id_roles_common-scenarios_aws-accounts.html[Providing access to an IAM user in another {aws} account that you own,type="documentation"] in the _IAM User Guide_. -* To learn how to provide access to your resources to third-party {aws} accounts, see link:IAM/latest/UserGuide/id_roles_common-scenarios_third-party.html[Providing access to {aws} accounts owned by third parties,type="documentation"] in the _IAM User Guide_. -* To learn how to provide access through identity federation, see link:IAM/latest/UserGuide/id_roles_common-scenarios_federated-users.html[Providing access to externally authenticated users (identity federation),type="documentation"] in the _IAM User Guide_. -* To learn the difference between using roles and resource-based policies for cross-account access, see link:IAM/latest/UserGuide/access_policies-cross-account-resource-access.html[Cross account resource access in IAM,type="documentation"] in the _IAM User Guide_. +* To learn how to provide access to your resources across {aws} accounts that you own, see link:IAM/latest/UserGuide/id_roles_common-scenarios_aws-accounts.html[Providing access to an IAM user in another {aws} account that you own,type="documentation"] in the _IAM User Guide_. +* To learn how to provide access to your resources to third-party {aws} accounts, see link:IAM/latest/UserGuide/id_roles_common-scenarios_third-party.html[Providing access to {aws} accounts owned by third parties,type="documentation"] in the _IAM User Guide_. +* To learn how to provide access through identity federation, see link:IAM/latest/UserGuide/id_roles_common-scenarios_federated-users.html[Providing access to externally authenticated users (identity federation),type="documentation"] in the _IAM User Guide_. +* To learn the difference between using roles and resource-based policies for cross-account access, see link:IAM/latest/UserGuide/access_policies-cross-account-resource-access.html[Cross account resource access in IAM,type="documentation"] in the _IAM User Guide_. -[[security-iam-troubleshoot-wrong-sts-endpoint,security-iam-troubleshoot-wrong-sts-endpoint.title]] +[#security-iam-troubleshoot-wrong-sts-endpoint] == Pod containers receive the following error: `An error occurred (SignatureDoesNotMatch) when calling the GetCallerIdentity operation: Credential should be scoped to a valid region` :info_titleabbrev: Credential should be scoped to a valid region error -Your containers receive this error if your application is explicitly making requests to the {aws} STS global endpoint (`https://sts.amazonaws`) and your [.noloc]`Kubernetes` service account is configured to use a regional endpoint. You can resolve the issue with one of the following options: +Your containers receive this error if your application is explicitly making requests to the {aws} STS global endpoint (`https://sts.amazonaws`) and your Kubernetes service account is configured to use a regional endpoint. You can resolve the issue with one of the following options: * Update your application code to remove explicit calls to the {aws} STS global endpoint. -* Update your application code to make explicit calls to regional endpoints such as `https://sts.us-west-2.amazonaws.com`. Your application should have redundancy built in to pick a different {aws} Region in the event of a failure of the service in the {aws} Region. For more information, see link:IAM/latest/UserGuide/id_credentials_temp_enable-regions.html[Managing {aws} STS in an {aws} Region,type="documentation"] in the IAM User Guide. -* Configure your service accounts to use the global endpoint. All versions earlier than `1.22` used the global endpoint by default, but version `1.22` and later clusters use the regional endpoint by default. For more information, see <>. +* Update your application code to make explicit calls to regional endpoints such as `https://sts.us-west-2.amazonaws.com`. Your application should have redundancy built in to pick a different {aws} Region in the event of a failure of the service in the {aws} Region. For more information, see link:IAM/latest/UserGuide/id_credentials_temp_enable-regions.html[Managing {aws} STS in an {aws} Region,type="documentation"] in the IAM User Guide. +* Configure your service accounts to use the global endpoint. Clusters use the regional endpoint by default. For more information, see <>. \ No newline at end of file diff --git a/latest/ug/security/iam-reference/security-iam.adoc b/latest/ug/security/iam-reference/security-iam.adoc index 7097c8d7f..daeb4f56d 100644 --- a/latest/ug/security/iam-reference/security-iam.adoc +++ b/latest/ug/security/iam-reference/security-iam.adoc @@ -1,14 +1,19 @@ -//!!NODE_ROOT
+include::../../attributes.txt[] + [.topic] -[[security-iam,security-iam.title]] +[#security-iam] = Identity and access management for Amazon EKS -:info_doctype: section -:info_title: Identity and access management for Amazon EKS :info_titleabbrev: IAM Reference -:info_abstract: How to authenticate requests and manage access your Amazon EKS \ - resources. -include::../../attributes.txt[] +include::security-iam-service-with-iam.adoc[leveloffset=+1] + +include::security-iam-id-based-policy-examples.adoc[leveloffset=+1] + +include::using-service-linked-roles.adoc[leveloffset=+1] + +include::pod-execution-role.adoc[leveloffset=+1] + +include::connector-iam-role.adoc[leveloffset=+1] include::security-iam-awsmanpol.adoc[leveloffset=+1] @@ -27,9 +32,9 @@ include::auto-create-node-role.adoc[leveloffset=+1] How to authenticate requests and manage access your Amazon EKS resources. -- -{aws} Identity and Access Management (IAM) is an {aws} service that helps an administrator securely control access to {aws} resources. IAM administrators control who can be _authenticated_ (signed in) and _authorized_ (have permissions) to use Amazon EKS resources. IAM is an {aws} service that you can use with no additional charge. +{aws} Identity and Access Management (IAM) is an {aws} service that helps an administrator securely control access to {aws} resources. IAM administrators control who can be _authenticated_ (signed in) and _authorized_ (have permissions) to use Amazon EKS resources. IAM is an {aws} service that you can use with no additional charge. -[[security-iam-audience,security-iam-audience.title]] +[#security-iam-audience] == Audience How you use {aws} Identity and Access Management (IAM) differs, depending on the work that you do in Amazon EKS. @@ -40,58 +45,58 @@ How you use {aws} Identity and Access Management (IAM) differs, depending on the *IAM administrator* – If you're an IAM administrator, you might want to learn details about how you can write policies to manage access to Amazon EKS. To view example Amazon EKS identity-based policies that you can use in IAM, see <>. -[[security-iam-authentication,security-iam-authentication.title]] +[#security-iam-authentication] == Authenticating with identities -Authentication is how you sign in to {aws} using your identity credentials. You must be _authenticated_ (signed in to {aws}) as the {aws} account root user, as an IAM user, or by assuming an IAM role. +Authentication is how you sign in to {aws} using your identity credentials. You must be _authenticated_ (signed in to {aws}) as the {aws} account root user, as an IAM user, or by assuming an IAM role. You can sign in to {aws} as a federated identity by using credentials provided through an identity source. {aws} IAM Identity Center (IAM Identity Center) users, your company's single sign-on authentication, and your Google or Facebook credentials are examples of federated identities. When you sign in as a federated identity, your administrator previously set up identity federation using IAM roles. When you access {aws} by using federation, you are indirectly assuming a role. Depending on the type of user you are, you can sign in to the {aws-management-console} or the {aws} access portal. For more information about signing in to {aws}, see link:signin/latest/userguide/how-to-sign-in.html[How to sign in to your {aws} account,type="documentation"] in the _{aws} Sign-In User Guide_. -If you access {aws} programmatically, {aws} provides a software development kit (SDK) and a command line interface (CLI) to cryptographically sign your requests by using your credentials. If you don't use {aws} tools, you must sign requests yourself. For more information about using the recommended method to sign requests yourself, see link:IAM/latest/UserGuide/reference_aws-signing.html[Signing {aws} API requests,type="documentation"] in the _IAM User Guide_. +If you access {aws} programmatically, {aws} provides a software development kit (SDK) and a command line interface (CLI) to cryptographically sign your requests by using your credentials. If you don't use {aws} tools, you must sign requests yourself. For more information about using the recommended method to sign requests yourself, see link:IAM/latest/UserGuide/reference_aws-signing.html[Signing {aws} API requests,type="documentation"] in the _IAM User Guide_. -Regardless of the authentication method that you use, you might be required to provide additional security information. For example, {aws} recommends that you use multi-factor authentication (MFA) to increase the security of your account. To learn more, see link:singlesignon/latest/userguide/enable-mfa.html[Multi-factor authentication,type="documentation"] in the _{aws} IAM Identity Center User Guide_ and link:IAM/latest/UserGuide/id_credentials_mfa.html[Using multi-factor authentication (MFA) in {aws},type="documentation"] in the _IAM User Guide_. +Regardless of the authentication method that you use, you might be required to provide additional security information. For example, {aws} recommends that you use multi-factor authentication (MFA) to increase the security of your account. To learn more, see link:singlesignon/latest/userguide/enable-mfa.html[Multi-factor authentication,type="documentation"] in the _{aws} IAM Identity Center User Guide_ and link:IAM/latest/UserGuide/id_credentials_mfa.html[Using multi-factor authentication (MFA) in {aws},type="documentation"] in the _IAM User Guide_. -[[security-iam-authentication-rootuser,security-iam-authentication-rootuser.title]] +[#security-iam-authentication-rootuser] === {aws} account root user -When you create an {aws} account, you begin with one sign-in identity that has complete access to all {aws} services and resources in the account. This identity is called the {aws} account _root user_ and is accessed by signing in with the email address and password that you used to create the account. We strongly recommend that you don't use the root user for your everyday tasks. Safeguard your root user credentials and use them to perform the tasks that only the root user can perform. For the complete list of tasks that require you to sign in as the root user, see link:IAM/latest/UserGuide/id_root-user.html#root-user-tasks[Tasks that require root user credentials,type="documentation"] in the _IAM User Guide_. +When you create an {aws} account, you begin with one sign-in identity that has complete access to all {aws} services and resources in the account. This identity is called the {aws} account _root user_ and is accessed by signing in with the email address and password that you used to create the account. We strongly recommend that you don't use the root user for your everyday tasks. Safeguard your root user credentials and use them to perform the tasks that only the root user can perform. For the complete list of tasks that require you to sign in as the root user, see link:IAM/latest/UserGuide/id_root-user.html#root-user-tasks[Tasks that require root user credentials,type="documentation"] in the _IAM User Guide_. -[[security-iam-authentication-iamuser,security-iam-authentication-iamuser.title]] +[#security-iam-authentication-iamuser] === IAM users and groups -An _ link:IAM/latest/UserGuide/id_users.html[IAM user,type="documentation"]_ is an identity within your {aws} account that has specific permissions for a single person or application. Where possible, we recommend relying on temporary credentials instead of creating IAM users who have long-term credentials such as passwords and access keys. However, if you have specific use cases that require long-term credentials with IAM users, we recommend that you rotate access keys. For more information, see link:IAM/latest/UserGuide/best-practices.html#rotate-credentials[Rotate access keys regularly for use cases that require long-term credentials,type="documentation"] in the _IAM User Guide_. +An link:IAM/latest/UserGuide/id_users.html[_IAM user_,type="documentation"] is an identity within your {aws} account that has specific permissions for a single person or application. Where possible, we recommend relying on temporary credentials instead of creating IAM users who have long-term credentials such as passwords and access keys. However, if you have specific use cases that require long-term credentials with IAM users, we recommend that you rotate access keys. For more information, see link:IAM/latest/UserGuide/best-practices.html#rotate-credentials[Rotate access keys regularly for use cases that require long-term credentials,type="documentation"] in the _IAM User Guide_. -An link:IAM/latest/UserGuide/id_groups.html[IAM group,type="documentation"] is an identity that specifies a collection of IAM users. You can't sign in as a group. You can use groups to specify permissions for multiple users at a time. Groups make permissions easier to manage for large sets of users. For example, you could have a group named _IAMAdmins_ and give that group permissions to administer IAM resources. +An link:IAM/latest/UserGuide/id_groups.html[IAM group,type="documentation"] is an identity that specifies a collection of IAM users. You can't sign in as a group. You can use groups to specify permissions for multiple users at a time. Groups make permissions easier to manage for large sets of users. For example, you could have a group named _IAMAdmins_ and give that group permissions to administer IAM resources. -Users are different from roles. A user is uniquely associated with one person or application, but a role is intended to be assumable by anyone who needs it. Users have permanent long-term credentials, but roles provide temporary credentials. To learn more, see link:IAM/latest/UserGuide/id.html#id_which-to-choose[When to create an IAM user (instead of a role),type="documentation"] in the _IAM User Guide_. +Users are different from roles. A user is uniquely associated with one person or application, but a role is intended to be assumable by anyone who needs it. Users have permanent long-term credentials, but roles provide temporary credentials. To learn more, see link:IAM/latest/UserGuide/id.html#id_which-to-choose[When to create an IAM user (instead of a role),type="documentation"] in the _IAM User Guide_. -[[security-iam-authentication-iamrole,security-iam-authentication-iamrole.title]] +[#security-iam-authentication-iamrole] === IAM roles -An _ link:IAM/latest/UserGuide/id_roles.html[IAM role,type="documentation"]_ is an identity within your {aws} account that has specific permissions. It is similar to an IAM user, but is not associated with a specific person. You can temporarily assume an IAM role in the {aws-management-console} by link:IAM/latest/UserGuide/id_roles_use_switch-role-console.html[switching roles,type="documentation"]. You can assume a role by calling an {aws} CLI or {aws} API operation or by using a custom URL. For more information about methods for using roles, see link:IAM/latest/UserGuide/id_roles_use.html[Using IAM roles,type="documentation"] in the _IAM User Guide_. +An link:IAM/latest/UserGuide/id_roles.html[_IAM role_,type="documentation"] is an identity within your {aws} account that has specific permissions. It is similar to an IAM user, but is not associated with a specific person. You can temporarily assume an IAM role in the {aws-management-console} by link:IAM/latest/UserGuide/id_roles_use_switch-role-console.html[switching roles,type="documentation"]. You can assume a role by calling an {aws} CLI or {aws} API operation or by using a custom URL. For more information about methods for using roles, see link:IAM/latest/UserGuide/id_roles_use.html[Using IAM roles,type="documentation"] in the _IAM User Guide_. IAM roles with temporary credentials are useful in the following situations: -* *Federated user access* – To assign permissions to a federated identity, you create a role and define permissions for the role. When a federated identity authenticates, the identity is associated with the role and is granted the permissions that are defined by the role. For information about roles for federation, see link:IAM/latest/UserGuide/id_roles_create_for-idp.html[Creating a role for a third-party Identity Provider,type="documentation"] in the _IAM User Guide_. If you use IAM Identity Center, you configure a permission set. To control what your identities can access after they authenticate, IAM Identity Center correlates the permission set to a role in IAM. For information about permissions sets, see link:singlesignon/latest/userguide/permissionsetsconcept.html[Permission sets,type="documentation"] in the _{aws} IAM Identity Center User Guide_. +* *Federated user access* – To assign permissions to a federated identity, you create a role and define permissions for the role. When a federated identity authenticates, the identity is associated with the role and is granted the permissions that are defined by the role. For information about roles for federation, see link:IAM/latest/UserGuide/id_roles_create_for-idp.html[Creating a role for a third-party Identity Provider,type="documentation"] in the _IAM User Guide_. If you use IAM Identity Center, you configure a permission set. To control what your identities can access after they authenticate, IAM Identity Center correlates the permission set to a role in IAM. For information about permissions sets, see link:singlesignon/latest/userguide/permissionsetsconcept.html[Permission sets,type="documentation"] in the _{aws} IAM Identity Center User Guide_. * *Temporary IAM user permissions* – An IAM user or role can assume an IAM role to temporarily take on different permissions for a specific task. -* *Cross-account access* – You can use an IAM role to allow someone (a trusted principal) in a different account to access resources in your account. Roles are the primary way to grant cross-account access. However, with some {aws} services, you can attach a policy directly to a resource (instead of using a role as a proxy). To learn the difference between roles and resource-based policies for cross-account access, see link:IAM/latest/UserGuide/access_policies-cross-account-resource-access.html[Cross account resource access in IAM,type="documentation"] in the _IAM User Guide_. +* *Cross-account access* – You can use an IAM role to allow someone (a trusted principal) in a different account to access resources in your account. Roles are the primary way to grant cross-account access. However, with some {aws} services, you can attach a policy directly to a resource (instead of using a role as a proxy). To learn the difference between roles and resource-based policies for cross-account access, see link:IAM/latest/UserGuide/access_policies-cross-account-resource-access.html[Cross account resource access in IAM,type="documentation"] in the _IAM User Guide_. * *Cross-service access* – Some {aws} services use features in other {aws} services. For example, when you make a call in a service, it's common for that service to run applications in Amazon EC2 or store objects in Amazon S3. A service might do this using the calling principal's permissions, using a service role, or using a service-linked role. + -** *Forward access sessions (FAS)* – When you use an IAM user or role to perform actions in {aws}, you are considered a principal. When you use some services, you might perform an action that then initiates another action in a different service. FAS uses the permissions of the principal calling an {aws} service, combined with the requesting {aws} service to make requests to downstream services. FAS requests are only made when a service receives a request that requires interactions with other {aws} services or resources to complete. In this case, you must have permissions to perform both actions. For policy details when making FAS requests, see link:IAM/latest/UserGuide/access_forward_access_sessions.html[Forward access sessions,type="documentation"]. -** *Service role* – A service role is an link:IAM/latest/UserGuide/id_roles.html[IAM role,type="documentation"] that a service assumes to perform actions on your behalf. An IAM administrator can create, modify, and delete a service role from within IAM. For more information, see link:IAM/latest/UserGuide/id_roles_create_for-service.html[Creating a role to delegate permissions to an {aws} service,type="documentation"] in the _IAM User Guide_. +** *Forward access sessions (FAS)* – When you use an IAM user or role to perform actions in {aws}, you are considered a principal. When you use some services, you might perform an action that then initiates another action in a different service. FAS uses the permissions of the principal calling an {aws} service, combined with the requesting {aws} service to make requests to downstream services. FAS requests are only made when a service receives a request that requires interactions with other {aws} services or resources to complete. In this case, you must have permissions to perform both actions. For policy details when making FAS requests, see link:IAM/latest/UserGuide/access_forward_access_sessions.html[Forward access sessions,type="documentation"]. +** *Service role* – A service role is an link:IAM/latest/UserGuide/id_roles.html[IAM role,type="documentation"] that a service assumes to perform actions on your behalf. An IAM administrator can create, modify, and delete a service role from within IAM. For more information, see link:IAM/latest/UserGuide/id_roles_create_for-service.html[Creating a role to delegate permissions to an {aws} service,type="documentation"] in the _IAM User Guide_. ** *Service-linked role* – A service-linked role is a type of service role that is linked to an {aws} service. The service can assume the role to perform an action on your behalf. Service-linked roles appear in your {aws} account and are owned by the service. An IAM administrator can view, but not edit the permissions for service-linked roles. -* *Applications running on Amazon EC2* – You can use an IAM role to manage temporary credentials for applications that are running on an EC2 instance and making {aws} CLI or {aws} API requests. This is preferable to storing access keys within the EC2 instance. To assign an {aws} role to an EC2 instance and make it available to all of its applications, you create an instance profile that is attached to the instance. An instance profile contains the role and enables programs that are running on the EC2 instance to get temporary credentials. For more information, see link:IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[Using an IAM role to grant permissions to applications running on Amazon EC2 instances,type="documentation"] in the _IAM User Guide_. +* *Applications running on Amazon EC2* – You can use an IAM role to manage temporary credentials for applications that are running on an EC2 instance and making {aws} CLI or {aws} API requests. This is preferable to storing access keys within the EC2 instance. To assign an {aws} role to an EC2 instance and make it available to all of its applications, you create an instance profile that is attached to the instance. An instance profile contains the role and enables programs that are running on the EC2 instance to get temporary credentials. For more information, see link:IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[Using an IAM role to grant permissions to applications running on Amazon EC2 instances,type="documentation"] in the _IAM User Guide_. -To learn whether to use IAM roles or IAM users, see link:IAM/latest/UserGuide/id.html#id_which-to-choose_role[When to create an IAM role (instead of a user),type="documentation"] in the _IAM User Guide_. +To learn whether to use IAM roles or IAM users, see link:IAM/latest/UserGuide/id.html#id_which-to-choose_role[When to create an IAM role (instead of a user),type="documentation"] in the _IAM User Guide_. -[[security-iam-access-manage,security-iam-access-manage.title]] +[#security-iam-access-manage] == Managing access using policies -You control access in {aws} by creating policies and attaching them to {aws} identities or resources. A policy is an object in {aws} that, when associated with an identity or resource, defines their permissions. {aws} evaluates these policies when a principal (user, root user, or role session) makes a request. Permissions in the policies determine whether the request is allowed or denied. Most policies are stored in {aws} as JSON documents. For more information about the structure and contents of JSON policy documents, see link:IAM/latest/UserGuide/access_policies.html#access_policies-json[Overview of JSON policies,type="documentation"] in the _IAM User Guide_. +You control access in {aws} by creating policies and attaching them to {aws} identities or resources. A policy is an object in {aws} that, when associated with an identity or resource, defines their permissions. {aws} evaluates these policies when a principal (user, root user, or role session) makes a request. Permissions in the policies determine whether the request is allowed or denied. Most policies are stored in {aws} as JSON documents. For more information about the structure and contents of JSON policy documents, see link:IAM/latest/UserGuide/access_policies.html#access_policies-json[Overview of JSON policies,type="documentation"] in the _IAM User Guide_. Administrators can use {aws} JSON policies to specify who has access to what. That is, which *principal* can perform *actions* on what *resources*, and under what *conditions*. @@ -99,1214 +104,41 @@ By default, users and roles have no permissions. To grant users permission to pe IAM policies define permissions for an action regardless of the method that you use to perform the operation. For example, suppose that you have a policy that allows the `iam:GetRole` action. A user with that policy can get role information from the {aws-management-console}, the {aws} CLI, or the {aws} API. -[[security-iam-access-manage-id-based-policies,security-iam-access-manage-id-based-policies.title]] +[#security-iam-access-manage-id-based-policies] === Identity-based policies -Identity-based policies are JSON permissions policy documents that you can attach to an identity, such as an IAM user, group of users, or role. These policies control what actions users and roles can perform, on which resources, and under what conditions. To learn how to create an identity-based policy, see link:IAM/latest/UserGuide/access_policies_create.html[Creating IAM policies,type="documentation"] in the _IAM User Guide_. +Identity-based policies are JSON permissions policy documents that you can attach to an identity, such as an IAM user, group of users, or role. These policies control what actions users and roles can perform, on which resources, and under what conditions. To learn how to create an identity-based policy, see link:IAM/latest/UserGuide/access_policies_create.html[Creating IAM policies,type="documentation"] in the _IAM User Guide_. -Identity-based policies can be further categorized as _inline policies_ or _managed policies_. Inline policies are embedded directly into a single user, group, or role. Managed policies are standalone policies that you can attach to multiple users, groups, and roles in your {aws} account. Managed policies include {aws} managed policies and customer managed policies. To learn how to choose between a managed policy or an inline policy, see link:IAM/latest/UserGuide/access_policies_managed-vs-inline.html#choosing-managed-or-inline[Choosing between managed policies and inline policies,type="documentation"] in the _IAM User Guide_. +Identity-based policies can be further categorized as _inline policies_ or _managed policies_. Inline policies are embedded directly into a single user, group, or role. Managed policies are standalone policies that you can attach to multiple users, groups, and roles in your {aws} account. Managed policies include {aws} managed policies and customer managed policies. To learn how to choose between a managed policy or an inline policy, see link:IAM/latest/UserGuide/access_policies_managed-vs-inline.html#choosing-managed-or-inline[Choosing between managed policies and inline policies,type="documentation"] in the _IAM User Guide_. -[[security-iam-access-manage-resource-based-policies,security-iam-access-manage-resource-based-policies.title]] +[#security-iam-access-manage-resource-based-policies] === Resource-based policies -Resource-based policies are JSON policy documents that you attach to a resource. Examples of resource-based policies are IAM _role trust policies_ and Amazon S3 _bucket policies_. In services that support resource-based policies, service administrators can use them to control access to a specific resource. For the resource where the policy is attached, the policy defines what actions a specified principal can perform on that resource and under what conditions. You must link:IAM/latest/UserGuide/reference_policies_elements_principal.html[specify a principal,type="documentation"] in a resource-based policy. Principals can include accounts, users, roles, federated users, or {aws} services. +Resource-based policies are JSON policy documents that you attach to a resource. Examples of resource-based policies are IAM _role trust policies_ and Amazon S3 _bucket policies_. In services that support resource-based policies, service administrators can use them to control access to a specific resource. For the resource where the policy is attached, the policy defines what actions a specified principal can perform on that resource and under what conditions. You must link:IAM/latest/UserGuide/reference_policies_elements_principal.html[specify a principal,type="documentation"] in a resource-based policy. Principals can include accounts, users, roles, federated users, or {aws} services. Resource-based policies are inline policies that are located in that service. You can't use {aws} managed policies from IAM in a resource-based policy. -[[security-iam-access-manage-acl,security-iam-access-manage-acl.title]] +[#security-iam-access-manage-acl] === Access control lists (ACLs) Access control lists (ACLs) control which principals (account members, users, or roles) have permissions to access a resource. ACLs are similar to resource-based policies, although they do not use the JSON policy document format. Amazon S3, {aws} WAF, and Amazon VPC are examples of services that support ACLs. To learn more about ACLs, see link:AmazonS3/latest/userguide/acl-overview.html[Access control list (ACL) overview,type="documentation"] in the _Amazon Simple Storage Service Developer Guide_. -[[security-iam-access-manage-other-policies,security-iam-access-manage-other-policies.title]] +[#security-iam-access-manage-other-policies] === Other policy types {aws} supports additional, less-common policy types. These policy types can set the maximum permissions granted to you by the more common policy types. -* *Permissions boundaries* – A permissions boundary is an advanced feature in which you set the maximum permissions that an identity-based policy can grant to an IAM entity (IAM user or role). You can set a permissions boundary for an entity. The resulting permissions are the intersection of an entity's identity-based policies and its permissions boundaries. Resource-based policies that specify the user or role in the `Principal` field are not limited by the permissions boundary. An explicit deny in any of these policies overrides the allow. For more information about permissions boundaries, see link:IAM/latest/UserGuide/access_policies_boundaries.html[Permissions boundaries for IAM entities,type="documentation"] in the _IAM User Guide_. -* *Service control policies (SCPs)* – SCPs are JSON policies that specify the maximum permissions for an organization or organizational unit (OU) in {aws} Organizations. {aws} Organizations is a service for grouping and centrally managing multiple {aws} accounts that your business owns. If you enable all features in an organization, then you can apply service control policies (SCPs) to any or all of your accounts. The SCP limits permissions for entities in member accounts, including each {aws} account root user. For more information about Organizations and SCPs, see link:organizations/latest/userguide/orgs_manage_policies_scps.html[Service control policies,type="documentation"] in the _{aws} Organizations User Guide_. -* *Session policies* – Session policies are advanced policies that you pass as a parameter when you programmatically create a temporary session for a role or federated user. The resulting session's permissions are the intersection of the user or role's identity-based policies and the session policies. Permissions can also come from a resource-based policy. An explicit deny in any of these policies overrides the allow. For more information, see link:IAM/latest/UserGuide/access_policies.html#policies_session[Session policies,type="documentation"] in the _IAM User Guide_. +* *Permissions boundaries* – A permissions boundary is an advanced feature in which you set the maximum permissions that an identity-based policy can grant to an IAM entity (IAM user or role). You can set a permissions boundary for an entity. The resulting permissions are the intersection of an entity's identity-based policies and its permissions boundaries. Resource-based policies that specify the user or role in the `Principal` field are not limited by the permissions boundary. An explicit deny in any of these policies overrides the allow. For more information about permissions boundaries, see link:IAM/latest/UserGuide/access_policies_boundaries.html[Permissions boundaries for IAM entities,type="documentation"] in the _IAM User Guide_. +* *Service control policies (SCPs)* – SCPs are JSON policies that specify the maximum permissions for an organization or organizational unit (OU) in {aws} Organizations. {aws} Organizations is a service for grouping and centrally managing multiple {aws} accounts that your business owns. If you enable all features in an organization, then you can apply service control policies (SCPs) to any or all of your accounts. The SCP limits permissions for entities in member accounts, including each {aws} account root user. For more information about Organizations and SCPs, see link:organizations/latest/userguide/orgs_manage_policies_scps.html[Service control policies,type="documentation"] in the _{aws} Organizations User Guide_. +* *Session policies* – Session policies are advanced policies that you pass as a parameter when you programmatically create a temporary session for a role or federated user. The resulting session's permissions are the intersection of the user or role's identity-based policies and the session policies. Permissions can also come from a resource-based policy. An explicit deny in any of these policies overrides the allow. For more information, see link:IAM/latest/UserGuide/access_policies.html#policies_session[Session policies,type="documentation"] in the _IAM User Guide_. -[[security-iam-access-manage-multiple-policies,security-iam-access-manage-multiple-policies.title]] +[#security-iam-access-manage-multiple-policies] === Multiple policy types -When multiple types of policies apply to a request, the resulting permissions are more complicated to understand. To learn how {aws} determines whether to allow a request when multiple policy types are involved, see link:IAM/latest/UserGuide/reference_policies_evaluation-logic.html[Policy evaluation logic,type="documentation"] in the _IAM User Guide_. - -[.topic] -[[security-iam-service-with-iam,security-iam-service-with-iam.title]] -== How Amazon EKS works with IAM - -Before you use IAM to manage access to Amazon EKS, you should understand what IAM features are available to use with Amazon EKS. To get a high-level view of how Amazon EKS and other {aws} services work with IAM, see link:IAM/latest/UserGuide/reference_aws-services-that-work-with-iam.html[{aws} services that work with IAM,type="documentation"] in the _IAM User Guide_. - -[.topiclist] -[[Topic List]] - -[[security-iam-service-with-iam-id-based-policies,security-iam-service-with-iam-id-based-policies.title]] -=== Amazon EKS identity-based policies - -With IAM identity-based policies, you can specify allowed or denied actions and resources as well as the conditions under which actions are allowed or denied. Amazon EKS supports specific actions, resources, and condition keys. To learn about all of the elements that you use in a JSON policy, see link:IAM/latest/UserGuide/reference_policies_elements.html[IAM JSON policy elements reference,type="documentation"] in the _IAM User Guide_. - -[[security-iam-service-with-iam-id-based-policies-actions,security-iam-service-with-iam-id-based-policies-actions.title]] -==== Actions - -Administrators can use {aws} JSON policies to specify who has access to what. That is, which *principal* can perform *actions* on what *resources*, and under what *conditions*. - -The `Action` element of a JSON policy describes the actions that you can use to allow or deny access in a policy. Policy actions usually have the same name as the associated {aws} API operation. There are some exceptions, such as _permission-only actions_ that don't have a matching API operation. There are also some operations that require multiple actions in a policy. These additional actions are called _dependent actions_. - -Include actions in a policy to grant permissions to perform the associated operation. - -Policy actions in Amazon EKS use the following prefix before the action: `eks:`. For example, to grant someone permission to get descriptive information about an Amazon EKS cluster, you include the `DescribeCluster` action in their policy. Policy statements must include either an `Action` or `NotAction` element. - -To specify multiple actions in a single statement, separate them with commas as follows: - -[source,json,subs="verbatim,attributes"] ----- -"Action": ["eks:action1", "eks:action2"] ----- - -You can specify multiple actions using wildcards (*). For example, to specify all actions that begin with the word `Describe`, include the following action: - -[source,json,subs="verbatim,attributes"] ----- -"Action": "eks:Describe*" ----- - - -To see a list of Amazon EKS actions, see link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html#amazonelastickubernetesservice-actions-as-permissions[Actions defined by Amazon Elastic Kubernetes Service,type="documentation"] in the _Service Authorization Reference_. - -[[security-iam-service-with-iam-id-based-policies-resources,security-iam-service-with-iam-id-based-policies-resources.title]] -==== Resources - -Administrators can use {aws} JSON policies to specify who has access to what. That is, which *principal* can perform *actions* on what *resources*, and under what *conditions*. - -The `Resource` JSON policy element specifies the object or objects to which the action applies. Statements must include either a `Resource` or a `NotResource` element. As a best practice, specify a resource using its link:IAM/latest/UserGuide/reference-arns.html[Amazon Resource Name (ARN),type="documentation"]. You can do this for actions that support a specific resource type, known as _resource-level permissions_. - -For actions that don't support resource-level permissions, such as listing operations, use a wildcard (*) to indicate that the statement applies to all resources. - -[source] ----- -"Resource": "*" ----- - -The Amazon EKS cluster resource has the following ARN. - -[source,none,subs="verbatim,attributes"] ----- -{arn-aws}eks:region-code:account-id:cluster/cluster-name ----- - -For more information about the format of ARNs, see link:general/latest/gr/aws-arns-and-namespaces.html[Amazon resource names (ARNs) and {aws} service namespaces,type="documentation"]. - -For example, to specify the cluster with the name [.replaceable]`my-cluster` in your statement, use the following ARN: - -[source,json,subs="verbatim,attributes"] ----- -"Resource": "{arn-aws}eks:region-code:111122223333:cluster/my-cluster" ----- - -To specify all clusters that belong to a specific account and {aws} Region, use the wildcard (*): - -[source,json,subs="verbatim,attributes"] ----- -"Resource": "{arn-aws}eks:region-code:111122223333:cluster/*" ----- - -Some Amazon EKS actions, such as those for creating resources, can't be performed on a specific resource. In those cases, you must use the wildcard (*). - -[source,json,subs="verbatim,attributes"] ----- -"Resource": "*" ----- - -To see a list of Amazon EKS resource types and their ARNs, see link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html#amazonelastickubernetesservice-resources-for-iam-policies[Resources defined by Amazon Elastic Kubernetes Service,type="documentation"] in the _Service Authorization Reference_. To learn with which actions you can specify the ARN of each resource, see link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html#amazonelastickubernetesservice-actions-as-permissions[Actions defined by Amazon Elastic Kubernetes Service,type="documentation"]. - -[[security-iam-service-with-iam-id-based-policies-conditionkeys,security-iam-service-with-iam-id-based-policies-conditionkeys.title]] -==== Condition keys - -Amazon EKS defines its own set of condition keys and also supports using some global condition keys. To see all {aws} global condition keys, see link:IAM/latest/UserGuide/reference_policies_condition-keys.html[{aws} Global Condition Context Keys,type="documentation"] in the _IAM User Guide_. - -You can set condition keys when associating an [.noloc]`OpenID Connect` provider to your cluster. For more information, see <>. - -All Amazon EC2 actions support the `aws:RequestedRegion` and `ec2:Region` condition keys. For more information, see link:AWSEC2/latest/UserGuide/ExamplePolicies_EC2.html#iam-example-region[Example: Restricting Access to a Specific {aws} Region,type="documentation"]. - -For a list of Amazon EKS condition keys, see link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html#amazonelastickubernetesservice-policy-keys[Conditions defined by Amazon Elastic Kubernetes Service,type="documentation"] in the _Service Authorization Reference_. To learn which actions and resources you can use a condition key with, see link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html#amazonelastickubernetesservice-actions-as-permissions[Actions defined by Amazon Elastic Kubernetes Service,type="documentation"]. - -[[security-iam-service-with-iam-id-based-policies-examples,security-iam-service-with-iam-id-based-policies-examples.title]] -==== Examples - - -To view examples of Amazon EKS identity-based policies, see <>. - -When you create an Amazon EKS cluster, the link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] that creates the cluster is automatically granted `system:masters` permissions in the cluster's role-based access control (RBAC) configuration in the Amazon EKS control plane. This principal doesn't appear in any visible configuration, so make sure to keep track of which principal originally created the cluster. To grant additional IAM principals the ability to interact with your cluster, edit the `aws-auth ConfigMap` within [.noloc]`Kubernetes` and create a [.noloc]`Kubernetes` `rolebinding` or `clusterrolebinding` with the name of a `group` that you specify in the `aws-auth ConfigMap`. - -For more information about working with the ConfigMap, see <>. - -[[security-iam-service-with-iam-resource-based-policies,security-iam-service-with-iam-resource-based-policies.title]] -=== Amazon EKS resource-based policies - -Amazon EKS does not support resource-based policies. - -[[security-iam-service-with-iam-tags,security-iam-service-with-iam-tags.title]] -=== Authorization based on Amazon EKS tags - -You can attach tags to Amazon EKS resources or pass tags in a request to Amazon EKS. To control access based on tags, you provide tag information in the link:IAM/latest/UserGuide/reference_policies_elements_condition.html[condition element,type="documentation"] of a policy using the `aws:ResourceTag/[.replaceable]``key-name```, `aws:RequestTag/[.replaceable]``key-name```, or `aws:TagKeys` condition keys. For more information about tagging Amazon EKS resources, see <>. For more information about which actions that you can use tags in condition keys with, see link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html#amazonelastickubernetesservice-actions-as-permissions[Actions defined by Amazon EKS,type="documentation"] in the link:service-authorization/latest/reference/reference.html[Service Authorization Reference,type="documentation"]. - -[[security-iam-service-with-iam-roles,security-iam-service-with-iam-roles.title]] -=== Amazon EKS IAM roles - -An link:IAM/latest/UserGuide/id_roles.html[IAM role,type="documentation"] is an entity within your {aws} account that has specific permissions. - -[[security-iam-service-with-iam-roles-tempcreds,security-iam-service-with-iam-roles-tempcreds.title]] -==== Using temporary credentials with Amazon EKS - -You can use temporary credentials to sign in with federation, assume an IAM role, or to assume a cross-account role. You obtain temporary security credentials by calling {aws} STS API operations such as link:STS/latest/APIReference/API_AssumeRole.html[AssumeRole,type="documentation"] or link:STS/latest/APIReference/API_GetFederationToken.html[GetFederationToken,type="documentation"]. - -Amazon EKS supports using temporary credentials. - -[[security-iam-service-with-iam-roles-service-linked,security-iam-service-with-iam-roles-service-linked.title]] -==== Service-linked roles - - link:IAM/latest/UserGuide/id_roles.html#iam-term-service-linked-role[Service-linked roles,type="documentation"] allow {aws} services to access resources in other services to complete an action on your behalf. Service-linked roles appear in your IAM account and are owned by the service. An administrator can view but can't edit the permissions for service-linked roles. - -Amazon EKS supports service-linked roles. For details about creating or managing Amazon EKS service-linked roles, see <>. - -[[security-iam-service-with-iam-roles-service,security-iam-service-with-iam-roles-service.title]] -==== Service roles - -This feature allows a service to assume a link:IAM/latest/UserGuide/id_roles.html#iam-term-service-role[service role,type="documentation"] on your behalf. This role allows the service to access resources in other services to complete an action on your behalf. Service roles appear in your IAM account and are owned by the account. This means that an IAM administrator can change the permissions for this role. However, doing so might break the functionality of the service. - -Amazon EKS supports service roles. For more information, see <> and <>. - -[[security-iam-service-with-iam-roles-choose,security-iam-service-with-iam-roles-choose.title]] -==== Choosing an IAM role in Amazon EKS - -When you create a cluster resource in Amazon EKS, you must choose a role to allow Amazon EKS to access several other {aws} resources on your behalf. If you have previously created a service role, then Amazon EKS provides you with a list of roles to choose from. It's important to choose a role that has the Amazon EKS managed policies attached to it. For more information, see <> and <>. - -[.topic] -[[security-iam-id-based-policy-examples,security-iam-id-based-policy-examples.title]] -== Amazon EKS identity-based policy examples - -By default, IAM users and roles don't have permission to create or modify Amazon EKS resources. They also can't perform tasks using the {aws-management-console}, {aws} CLI, or {aws} API. An IAM administrator must create IAM policies that grant users and roles permission to perform specific API operations on the specified resources they need. The administrator must then attach those policies to the IAM users or groups that require those permissions. - -To learn how to create an IAM identity-based policy using these example JSON policy documents, see link:IAM/latest/UserGuide/access_policies_create.html#access_policies_create-json-editor[Creating policies on the JSON tab,type="documentation"] in the _IAM User Guide_. - -When you create an Amazon EKS cluster, the link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] that creates the cluster is automatically granted `system:masters` permissions in the cluster's role-based access control (RBAC) configuration in the Amazon EKS control plane. This principal doesn't appear in any visible configuration, so make sure to keep track of which principal originally created the cluster. To grant additional IAM principals the ability to interact with your cluster, edit the `aws-auth ConfigMap` within [.noloc]`Kubernetes` and create a [.noloc]`Kubernetes` `rolebinding` or `clusterrolebinding` with the name of a `group` that you specify in the `aws-auth ConfigMap`. - -For more information about working with the ConfigMap, see <>. - -[.topiclist] -[[Topic List]] - -[[security-iam-service-with-iam-policy-best-practices,security-iam-service-with-iam-policy-best-practices.title]] -=== Policy best practices - -Identity-based policies determine whether someone can create, access, or delete Amazon EKS resources in your account. These actions can incur costs for your {aws} account. When you create or edit identity-based policies, follow these guidelines and recommendations: - - - -* *Get started with {aws} managed policies and move toward least-privilege permissions* – To get started granting permissions to your users and workloads, use the _{aws} managed policies_ that grant permissions for many common use cases. They are available in your {aws} account. We recommend that you reduce permissions further by defining {aws} customer managed policies that are specific to your use cases. For more information, see link:IAM/latest/UserGuide/access_policies_managed-vs-inline.html#aws-managed-policies[{aws} managed policies,type="documentation"] or link:IAM/latest/UserGuide/access_policies_job-functions.html[{aws} managed policies for job functions,type="documentation"] in the _IAM User Guide_. -* *Apply least-privilege permissions* – When you set permissions with IAM policies, grant only the permissions required to perform a task. You do this by defining the actions that can be taken on specific resources under specific conditions, also known as _least-privilege permissions_. For more information about using IAM to apply permissions, see link:IAM/latest/UserGuide/access_policies.html[Policies and permissions in IAM,type="documentation"] in the _IAM User Guide_. -* *Use conditions in IAM policies to further restrict access* – You can add a condition to your policies to limit access to actions and resources. For example, you can write a policy condition to specify that all requests must be sent using SSL. You can also use conditions to grant access to service actions if they are used through a specific {aws} service, such as {aws} CloudFormation. For more information, see link:IAM/latest/UserGuide/reference_policies_elements_condition.html[IAM JSON policy elements: Condition,type="documentation"] in the _IAM User Guide_. -* *Use IAM Access Analyzer to validate your IAM policies to ensure secure and functional permissions* – IAM Access Analyzer validates new and existing policies so that the policies adhere to the IAM policy language (JSON) and IAM best practices. IAM Access Analyzer provides more than 100 policy checks and actionable recommendations to help you author secure and functional policies. For more information, see link:IAM/latest/UserGuide/access-analyzer-policy-validation.html[IAM Access Analyzer policy validation,type="documentation"] in the _IAM User Guide_. -* *Require multi-factor authentication (MFA)* – If you have a scenario that requires IAM users or a root user in your {aws} account, turn on MFA for additional security. To require MFA when API operations are called, add MFA conditions to your policies. For more information, see link:IAM/latest/UserGuide/id_credentials_mfa_configure-api-require.html[Configuring MFA-protected API access,type="documentation"] in the _IAM User Guide_. - -For more information about best practices in IAM, see link:IAM/latest/UserGuide/best-practices.html[Security best practices in IAM,type="documentation"] in the _IAM User Guide_. - -[[security-iam-id-based-policy-examples-console,security-iam-id-based-policy-examples-console.title]] -=== Using the Amazon EKS console - -To access the Amazon EKS console, an link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"], must have a minimum set of permissions. These permissions allow the principal to list and view details about the Amazon EKS resources in your {aws} account. If you create an identity-based policy that is more restrictive than the minimum required permissions, the console won't function as intended for principals with that policy attached to them. - -To ensure that your IAM principals can still use the Amazon EKS console, create a policy with your own unique name, such as `AmazonEKSAdminPolicy`. Attach the policy to the principals. For more information, see link:IAM/latest/UserGuide/access_policies_manage-attach-detach.html[Adding and removing IAM identity permissions,type="documentation"] in the _IAM User Guide_. - -[IMPORTANT] -==== - -The following example policy allows a principal to view information on the *Configuration* tab in the console. To view information on the *Overview* and *Resources* tabs in the {aws-management-console}, the principal also needs [.noloc]`Kubernetes` permissions. For more information, see <>. - -==== - -[source,json,subs="verbatim,attributes"] ----- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "eks:*" - ], - "Resource": "*" - }, - { - "Effect": "Allow", - "Action": "iam:PassRole", - "Resource": "*", - "Condition": { - "StringEquals": { - "iam:PassedToService": "eks.amazonaws.com" - } - } - } - ] -} ----- - -You don't need to allow minimum console permissions for principals that are making calls only to the {aws} CLI or the {aws} API. Instead, allow access to only the actions that match the API operation that you're trying to perform. - -[[security-iam-id-based-policy-examples-view-own-permissions,security-iam-id-based-policy-examples-view-own-permissions.title]] -=== Allow IAM users to view their own permissions - -This example shows how you might create a policy that allows IAM users to view the inline and managed policies that are attached to their user identity. This policy includes permissions to complete this action on the console or programmatically using the {aws} CLI or {aws} API. - -[source,json,subs="verbatim,attributes"] ----- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "ViewOwnUserInfo", - "Effect": "Allow", - "Action": [ - "iam:GetUserPolicy", - "iam:ListGroupsForUser", - "iam:ListAttachedUserPolicies", - "iam:ListUserPolicies", - "iam:GetUser" - ], - "Resource": ["{arn-aws}iam::*:user/${aws:username}"] - }, - { - "Sid": "NavigateInConsole", - "Effect": "Allow", - "Action": [ - "iam:GetGroupPolicy", - "iam:GetPolicyVersion", - "iam:GetPolicy", - "iam:ListAttachedGroupPolicies", - "iam:ListGroupPolicies", - "iam:ListPolicyVersions", - "iam:ListPolicies", - "iam:ListUsers" - ], - "Resource": "*" - } - ] -} ----- - - -[[policy-create-cluster,policy-create-cluster.title]] -=== Create a [.noloc]`Kubernetes` cluster on the {aws} Cloud - -This example policy includes the minimum permissions required to create an Amazon EKS cluster named [.replaceable]`my-cluster` in the [.replaceable]`us-west-2` {aws} Region. You can replace the {aws} Region with the {aws} Region that you want to create a cluster in. If you see a warning that says *The actions in your policy do not support resource-level permissions and require you to choose `All resources`* in the {aws-management-console}, it can be safely ignored. If your account already has the [.replaceable]`AWSServiceRoleForAmazonEKS` role, you can remove the `iam:CreateServiceLinkedRole` action from the policy. If you've ever created an Amazon EKS cluster in your account then this role already exists, unless you deleted it. - -[source,json,subs="verbatim,attributes"] ----- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": "eks:CreateCluster", - "Resource": "{arn-aws}eks:us-west-2:111122223333:cluster/my-cluster" - }, - { - "Effect": "Allow", - "Action": "iam:CreateServiceLinkedRole", - "Resource": "{arn-aws}iam::111122223333:role/aws-service-role/eks.amazonaws.com/AWSServiceRoleForAmazonEKS", - "Condition": { - "ForAnyValue:StringEquals": { - "iam:AWSServiceName": "eks" - } - } - }, - { - "Effect": "Allow", - "Action": "iam:PassRole", - "Resource": "{arn-aws}iam::111122223333:role/cluster-role-name" - } - ] -} ----- - - -[[policy-create-local-cluster,policy-create-local-cluster.title]] -=== Create a local [.noloc]`Kubernetes` cluster on an Outpost - -This example policy includes the minimum permissions required to create an Amazon EKS local cluster named [.replaceable]`my-cluster` on an Outpost in the [.replaceable]`us-west-2` {aws} Region. You can replace the {aws} Region with the {aws} Region that you want to create a cluster in. If you see a warning that says *The actions in your policy do not support resource-level permissions and require you to choose `All resources`* in the {aws-management-console}, it can be safely ignored. If your account already has the `AWSServiceRoleForAmazonEKSLocalOutpost` role, you can remove the `iam:CreateServiceLinkedRole` action from the policy. If you've ever created an Amazon EKS local cluster on an Outpost in your account then this role already exists, unless you deleted it. - -[source,json,subs="verbatim,attributes"] ----- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": "eks:CreateCluster", - "Resource": "{arn-aws}eks:us-west-2:111122223333:cluster/my-cluster" - }, - { - "Action": [ - "ec2:DescribeSubnets", - "ec2:DescribeVpcs", - "iam:GetRole" - ], - "Resource": "*", - "Effect": "Allow" - }, - { - "Effect": "Allow", - "Action": "iam:CreateServiceLinkedRole", - "Resource": "{arn-aws}iam::111122223333:role/aws-service-role/outposts.eks-local.amazonaws.com/AWSServiceRoleForAmazonEKSLocalOutpost" - }, - { - "Effect": "Allow", - "Action": [ - "iam:PassRole", - "iam:ListAttachedRolePolicies" - ] - "Resource": "{arn-aws}iam::111122223333:role/cluster-role-name" - }, - { - "Action": [ - "iam:CreateInstanceProfile", - "iam:TagInstanceProfile", - "iam:AddRoleToInstanceProfile", - "iam:GetInstanceProfile", - "iam:DeleteInstanceProfile", - "iam:RemoveRoleFromInstanceProfile" - ], - "Resource": "{arn-aws}iam::*:instance-profile/eks-local-*", - "Effect": "Allow" - }, - ] -} ----- - - -[[policy-example1,policy-example1.title]] -=== Update a [.noloc]`Kubernetes` cluster - -This example policy includes the minimum permission required to update a cluster named [.replaceable]`my-cluster` in the us-west-2 {aws} Region. - -[source,json,subs="verbatim,attributes"] ----- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": "eks:UpdateClusterVersion", - "Resource": "{arn-aws}eks:us-west-2:111122223333:cluster/my-cluster" - } - ] -} ----- - - -[[policy-example2,policy-example2.title]] -=== List or describe all clusters - -This example policy includes the minimum permissions required to list and describe all clusters in your account. An link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] must be able to list and describe clusters to use the `update-kubeconfig` {aws} CLI command. - -[source,json,subs="verbatim,attributes"] ----- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "eks:DescribeCluster", - "eks:ListClusters" - ], - "Resource": "*" - } - ] -} ----- - - -[.topic] -[[using-service-linked-roles,using-service-linked-roles.title]] -== Using service-linked roles for Amazon EKS - -[abstract] --- -How to use service-linked roles to give Amazon EKS access to resources in your {aws} account. --- - -Amazon Elastic Kubernetes Service uses {aws} Identity and Access Management (IAM) link:IAM/latest/UserGuide/id_roles.html#iam-term-service-linked-role[service-linked roles,type="documentation"]. A service-linked role is a unique type of IAM role that is linked directly to Amazon EKS. Service-linked roles are predefined by Amazon EKS and include all the permissions that the service requires to call other {aws} services on your behalf. - -[.topiclist] -[[Topic List]] - -[.topic] -[[using-service-linked-roles-eks,using-service-linked-roles-eks.title]] -=== Using roles for Amazon EKS clusters - -[abstract] --- -How to use service-linked roles to give Amazon EKS access to resources in your {aws} account. --- - -Amazon Elastic Kubernetes Service uses {aws} Identity and Access Management (IAM) link:IAM/latest/UserGuide/id_roles.html#iam-term-service-linked-role[service-linked roles,type="documentation"]. A service-linked role is a unique type of IAM role that is linked directly to Amazon EKS. Service-linked roles are predefined by Amazon EKS and include all the permissions that the service requires to call other {aws} services on your behalf. - -A service-linked role makes setting up Amazon EKS easier because you don't have to manually add the necessary permissions. Amazon EKS defines the permissions of its service-linked roles, and unless defined otherwise, only Amazon EKS can assume its roles. The defined permissions include the trust policy and the permissions policy, and that permissions policy cannot be attached to any other IAM entity. - -You can delete a service-linked role only after first deleting their related resources. This protects your Amazon EKS resources because you can't inadvertently remove permission to access the resources. - -For information about other services that support service-linked roles, see link:IAM/latest/UserGuide/reference_aws-services-that-work-with-iam.html[{aws} services that work with IAM,type="documentation"] and look for the services that have *Yes* in the *Service-linked role* column. Choose a *Yes* with a link to view the service-linked role documentation for that service. - -[[service-linked-role-permissions-eks,service-linked-role-permissions-eks.title]] -==== Service-linked role permissions for Amazon EKS - -Amazon EKS uses the service-linked role named `AWSServiceRoleForAmazonEKS`. The role allows Amazon EKS to manage clusters in your account. The attached policies allow the role to manage the following resources: network interfaces, security groups, logs, and VPCs. - -[NOTE] -==== - -The `AWSServiceRoleForAmazonEKS` service-linked role is distinct from the role required for cluster creation. For more information, see <>. - -==== - -The `AWSServiceRoleForAmazonEKS` service-linked role trusts the following services to assume the role: - -* `eks.amazonaws.com` - -The role permissions policy allows Amazon EKS to complete the following actions on the specified resources: - -* link:aws-managed-policy/latest/reference/AmazonEKSServiceRolePolicy.html[AmazonEKSServiceRolePolicy,type="documentation"] - -You must configure permissions to allow an IAM entity (such as a user, group, or role) to create, edit, or delete a service-linked role. For more information, see link:IAM/latest/UserGuide/using-service-linked-roles.html#service-linked-role-permissions[Service-linked role permissions,type="documentation"] in the _IAM User Guide_. - -[[create-service-linked-role-eks,create-service-linked-role-eks.title]] -==== Creating a service-linked role for Amazon EKS - -You don't need to manually create a service-linked role. When you create a cluster in the {aws-management-console}, the {aws} CLI, or the {aws} API, Amazon EKS creates the service-linked role for you. - -If you delete this service-linked role, and then need to create it again, you can use the same process to recreate the role in your account. When you create a cluster, Amazon EKS creates the service-linked role for you again. - -[[edit-service-linked-role-eks,edit-service-linked-role-eks.title]] -==== Editing a service-linked role for Amazon EKS - -Amazon EKS does not allow you to edit the `AWSServiceRoleForAmazonEKS` service-linked role. After you create a service-linked role, you cannot change the name of the role because various entities might reference the role. However, you can edit the description of the role using IAM. For more information, see link:IAM/latest/UserGuide/using-service-linked-roles.html#edit-service-linked-role[Editing a service-linked role,type="documentation"] in the _IAM User Guide_. - -[[delete-service-linked-role-eks,delete-service-linked-role-eks.title]] -==== Deleting a service-linked role for Amazon EKS - -If you no longer need to use a feature or service that requires a service-linked role, we recommend that you delete that role. That way you don't have an unused entity that is not actively monitored or maintained. However, you must clean up your service-linked role before you can manually delete it. - -[[service-linked-role-review-before-delete-eks,service-linked-role-review-before-delete-eks.title]] -===== Cleaning up a service-linked role - -Before you can use IAM to delete a service-linked role, you must first delete any resources used by the role. - -[NOTE] -==== - -If the Amazon EKS service is using the role when you try to delete the resources, then the deletion might fail. If that happens, wait for a few minutes and try the operation again. - -==== -. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. -. In the left navigation pane, choose *Clusters*. -. If your cluster has any node groups or Fargate profiles, you must delete them before you can delete the cluster. For more information, see <> and <>. -. On the *Clusters* page, choose the cluster that you want to delete and choose *Delete*. -. Type the name of the cluster in the deletion confirmation window, and then choose *Delete*. -. Repeat this procedure for any other clusters in your account. Wait for all of the delete operations to finish. - - -[[slr-manual-delete-eks,slr-manual-delete-eks.title]] -===== Manually delete the service-linked role - -Use the IAM console, the {aws} CLI, or the {aws} API to delete the `AWSServiceRoleForAmazonEKS` service-linked role. For more information, see link:IAM/latest/UserGuide/using-service-linked-roles.html#delete-service-linked-role[Deleting a service-linked role,type="documentation"] in the _IAM User Guide_. - -[[slr-regions-eks,slr-regions-eks.title]] -==== Supported regions for Amazon EKS service-linked roles - -Amazon EKS supports using service-linked roles in all of the regions where the service is available. For more information, see link:general/latest/gr/eks.html[Amazon EKS endpoints and quotas,type="documentation"]. - -[.topic] -[[using-service-linked-roles-eks-nodegroups,using-service-linked-roles-eks-nodegroups.title]] -=== Using roles for Amazon EKS node groups - -[abstract] --- -How to use service-linked roles to give Amazon EKS access to resources in your {aws} account. --- - -Amazon EKS uses {aws} Identity and Access Management (IAM) link:IAM/latest/UserGuide/id_roles.html#iam-term-service-linked-role[service-linked roles,type="documentation"]. A service-linked role is a unique type of IAM role that is linked directly to Amazon EKS. Service-linked roles are predefined by Amazon EKS and include all the permissions that the service requires to call other {aws} services on your behalf. - -A service-linked role makes setting up Amazon EKS easier because you don't have to manually add the necessary permissions. Amazon EKS defines the permissions of its service-linked roles, and unless defined otherwise, only Amazon EKS can assume its roles. The defined permissions include the trust policy and the permissions policy, and that permissions policy cannot be attached to any other IAM entity. - -You can delete a service-linked role only after first deleting their related resources. This protects your Amazon EKS resources because you can't inadvertently remove permission to access the resources. - -For information about other services that support service-linked roles, see link:IAM/latest/UserGuide/reference_aws-services-that-work-with-iam.html[{aws} services that work with IAM,type="documentation"] and look for the services that have *Yes* in the *Service-linked role* column. Choose a *Yes* with a link to view the service-linked role documentation for that service. - -[[service-linked-role-permissions-eks-nodegroups,service-linked-role-permissions-eks-nodegroups.title]] -==== Service-linked role permissions for Amazon EKS - -Amazon EKS uses the service-linked role named `AWSServiceRoleForAmazonEKSNodegroup`. The role allows Amazon EKS to manage node groups in your account. The attached `AWSServiceRoleForAmazonEKSNodegroup` policy allows the role to manage the following resources: Auto Scaling groups, security groups, launch templates, and IAM instance profiles. For more information, see <>. - -The `AWSServiceRoleForAmazonEKSNodegroup` service-linked role trusts the following services to assume the role: - -* `eks-nodegroup.amazonaws.com` - -The role permissions policy allows Amazon EKS to complete the following actions on the specified resources: - -* link:aws-managed-policy/latest/reference/AWSServiceRoleForAmazonEKSNodegroup.html[AWSServiceRoleForAmazonEKSNodegroup,type="documentation"] - -You must configure permissions to allow an IAM entity (such as a user, group, or role) to create, edit, or delete a service-linked role. For more information, see link:IAM/latest/UserGuide/using-service-linked-roles.html#service-linked-role-permissions[Service-linked role permissions,type="documentation"] in the _IAM User Guide_. - -[[create-service-linked-role-eks-nodegroups,create-service-linked-role-eks-nodegroups.title]] -==== Creating a service-linked role for Amazon EKS - -You don't need to manually create a service-linked role. When you CreateNodegroup in the {aws-management-console}, the {aws} CLI, or the {aws} API, Amazon EKS creates the service-linked role for you. - -[IMPORTANT] -==== - -This service-linked role can appear in your account if you completed an action in another service that uses the features supported by this role. If you were using the Amazon EKS service before January 1, 2017, when it began supporting service-linked roles, then Amazon EKS created the AWSServiceRoleForAmazonEKSNodegroup role in your account. To learn more, see link:IAM/latest/UserGuide/troubleshoot_roles.html#troubleshoot_roles_new-role-appeared[A new role appeared in my IAM account,type="documentation"]. - -==== - -[[create-service-linked-role-service-api-eks-nodegroups,create-service-linked-role-service-api-eks-nodegroups.title]] -===== Creating a service-linked role in Amazon EKS ({aws} API) - -You don't need to manually create a service-linked role. When you create a managed node group in the {aws-management-console}, the {aws} CLI, or the {aws} API, Amazon EKS creates the service-linked role for you. - -If you delete this service-linked role, and then need to create it again, you can use the same process to recreate the role in your account. When you create another managed node group, Amazon EKS creates the service-linked role for you again. - -[[edit-service-linked-role-eks-nodegroups,edit-service-linked-role-eks-nodegroups.title]] -==== Editing a service-linked role for Amazon EKS - -Amazon EKS does not allow you to edit the `AWSServiceRoleForAmazonEKSNodegroup` service-linked role. After you create a service-linked role, you cannot change the name of the role because various entities might reference the role. However, you can edit the description of the role using IAM. For more information, see link:IAM/latest/UserGuide/using-service-linked-roles.html#edit-service-linked-role[Editing a service-linked role,type="documentation"] in the _IAM User Guide_. - -[[delete-service-linked-role-eks-nodegroups,delete-service-linked-role-eks-nodegroups.title]] -==== Deleting a service-linked role for Amazon EKS - -If you no longer need to use a feature or service that requires a service-linked role, we recommend that you delete that role. That way you don't have an unused entity that is not actively monitored or maintained. However, you must clean up your service-linked role before you can manually delete it. - -[[service-linked-role-review-before-delete-eks-nodegroups,service-linked-role-review-before-delete-eks-nodegroups.title]] -===== Cleaning up a service-linked role - -Before you can use IAM to delete a service-linked role, you must first delete any resources used by the role. - -[NOTE] -==== - -If the Amazon EKS service is using the role when you try to delete the resources, then the deletion might fail. If that happens, wait for a few minutes and try the operation again. - -==== -. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. -. In the left navigation pane, choose *Clusters*. -. Select the *Compute* tab. -. In the *Node groups* section, choose the node group to delete. -. Type the name of the node group in the deletion confirmation window, and then choose *Delete*. -. Repeat this procedure for any other node groups in the cluster. Wait for all of the delete operations to finish. - - -[[slr-manual-delete-eks-nodegroups,slr-manual-delete-eks-nodegroups.title]] -===== Manually delete the service-linked role - -Use the IAM console, the {aws} CLI, or the {aws} API to delete the `AWSServiceRoleForAmazonEKSNodegroup` service-linked role. For more information, see link:IAM/latest/UserGuide/using-service-linked-roles.html#delete-service-linked-role[Deleting a service-linked role,type="documentation"] in the _IAM User Guide_. - -[[slr-regions-eks-nodegroups,slr-regions-eks-nodegroups.title]] -==== Supported regions for Amazon EKS service-linked roles - -Amazon EKS supports using service-linked roles in all of the regions where the service is available. For more information, see link:general/latest/gr/eks.html[Amazon EKS endpoints and quotas,type="documentation"]. - -[.topic] -[[using-service-linked-roles-eks-fargate,using-service-linked-roles-eks-fargate.title]] -=== Using roles for Amazon EKS Fargate profiles - -[abstract] --- -How to use service-linked roles to give Amazon EKS access to resources in your {aws} account. --- - -Amazon EKS uses {aws} Identity and Access Management (IAM) link:IAM/latest/UserGuide/id_roles.html#iam-term-service-linked-role[service-linked roles,type="documentation"]. A service-linked role is a unique type of IAM role that is linked directly to Amazon EKS. Service-linked roles are predefined by Amazon EKS and include all the permissions that the service requires to call other {aws} services on your behalf. - -A service-linked role makes setting up Amazon EKS easier because you don't have to manually add the necessary permissions. Amazon EKS defines the permissions of its service-linked roles, and unless defined otherwise, only Amazon EKS can assume its roles. The defined permissions include the trust policy and the permissions policy, and that permissions policy cannot be attached to any other IAM entity. - -You can delete a service-linked role only after first deleting their related resources. This protects your Amazon EKS resources because you can't inadvertently remove permission to access the resources. - -For information about other services that support service-linked roles, see link:IAM/latest/UserGuide/reference_aws-services-that-work-with-iam.html[{aws} services that work with IAM,type="documentation"] and look for the services that have *Yes* in the *Service-linked role* column. Choose a *Yes* with a link to view the service-linked role documentation for that service. - -[[service-linked-role-permissions-eks-fargate,service-linked-role-permissions-eks-fargate.title]] -==== Service-linked role permissions for Amazon EKS - -Amazon EKS uses the service-linked role named `AWSServiceRoleForAmazonEKSForFargate`. The role allows Amazon EKS Fargate to configure VPC networking required for Fargate [.noloc]`Pods`. The attached policies allow the role to create and delete elastic network interfaces and describe elastic network Interfaces and resources. - -The `AWSServiceRoleForAmazonEKSForFargate` service-linked role trusts the following services to assume the role: - -* `eks-fargate.amazonaws.com` - -The role permissions policy allows Amazon EKS to complete the following actions on the specified resources: - -* link:aws-managed-policy/latest/reference/AmazonEKSForFargateServiceRolePolicy.html[AmazonEKSForFargateServiceRolePolicy,type="documentation"] - -You must configure permissions to allow an IAM entity (such as a user, group, or role) to create, edit, or delete a service-linked role. For more information, see link:IAM/latest/UserGuide/using-service-linked-roles.html#service-linked-role-permissions[Service-linked role permissions,type="documentation"] in the _IAM User Guide_. - -[[create-service-linked-role-eks-fargate,create-service-linked-role-eks-fargate.title]] -==== Creating a service-linked role for Amazon EKS - -You don't need to manually create a service-linked role. When you create a Fargate profile in the {aws-management-console}, the {aws} CLI, or the {aws} API, Amazon EKS creates the service-linked role for you. - -[IMPORTANT] -==== - -This service-linked role can appear in your account if you completed an action in another service that uses the features supported by this role. If you were using the Amazon EKS service before December 13, 2019, when it began supporting service-linked roles, then Amazon EKS created the AWSServiceRoleForAmazonEKSForFargate role in your account. To learn more, see link:IAM/latest/UserGuide/troubleshoot_roles.html#troubleshoot_roles_new-role-appeared[A New role appeared in my IAM account,type="documentation"]. - -==== - -[[create-service-linked-role-service-api-eks-fargate,create-service-linked-role-service-api-eks-fargate.title]] -===== Creating a service-linked role in Amazon EKS ({aws} API) - -You don't need to manually create a service-linked role. When you create a Fargate profile in the {aws-management-console}, the {aws} CLI, or the {aws} API, Amazon EKS creates the service-linked role for you. - -If you delete this service-linked role, and then need to create it again, you can use the same process to recreate the role in your account. When you create another managed node group, Amazon EKS creates the service-linked role for you again. - -[[edit-service-linked-role-eks-fargate,edit-service-linked-role-eks-fargate.title]] -==== Editing a service-linked role for Amazon EKS - -Amazon EKS does not allow you to edit the `AWSServiceRoleForAmazonEKSForFargate` service-linked role. After you create a service-linked role, you cannot change the name of the role because various entities might reference the role. However, you can edit the description of the role using IAM. For more information, see link:IAM/latest/UserGuide/using-service-linked-roles.html#edit-service-linked-role[Editing a service-linked role,type="documentation"] in the _IAM User Guide_. - -[[delete-service-linked-role-eks-fargate,delete-service-linked-role-eks-fargate.title]] -==== Deleting a service-linked role for Amazon EKS - -If you no longer need to use a feature or service that requires a service-linked role, we recommend that you delete that role. That way you don't have an unused entity that is not actively monitored or maintained. However, you must clean up your service-linked role before you can manually delete it. - -[[service-linked-role-review-before-delete-eks-fargate,service-linked-role-review-before-delete-eks-fargate.title]] -===== Cleaning up a service-linked role - -Before you can use IAM to delete a service-linked role, you must first delete any resources used by the role. - -[NOTE] -==== - -If the Amazon EKS service is using the role when you try to delete the resources, then the deletion might fail. If that happens, wait for a few minutes and try the operation again. - -==== -. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. -. In the left navigation pane, choose *Clusters*. -. On the *Clusters* page, select your cluster. -. Select the *Compute* tab. -. If there are any Fargate profiles in the *Fargate profiles* section, select each one individually, and then choose *Delete*. -. Type the name of the profile in the deletion confirmation window, and then choose *Delete*. -. Repeat this procedure for any other Fargate profiles in the cluster and for any other clusters in your account. - - -[[slr-manual-delete-eks-fargate,slr-manual-delete-eks-fargate.title]] -===== Manually delete the service-linked role - -Use the IAM console, the {aws} CLI, or the {aws} API to delete the AWSServiceRoleForAmazonEKSForFargate service-linked role. For more information, see link:IAM/latest/UserGuide/using-service-linked-roles.html#delete-service-linked-role[Deleting a service-linked role,type="documentation"] in the _IAM User Guide_. - -[[slr-regions-eks-fargate,slr-regions-eks-fargate.title]] -==== Supported regions for Amazon EKS service-linked roles - -Amazon EKS supports using service-linked roles in all of the regions where the service is available. For more information, see link:general/latest/gr/eks.html[Amazon EKS endpoints and quotas,type="documentation"]. - -[.topic] -[[using-service-linked-roles-eks-connector,using-service-linked-roles-eks-connector.title]] -=== Using roles to connect a [.noloc]`Kubernetes` cluster to Amazon EKS - -[abstract] --- -How to use service-linked roles to give Amazon EKS access to resources in your {aws} account. --- - -Amazon EKS uses {aws} Identity and Access Management (IAM) link:IAM/latest/UserGuide/id_roles.html#iam-term-service-linked-role[service-linked roles,type="documentation"]. A service-linked role is a unique type of IAM role that is linked directly to Amazon EKS. Service-linked roles are predefined by Amazon EKS and include all the permissions that the service requires to call other {aws} services on your behalf. - -A service-linked role makes setting up Amazon EKS easier because you don't have to manually add the necessary permissions. Amazon EKS defines the permissions of its service-linked roles, and unless defined otherwise, only Amazon EKS can assume its roles. The defined permissions include the trust policy and the permissions policy, and that permissions policy cannot be attached to any other IAM entity. - -You can delete a service-linked role only after first deleting their related resources. This protects your Amazon EKS resources because you can't inadvertently remove permission to access the resources. - -For information about other services that support service-linked roles, see link:IAM/latest/UserGuide/reference_aws-services-that-work-with-iam.html[{aws} services that work with IAM,type="documentation"] and look for the services that have *Yes* in the *Service-linked role* column. Choose a *Yes* with a link to view the service-linked role documentation for that service. - -[[service-linked-role-permissions-eks-connector,service-linked-role-permissions-eks-connector.title]] -==== Service-linked role permissions for Amazon EKS - -Amazon EKS uses the service-linked role named `AWSServiceRoleForAmazonEKSConnector`. The role allows Amazon EKS to connect [.noloc]`Kubernetes` clusters. The attached policies allow the role to manage necessary resources to connect to your registered [.noloc]`Kubernetes` cluster. - -The `AWSServiceRoleForAmazonEKSConnector` service-linked role trusts the following services to assume the role: - -* `eks-connector.amazonaws.com` - -The role permissions policy allows Amazon EKS to complete the following actions on the specified resources: - -* link:aws-managed-policy/latest/reference/AmazonEKSConnectorServiceRolePolicy.html[AmazonEKSConnectorServiceRolePolicy,type="documentation"] - -You must configure permissions to allow an IAM entity (such as a user, group, or role) to create, edit, or delete a service-linked role. For more information, see link:IAM/latest/UserGuide/using-service-linked-roles.html#service-linked-role-permissions[Service-linked role permissions,type="documentation"] in the _IAM User Guide_. - -[[create-service-linked-role-eks-connector,create-service-linked-role-eks-connector.title]] -==== Creating a service-linked role for Amazon EKS - -You don't need to manually create a service-linked role to connect a cluster. When you connect a cluster in the {aws-management-console}, the {aws} CLI, `eksctl`, or the {aws} API, Amazon EKS creates the service-linked role for you. - -If you delete this service-linked role, and then need to create it again, you can use the same process to recreate the role in your account. When you connect a cluster, Amazon EKS creates the service-linked role for you again. - -[[edit-service-linked-role-eks-connector,edit-service-linked-role-eks-connector.title]] -==== Editing a service-linked role for Amazon EKS - -Amazon EKS does not allow you to edit the `AWSServiceRoleForAmazonEKSConnector` service-linked role. After you create a service-linked role, you cannot change the name of the role because various entities might reference the role. However, you can edit the description of the role using IAM. For more information, see link:IAM/latest/UserGuide/using-service-linked-roles.html#edit-service-linked-role[Editing a service-linked role,type="documentation"] in the _IAM User Guide_. - -[[delete-service-linked-role-eks-connector,delete-service-linked-role-eks-connector.title]] -==== Deleting a service-linked role for Amazon EKS - -If you no longer need to use a feature or service that requires a service-linked role, we recommend that you delete that role. That way you don't have an unused entity that is not actively monitored or maintained. However, you must clean up your service-linked role before you can manually delete it. - -[[service-linked-role-review-before-delete-eks-connector,service-linked-role-review-before-delete-eks-connector.title]] -===== Cleaning up a service-linked role - -Before you can use IAM to delete a service-linked role, you must first delete any resources used by the role. - -[NOTE] -==== - -If the Amazon EKS service is using the role when you try to delete the resources, then the deletion might fail. If that happens, wait for a few minutes and try the operation again. - -==== -. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. -. In the left navigation pane, choose *Clusters*. -. On the *Clusters* page, select your cluster. -. Select the *Deregister* tab and then select the *Ok* tab. - - -[[slr-manual-delete-eks-connector,slr-manual-delete-eks-connector.title]] -===== Manually delete the service-linked role - -Use the IAM console, the {aws} CLI, or the {aws} API to delete the AWSServiceRoleForAmazonEKSConnector service-linked role. For more information, see link:IAM/latest/UserGuide/using-service-linked-roles.html#delete-service-linked-role[Deleting a service-linked role,type="documentation"] in the _IAM User Guide_. - -[.topic] -[[using-service-linked-roles-eks-outpost,using-service-linked-roles-eks-outpost.title]] -=== Using roles for Amazon EKS local clusters on Outpost - -[abstract] --- -How to use service-linked roles to give Amazon EKS access to resources in your {aws} account. --- - -Amazon Elastic Kubernetes Service uses {aws} Identity and Access Management (IAM) link:IAM/latest/UserGuide/id_roles.html#iam-term-service-linked-role[service-linked roles,type="documentation"]. A service-linked role is a unique type of IAM role that is linked directly to Amazon EKS. Service-linked roles are predefined by Amazon EKS and include all the permissions that the service requires to call other {aws} services on your behalf. - -A service-linked role makes setting up Amazon EKS easier because you don't have to manually add the necessary permissions. Amazon EKS defines the permissions of its service-linked roles, and unless defined otherwise, only Amazon EKS can assume its roles. The defined permissions include the trust policy and the permissions policy, and that permissions policy cannot be attached to any other IAM entity. - -You can delete a service-linked role only after first deleting their related resources. This protects your Amazon EKS resources because you can't inadvertently remove permission to access the resources. - -For information about other services that support service-linked roles, see link:IAM/latest/UserGuide/reference_aws-services-that-work-with-iam.html[{aws} services that work with IAM,type="documentation"] and look for the services that have *Yes* in the *Service-linked role* column. Choose a *Yes* with a link to view the service-linked role documentation for that service. - -[[service-linked-role-permissions,service-linked-role-permissions.title]] -==== Service-linked role permissions for Amazon EKS - -Amazon EKS uses the service-linked role named `AWSServiceRoleForAmazonEKSLocalOutpost`. The role allows Amazon EKS to manage local clusters in your account. The attached policies allow the role to manage the following resources: network interfaces, security groups, logs, and Amazon EC2 instances. - -[NOTE] -==== - -The `AWSServiceRoleForAmazonEKSLocalOutpost` service-linked role is distinct from the role required for cluster creation. For more information, see <>. - -==== - -The `AWSServiceRoleForAmazonEKSLocalOutpost` service-linked role trusts the following services to assume the role: - - - -* `outposts.eks-local.amazonaws.com` - -The role permissions policy allows Amazon EKS to complete the following actions on the specified resources: - - - -* link:aws-managed-policy/latest/reference/AmazonEKSServiceRolePolicy.html[AmazonEKSServiceRolePolicy,type="documentation"] - -You must configure permissions to allow an IAM entity (such as a user, group, or role) to create, edit, or delete a service-linked role. For more information, see link:IAM/latest/UserGuide/using-service-linked-roles.html#service-linked-role-permissions[Service-linked role permissions,type="documentation"] in the _IAM User Guide_. - -[[create-service-linked-role-eks-outpost,create-service-linked-role-eks-outpost.title]] -==== Creating a service-linked role for Amazon EKS - -You don't need to manually create a service-linked role. When you create a cluster in the {aws-management-console}, the {aws} CLI, or the {aws} API, Amazon EKS creates the service-linked role for you. - -If you delete this service-linked role, and then need to create it again, you can use the same process to recreate the role in your account. When you create a cluster, Amazon EKS creates the service-linked role for you again. - -[[edit-service-linked-role-eks-outpost,edit-service-linked-role-eks-outpost.title]] -==== Editing a service-linked role for Amazon EKS - -Amazon EKS does not allow you to edit the `AWSServiceRoleForAmazonEKSLocalOutpost` service-linked role. After you create a service-linked role, you can't change the name of the role because various entities might reference the role. However, you can edit the description of the role using IAM. For more information, see link:IAM/latest/UserGuide/using-service-linked-roles.html#edit-service-linked-role[Editing a service-linked role,type="documentation"] in the _IAM User Guide_. - -[[delete-service-linked-role-eks-outpost,delete-service-linked-role-eks-outpost.title]] -==== Deleting a service-linked role for Amazon EKS - -If you no longer need to use a feature or service that requires a service-linked role, we recommend that you delete that role. That way you don't have an unused entity that is not actively monitored or maintained. However, you must clean up your service-linked role before you can manually delete it. - -[[service-linked-role-review-before-delete-eks-outpost,service-linked-role-review-before-delete-eks-outpost.title]] -===== Cleaning up a service-linked role - -Before you can use IAM to delete a service-linked role, you must first delete any resources used by the role. - -[NOTE] -==== - -If the Amazon EKS service is using the role when you try to delete the resources, then the deletion might fail. If that happens, wait for a few minutes and try the operation again. - -==== -. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. -. In the left navigation pane, choose Amazon EKS *Clusters*. -. If your cluster has any node groups or Fargate profiles, you must delete them before you can delete the cluster. For more information, see <> and <>. -. On the *Clusters* page, choose the cluster that you want to delete and choose *Delete*. -. Type the name of the cluster in the deletion confirmation window, and then choose *Delete*. -. Repeat this procedure for any other clusters in your account. Wait for all of the delete operations to finish. - - -[[slr-manual-delete-eks-outpost,slr-manual-delete-eks-outpost.title]] -===== Manually delete the service-linked role - -Use the IAM console, the {aws} CLI, or the {aws} API to delete the `AWSServiceRoleForAmazonEKSLocalOutpost` service-linked role. For more information, see link:IAM/latest/UserGuide/using-service-linked-roles.html#delete-service-linked-role[Deleting a service-linked role,type="documentation"] in the _IAM User Guide_. - -[[slr-regions-eks-connector,slr-regions-eks-connector.title]] -==== Supported regions for Amazon EKS service-linked roles - -Amazon EKS supports using service-linked roles in all of the regions where the service is available. For more information, see link:general/latest/gr/eks.html[Amazon EKS endpoints and quotas,type="documentation"]. - - - - -[.topic] -[[pod-execution-role,pod-execution-role.title]] -== Amazon EKS [.noloc]`Pod` execution IAM role - -The Amazon EKS [.noloc]`Pod` execution role is required to run [.noloc]`Pods` on {aws} Fargate infrastructure. - -When your cluster creates [.noloc]`Pods` on {aws} Fargate infrastructure, the components running on the Fargate infrastructure must make calls to {aws} APIs on your behalf. This is so that they can do actions such as pull container images from Amazon ECR or route logs to other {aws} services. The Amazon EKS [.noloc]`Pod` execution role provides the IAM permissions to do this. - -When you create a Fargate profile, you must specify a [.noloc]`Pod` execution role for the Amazon EKS components that run on the Fargate infrastructure using the profile. This role is added to the cluster's [.noloc]`Kubernetes` https://kubernetes.io/docs/reference/access-authn-authz/rbac/[Role based access control] (RBAC) for authorization. This allows the `kubelet` that's running on the Fargate infrastructure to register with your Amazon EKS cluster so that it can appear in your cluster as a node. - -[NOTE] -==== - -The Fargate profile must have a different IAM role than Amazon EC2 node groups. - -==== - -[IMPORTANT] -==== - -The containers running in the Fargate [.noloc]`Pod` can't assume the IAM permissions associated with a [.noloc]`Pod` execution role. To give the containers in your Fargate [.noloc]`Pod` permissions to access other {aws} services, you must use <>. - -==== - -Before you create a Fargate profile, you must create an IAM role with the link:aws-managed-policy/latest/reference/AmazonEKSFargatePodExecutionRolePolicy.html[AmazonEKSFargatePodExecutionRolePolicy,type="documentation"]. -[[check-pod-execution-role,check-pod-execution-role.title]] -=== Check for a correctly configured existing [.noloc]`Pod` execution role - -You can use the following procedure to check and see if your account already has a correctly configured Amazon EKS [.noloc]`Pod` execution role. To avoid a confused deputy security problem, it's important that the role restricts access based on `SourceArn`. You can modify the execution role as needed to include support for Fargate profiles on other clusters. - -. Open the IAM console at https://console.aws.amazon.com/iam/. -. In the left navigation pane, choose *Roles*. -. On the *Roles* page, search the list of roles for *AmazonEKSFargatePodExecutionRole*. If the role doesn't exist, see <> to create the role. If the role does exist, choose the role. -. On the *AmazonEKSFargatePodExecutionRole* page, do the following: -+ -.. Choose *Permissions*. -.. Ensure that the *AmazonEKSFargatePodExecutionRolePolicy* Amazon managed policy is attached to the role. -.. Choose *Trust relationships*. -.. Choose *Edit trust policy*. -. On the *Edit trust policy* page, verify that the trust relationship contains the following policy and has a line for Fargate profiles on your cluster. If so, choose *Cancel*. -+ -[source,json,subs="verbatim,attributes"] ----- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Condition": { - "ArnLike": { - "aws:SourceArn": "{arn-aws}eks:region-code:111122223333:fargateprofile/my-cluster/*" - } - }, - "Principal": { - "Service": "eks-fargate-pods.amazonaws.com" - }, - "Action": "sts:AssumeRole" - } - ] -} ----- -+ -If the policy matches but doesn't have a line specifying the Fargate profiles on your cluster, you can add the following line at the top of the `ArnLike` object. Replace [.replaceable]`region-code` with the {aws} Region that your cluster is in, [.replaceable]`111122223333` with your account ID, and [.replaceable]`my-cluster` with the name of your cluster. -+ -[source,json,subs="verbatim,attributes"] ----- -"aws:SourceArn": "{arn-aws}eks:region-code:111122223333:fargateprofile/my-cluster/*", ----- -+ -If the policy doesn't match, copy the full previous policy into the form and choose *Update policy*. Replace [.replaceable]`region-code` with the {aws} Region that your cluster is in. If you want to use the same role in all {aws} Regions in your account, replace [.replaceable]`region-code` with `{asterisk}`. Replace [.replaceable]`111122223333` with your account ID and [.replaceable]`my-cluster` with the name of your cluster. If you want to use the same role for all clusters in your account, replace [.replaceable]`my-cluster` with `{asterisk}`. - - -[[create-pod-execution-role,create-pod-execution-role.title]] -=== Creating the Amazon EKS [.noloc]`Pod` execution role - -If you don't already have the Amazon EKS [.noloc]`Pod` execution role for your cluster, you can use the {aws-management-console} or the {aws} CLI to create it. - - - -{aws-management-console}:: -.. Open the IAM console at https://console.aws.amazon.com/iam/. -.. In the left navigation pane, choose *Roles*. -.. On the *Roles* page, choose *Create role*. -.. On the *Select trusted entity* page, do the following: -+ -... In the *Trusted entity type* section, choose *{aws} service*. -... From the *Use cases for other {aws} services* dropdown list, choose *EKS*. -... Choose *EKS - Fargate [.noloc]`Pod`*. -... Choose *Next*. -.. On the *Add permissions* page, choose *Next*. -.. On the *Name, review, and create* page, do the following: -+ -... For *Role name*, enter a unique name for your role, such as `AmazonEKSFargatePodExecutionRole`. -... Under *Add tags (Optional)*, add metadata to the role by attaching tags as key-value pairs. For more information about using tags in IAM, see link:IAM/latest/UserGuide/id_tags.html[Tagging IAM resources,type="documentation"] in the _IAM User Guide_. -... Choose *Create role*. -.. On the *Roles* page, search the list of roles for *AmazonEKSFargatePodExecutionRole*. Choose the role. -.. On the *AmazonEKSFargatePodExecutionRole* page, do the following: -+ -... Choose *Trust relationships*. -... Choose *Edit trust policy*. -.. On the *Edit trust policy* page, do the following: -+ -... Copy and paste the following contents into the *Edit trust policy* form. Replace [.replaceable]`region-code` with the {aws} Region that your cluster is in. If you want to use the same role in all {aws} Regions in your account, replace [.replaceable]`region-code` with `{asterisk}`. Replace [.replaceable]`111122223333` with your account ID and [.replaceable]`my-cluster` with the name of your cluster. If you want to use the same role for all clusters in your account, replace [.replaceable]`my-cluster` with `{asterisk}`. -+ -[source,json,subs="verbatim,attributes"] ----- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Condition": { - "ArnLike": { - "aws:SourceArn": "{arn-aws}eks:region-code:111122223333:fargateprofile/my-cluster/*" - } - }, - "Principal": { - "Service": "eks-fargate-pods.amazonaws.com" - }, - "Action": "sts:AssumeRole" - } - ] -} ----- -... Choose *Update policy*. - - -{aws} CLI:: -.. Copy and paste the following contents to a file named `pod-execution-role-trust-policy.json`. Replace [.replaceable]`region-code` with the {aws} Region that your cluster is in. If you want to use the same role in all {aws} Regions in your account, replace [.replaceable]`region-code` with `{asterisk}`. Replace [.replaceable]`111122223333` with your account ID and [.replaceable]`my-cluster` with the name of your cluster. If you want to use the same role for all clusters in your account, replace [.replaceable]`my-cluster` with `{asterisk}`. -+ -[source,json,subs="verbatim,attributes"] ----- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Condition": { - "ArnLike": { - "aws:SourceArn": "{arn-aws}eks:region-code:111122223333:fargateprofile/my-cluster/*" - } - }, - "Principal": { - "Service": "eks-fargate-pods.amazonaws.com" - }, - "Action": "sts:AssumeRole" - } - ] -} ----- -.. Create a [.noloc]`Pod` execution IAM role. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws iam create-role \ - --role-name AmazonEKSFargatePodExecutionRole \ - --assume-role-policy-document file://"pod-execution-role-trust-policy.json" ----- -.. Attach the required Amazon EKS managed IAM policy to the role. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws iam attach-role-policy \ - --policy-arn {arn-aws}iam::aws:policy/AmazonEKSFargatePodExecutionRolePolicy \ - --role-name AmazonEKSFargatePodExecutionRole ----- - - -[.topic] -[[connector-iam-role,connector-iam-role.title]] -== Amazon EKS connector IAM role - -You can connect [.noloc]`Kubernetes` clusters to view them in your {aws-management-console}. To connect to a [.noloc]`Kubernetes` cluster, create an IAM role. - -[[check-connector-role,check-connector-role.title]] -=== Check for an existing EKS connector role - -You can use the following procedure to check and see if your account already has the Amazon EKS connector role. - -. Open the IAM console at https://console.aws.amazon.com/iam/. -. In the left navigation pane, choose *Roles*. -. Search the list of roles for `AmazonEKSConnectorAgentRole`. If a role that includes `AmazonEKSConnectorAgentRole` doesn't exist, then see <> to create the role. If a role that includes `AmazonEKSConnectorAgentRole` does exist, then select the role to view the attached policies. -. Choose *Permissions*. -. Ensure that the *AmazonEKSConnectorAgentPolicy* managed policy is attached to the role. If the policy is attached, your Amazon EKS connector role is properly configured. -. Choose *Trust relationships*, and then choose *Edit trust policy*. -. Verify that the trust relationship contains the following policy. If the trust relationship matches the following policy, choose *Cancel*. If the trust relationship doesn't match, copy the policy into the *Edit trust policy* window and choose *Update policy*. -+ -[source,json,subs="verbatim,attributes"] ----- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Service": [ - "ssm.amazonaws.com" - ] - }, - "Action": "sts:AssumeRole" - } - ] -} ----- - - -[[create-connector-role,create-connector-role.title]] -=== Creating the Amazon EKS connector agent role - -You can use the {aws-management-console} or {aws} CloudFormation to create the connector agent role. - -{aws} CLI:: -.. Create a file named `eks-connector-agent-trust-policy.json` that contains the following JSON to use for the IAM role. -+ -[source,json,subs="verbatim,attributes"] ----- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Service": [ - "ssm.amazonaws.com" - ] - }, - "Action": "sts:AssumeRole" - } - ] -} ----- -.. Create a file named `eks-connector-agent-policy.json` that contains the following JSON to use for the IAM role. -+ -[source,json,subs="verbatim,attributes"] ----- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "SsmControlChannel", - "Effect": "Allow", - "Action": [ - "ssmmessages:CreateControlChannel" - ], - "Resource": "{arn-aws}eks:*:*:cluster/*" - }, - { - "Sid": "ssmDataplaneOperations", - "Effect": "Allow", - "Action": [ - "ssmmessages:CreateDataChannel", - "ssmmessages:OpenDataChannel", - "ssmmessages:OpenControlChannel" - ], - "Resource": "*" - } - ] -} ----- -.. Create the Amazon EKS Connector agent role using the trust policy and policy you created in the previous list items. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws iam create-role \ - --role-name AmazonEKSConnectorAgentRole \ - --assume-role-policy-document file://eks-connector-agent-trust-policy.json ----- -.. Attach the policy to your Amazon EKS Connector agent role. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws iam put-role-policy \ - --role-name AmazonEKSConnectorAgentRole \ - --policy-name AmazonEKSConnectorAgentPolicy \ - --policy-document file://eks-connector-agent-policy.json ----- - - -{aws} CloudFormation:: -.. Save the following {aws} CloudFormation template to a text file on your local system. -+ -NOTE: This template also creates the service-linked role that would otherwise be created when the `registerCluster` API is called. See <> for details. -+ -[source,yaml,subs="verbatim,attributes"] ----- ---- -AWSTemplateFormatVersion: '2010-09-09' -Description: 'Provisions necessary resources needed to register clusters in EKS' -Parameters: {} -Resources: - EKSConnectorSLR: - Type: {aws}::IAM::ServiceLinkedRole - Properties: - AWSServiceName: eks-connector.amazonaws.com - - EKSConnectorAgentRole: - Type: {aws}::IAM::Role - Properties: - AssumeRolePolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Action: [ 'sts:AssumeRole' ] - Principal: - Service: 'ssm.amazonaws.com' +When multiple types of policies apply to a request, the resulting permissions are more complicated to understand. To learn how {aws} determines whether to allow a request when multiple policy types are involved, see link:IAM/latest/UserGuide/reference_policies_evaluation-logic.html[Policy evaluation logic,type="documentation"] in the _IAM User Guide_. - EKSConnectorAgentPolicy: - Type: {aws}::IAM::Policy - Properties: - PolicyName: EKSConnectorAgentPolicy - Roles: - - {Ref: 'EKSConnectorAgentRole'} - PolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: 'Allow' - Action: [ 'ssmmessages:CreateControlChannel' ] - Resource: - - Fn::Sub: 'arn:${{aws}::Partition}:eks:*:*:cluster/*' - - Effect: 'Allow' - Action: [ 'ssmmessages:CreateDataChannel', 'ssmmessages:OpenDataChannel', 'ssmmessages:OpenControlChannel' ] - Resource: "*" -Outputs: - EKSConnectorAgentRoleArn: - Description: The agent role that EKS connector uses to communicate with {aws} services. - Value: !GetAtt EKSConnectorAgentRole.Arn ----- -.. Open the link:cloudformation/[{aws} CloudFormation console,type="console"]. -.. Choose *Create stack* with new resources (standard). -.. For *Specify template*, select *Upload a template file*, and then choose *Choose file*. -.. Choose the file you created earlier, and then choose *Next*. -.. For *Stack name*, enter a name for your role, such as `eksConnectorAgentRole`, and then choose *Next*. -.. On the *Configure stack options* page, choose *Next*. -.. On the *Review* page, review your information, acknowledge that the stack might create IAM resources, and then choose *Create stack*. diff --git a/latest/ug/security/iam-reference/using-service-linked-roles-eks-connector.adoc b/latest/ug/security/iam-reference/using-service-linked-roles-eks-connector.adoc new file mode 100644 index 000000000..7248c5b47 --- /dev/null +++ b/latest/ug/security/iam-reference/using-service-linked-roles-eks-connector.adoc @@ -0,0 +1,75 @@ +include::../../attributes.txt[] + +[.topic] +[#using-service-linked-roles-eks-connector] += Using roles to connect a Kubernetes cluster to Amazon EKS +:info_titleabbrev: Cluster connector role + +[abstract] +-- +How to use service-linked roles to give Amazon EKS access to resources in your {aws} account. +-- + +Amazon EKS uses {aws} Identity and Access Management (IAM) link:IAM/latest/UserGuide/id_roles.html#iam-term-service-linked-role[service-linked roles,type="documentation"]. A service-linked role is a unique type of IAM role that is linked directly to Amazon EKS. Service-linked roles are predefined by Amazon EKS and include all the permissions that the service requires to call other {aws} services on your behalf. + +A service-linked role makes setting up Amazon EKS easier because you don't have to manually add the necessary permissions. Amazon EKS defines the permissions of its service-linked roles, and unless defined otherwise, only Amazon EKS can assume its roles. The defined permissions include the trust policy and the permissions policy, and that permissions policy cannot be attached to any other IAM entity. + +You can delete a service-linked role only after first deleting their related resources. This protects your Amazon EKS resources because you can't inadvertently remove permission to access the resources. + +For information about other services that support service-linked roles, see link:IAM/latest/UserGuide/reference_aws-services-that-work-with-iam.html[{aws} services that work with IAM,type="documentation"] and look for the services that have *Yes* in the *Service-linked role* column. Choose a *Yes* with a link to view the service-linked role documentation for that service. + +[#service-linked-role-permissions-eks-connector] +== Service-linked role permissions for Amazon EKS + +Amazon EKS uses the service-linked role named `AWSServiceRoleForAmazonEKSConnector`. The role allows Amazon EKS to connect Kubernetes clusters. The attached policies allow the role to manage necessary resources to connect to your registered Kubernetes cluster. + +The `AWSServiceRoleForAmazonEKSConnector` service-linked role trusts the following services to assume the role: + +* `eks-connector.amazonaws.com` + +The role permissions policy allows Amazon EKS to complete the following actions on the specified resources: + +* link:aws-managed-policy/latest/reference/AmazonEKSConnectorServiceRolePolicy.html[AmazonEKSConnectorServiceRolePolicy,type="documentation"] + +You must configure permissions to allow an IAM entity (such as a user, group, or role) to create, edit, or delete a service-linked role. For more information, see link:IAM/latest/UserGuide/using-service-linked-roles.html#service-linked-role-permissions[Service-linked role permissions,type="documentation"] in the _IAM User Guide_. + +This role uses SSM (Systems Manager) permissions to establish secure connections and manage connected Kubernetes clusters. + +[#create-service-linked-role-eks-connector] +== Creating a service-linked role for Amazon EKS + +You don't need to manually create a service-linked role to connect a cluster. When you connect a cluster in the {aws-management-console}, the {aws} CLI, `eksctl`, or the {aws} API, Amazon EKS creates the service-linked role for you. + +If you delete this service-linked role, and then need to create it again, you can use the same process to recreate the role in your account. When you connect a cluster, Amazon EKS creates the service-linked role for you again. + +[#edit-service-linked-role-eks-connector] +== Editing a service-linked role for Amazon EKS + +Amazon EKS does not allow you to edit the `AWSServiceRoleForAmazonEKSConnector` service-linked role. After you create a service-linked role, you cannot change the name of the role because various entities might reference the role. However, you can edit the description of the role using IAM. For more information, see link:IAM/latest/UserGuide/using-service-linked-roles.html#edit-service-linked-role[Editing a service-linked role,type="documentation"] in the _IAM User Guide_. + +[#delete-service-linked-role-eks-connector] +== Deleting a service-linked role for Amazon EKS + +If you no longer need to use a feature or service that requires a service-linked role, we recommend that you delete that role. That way you don't have an unused entity that is not actively monitored or maintained. However, you must clean up your service-linked role before you can manually delete it. + +[#service-linked-role-review-before-delete-eks-connector] +=== Cleaning up a service-linked role + +Before you can use IAM to delete a service-linked role, you must first delete any resources used by the role. + +[NOTE] +==== + +If the Amazon EKS service is using the role when you try to delete the resources, then the deletion might fail. If that happens, wait for a few minutes and try the operation again. + +==== +. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. +. In the left navigation pane, choose *Clusters*. +. On the *Clusters* page, select your cluster. +. Select the *Deregister* tab and then select the *Ok* tab. + + +[#slr-manual-delete-eks-connector] +=== Manually delete the service-linked role + +Use the IAM console, the {aws} CLI, or the {aws} API to delete the AWSServiceRoleForAmazonEKSConnector service-linked role. For more information, see link:IAM/latest/UserGuide/using-service-linked-roles.html#delete-service-linked-role[Deleting a service-linked role,type="documentation"] in the _IAM User Guide_. \ No newline at end of file diff --git a/latest/ug/security/iam-reference/using-service-linked-roles-eks-dashboard.adoc b/latest/ug/security/iam-reference/using-service-linked-roles-eks-dashboard.adoc new file mode 100644 index 000000000..3afdd00e7 --- /dev/null +++ b/latest/ug/security/iam-reference/using-service-linked-roles-eks-dashboard.adoc @@ -0,0 +1,89 @@ +include::../../attributes.txt[] + +[.topic] +[#using-service-linked-roles-eks-dashboard] += Using roles for Amazon EKS Dashboard +:info_titleabbrev: Dashboard role + +[abstract] +-- +How to use service-linked roles to give Amazon EKS access to resources in your {aws} accounts for the Amazon EKS Dashboard. +-- + +The Amazon EKS Dashboard uses this service-linked role to aggregate information about cluster resources from multiple regions and accounts. The dashboard integrates with {aws} Organizations to securely read information about multiple accounts in your organization. + +To learn more about the Amazon EKS Dashboard, see <>. + +== Background + +Amazon EKS uses {aws} Identity and Access Management (IAM) link:IAM/latest/UserGuide/id_roles.html#iam-term-service-linked-role[service-linked roles,type="documentation"]. A service-linked role is a unique type of IAM role that is linked directly to Amazon EKS. Service-linked roles are predefined by Amazon EKS and include all the permissions that the service requires to call other {aws} services on your behalf. + +A service-linked role makes setting up Amazon EKS easier because you don't have to manually add the necessary permissions. Amazon EKS defines the permissions of its service-linked roles, and unless defined otherwise, only Amazon EKS can assume its roles. The defined permissions include the trust policy and the permissions policy, and that permissions policy cannot be attached to any other IAM entity. + +You can delete a service-linked role only after first deleting their related resources. This protects your Amazon EKS resources because you can't inadvertently remove permission to access the resources. + +For information about other services that support service-linked roles, see link:IAM/latest/UserGuide/reference_aws-services-that-work-with-iam.html[{aws} services that work with IAM,type="documentation"] and look for the services that have *Yes* in the *Service-linked role* column. Choose a *Yes* with a link to view the service-linked role documentation for that service. + +[#service-linked-role-permissions-eks-dashboard] +== Service-linked role permissions for Amazon EKS + +Amazon EKS uses the service-linked role named `AWSServiceRoleForAmazonEKSDashboard`. The role allows Amazon EKS to generate and display the EKS Dashboard, including aggregated information about cluster resources. The attached `AmazonEKSDashboardServiceRolePolicy` policy allows the role to manage the following resources: Auto Scaling groups, security groups, launch templates, and IAM instance profiles. For more information, see <>. + +The `AWSServiceRoleForAmazonEKSDashboard` service-linked role trusts the following services to assume the role: + +* `dashboard.eks.amazonaws.com` + +To view the latest version of the JSON policy document, see link:aws-managed-policy/latest/reference/AmazonEKSDashboardServiceRolePolicy.html#AmazonEKSDashboardServiceRolePolicy-json[AmazonEKSDashboardServiceRolePolicy,type="documentation"] in the {aws} Managed Policy Reference Guide. + +You must configure permissions to allow an IAM entity (such as a user, group, or role) to create, edit, or delete a service-linked role. For more information, see link:IAM/latest/UserGuide/using-service-linked-roles.html#service-linked-role-permissions[Service-linked role permissions,type="documentation"] in the _IAM User Guide_. + +[#create-service-linked-role-eks-dashboard] +== Creating a service-linked role for Amazon EKS + +You don't need to manually create a service-linked role. When you enable the dashboard in the {aws} console, Amazon EKS creates the service-linked role for you. + +For more information about enabling the EKS Dashboard, see <>. + +[IMPORTANT] +==== + +This service-linked role can appear in your account if you completed an action in another service that uses the features supported by this role. + +==== + +[#edit-service-linked-role-eks-dashboard] +== Editing a service-linked role for Amazon EKS + +Amazon EKS does not allow you to edit the `AWSServiceRoleForAmazonEKSDashboard` service-linked role. After you create a service-linked role, you cannot change the name of the role because various entities might reference the role. However, you can edit the description of the role using IAM. For more information, see link:IAM/latest/UserGuide/using-service-linked-roles.html#edit-service-linked-role[Editing a service-linked role,type="documentation"] in the _IAM User Guide_. + +[#delete-service-linked-role-eks-dashboard] +== Deleting a service-linked role for Amazon EKS + +If you no longer need to use a feature or service that requires a service-linked role, we recommend that you delete that role. That way you don't have an unused entity that is not actively monitored or maintained. However, you must clean up your service-linked role before you can manually delete it. + + + +[#service-linked-role-review-before-delete-eks-dashboard] +=== Cleaning up a service-linked role + +Before you can use IAM to delete a service-linked role, you must first delete any resources used by the role. + +[NOTE] +==== + +If the Amazon EKS service is using the role when you try to delete the resources, then the deletion might fail. If that happens, wait for a few minutes and try the operation again. + +==== + +For more information about disabling the EKS Dashboard, see <>. + + +[#slr-manual-delete-eks-dashboard] +=== Manually delete the service-linked role + +Use the IAM console, the {aws} CLI, or the {aws} API to delete the `AWSServiceRoleForAmazonEKSDashboard` service-linked role. For more information, see link:IAM/latest/UserGuide/using-service-linked-roles.html#delete-service-linked-role[Deleting a service-linked role,type="documentation"] in the _IAM User Guide_. + +[#slr-regions-eks-dashboard] +== Supported regions for Amazon EKS service-linked roles + +Amazon EKS supports using service-linked roles in all of the regions where the service is available. For more information, see link:general/latest/gr/eks.html[Amazon EKS endpoints and quotas,type="documentation"]. \ No newline at end of file diff --git a/latest/ug/security/iam-reference/using-service-linked-roles-eks-fargate.adoc b/latest/ug/security/iam-reference/using-service-linked-roles-eks-fargate.adoc new file mode 100644 index 000000000..9989b46df --- /dev/null +++ b/latest/ug/security/iam-reference/using-service-linked-roles-eks-fargate.adoc @@ -0,0 +1,93 @@ +include::../../attributes.txt[] + +[.topic] +[#using-service-linked-roles-eks-fargate] += Using roles for Amazon EKS Fargate profiles +:info_titleabbrev: Fargate profile role + +[abstract] +-- +How to use service-linked roles to give Amazon EKS access to resources in your {aws} account. +-- + +Amazon EKS uses {aws} Identity and Access Management (IAM) link:IAM/latest/UserGuide/id_roles.html#iam-term-service-linked-role[service-linked roles,type="documentation"]. A service-linked role is a unique type of IAM role that is linked directly to Amazon EKS. Service-linked roles are predefined by Amazon EKS and include all the permissions that the service requires to call other {aws} services on your behalf. + +A service-linked role makes setting up Amazon EKS easier because you don't have to manually add the necessary permissions. Amazon EKS defines the permissions of its service-linked roles, and unless defined otherwise, only Amazon EKS can assume its roles. The defined permissions include the trust policy and the permissions policy, and that permissions policy cannot be attached to any other IAM entity. + +You can delete a service-linked role only after first deleting their related resources. This protects your Amazon EKS resources because you can't inadvertently remove permission to access the resources. + +For information about other services that support service-linked roles, see link:IAM/latest/UserGuide/reference_aws-services-that-work-with-iam.html[{aws} services that work with IAM,type="documentation"] and look for the services that have *Yes* in the *Service-linked role* column. Choose a *Yes* with a link to view the service-linked role documentation for that service. + +[#service-linked-role-permissions-eks-fargate] +== Service-linked role permissions for Amazon EKS + +Amazon EKS uses the service-linked role named `AWSServiceRoleForAmazonEKSForFargate`. The role allows Amazon EKS Fargate to configure VPC networking required for Fargate Pods. The attached policies allow the role to create and delete elastic network interfaces and describe elastic network Interfaces and resources. + +The `AWSServiceRoleForAmazonEKSForFargate` service-linked role trusts the following services to assume the role: + +* `eks-fargate.amazonaws.com` + +The role permissions policy allows Amazon EKS to complete the following actions on the specified resources: + +* link:aws-managed-policy/latest/reference/AmazonEKSForFargateServiceRolePolicy.html[AmazonEKSForFargateServiceRolePolicy,type="documentation"] + +You must configure permissions to allow an IAM entity (such as a user, group, or role) to create, edit, or delete a service-linked role. For more information, see link:IAM/latest/UserGuide/using-service-linked-roles.html#service-linked-role-permissions[Service-linked role permissions,type="documentation"] in the _IAM User Guide_. + +[#create-service-linked-role-eks-fargate] +== Creating a service-linked role for Amazon EKS + +You don't need to manually create a service-linked role. When you create a Fargate profile in the {aws-management-console}, the {aws} CLI, or the {aws} API, Amazon EKS creates the service-linked role for you. + +[IMPORTANT] +==== + +This service-linked role can appear in your account if you completed an action in another service that uses the features supported by this role. If you were using the Amazon EKS service before December 13, 2019, when it began supporting service-linked roles, then Amazon EKS created the AWSServiceRoleForAmazonEKSForFargate role in your account. To learn more, see link:IAM/latest/UserGuide/troubleshoot_roles.html#troubleshoot_roles_new-role-appeared[A New role appeared in my IAM account,type="documentation"]. + +==== + +[#create-service-linked-role-service-api-eks-fargate] +=== Creating a service-linked role in Amazon EKS ({aws} API) + +You don't need to manually create a service-linked role. When you create a Fargate profile in the {aws-management-console}, the {aws} CLI, or the {aws} API, Amazon EKS creates the service-linked role for you. + +If you delete this service-linked role, and then need to create it again, you can use the same process to recreate the role in your account. When you create another managed node group, Amazon EKS creates the service-linked role for you again. + +[#edit-service-linked-role-eks-fargate] +== Editing a service-linked role for Amazon EKS + +Amazon EKS does not allow you to edit the `AWSServiceRoleForAmazonEKSForFargate` service-linked role. After you create a service-linked role, you cannot change the name of the role because various entities might reference the role. However, you can edit the description of the role using IAM. For more information, see link:IAM/latest/UserGuide/using-service-linked-roles.html#edit-service-linked-role[Editing a service-linked role,type="documentation"] in the _IAM User Guide_. + +[#delete-service-linked-role-eks-fargate] +== Deleting a service-linked role for Amazon EKS + +If you no longer need to use a feature or service that requires a service-linked role, we recommend that you delete that role. That way you don't have an unused entity that is not actively monitored or maintained. However, you must clean up your service-linked role before you can manually delete it. + +[#service-linked-role-review-before-delete-eks-fargate] +=== Cleaning up a service-linked role + +Before you can use IAM to delete a service-linked role, you must first delete any resources used by the role. + +[NOTE] +==== + +If the Amazon EKS service is using the role when you try to delete the resources, then the deletion might fail. If that happens, wait for a few minutes and try the operation again. + +==== +. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. +. In the left navigation pane, choose *Clusters*. +. On the *Clusters* page, select your cluster. +. Select the *Compute* tab. +. If there are any Fargate profiles in the *Fargate profiles* section, select each one individually, and then choose *Delete*. +. Type the name of the profile in the deletion confirmation window, and then choose *Delete*. +. Repeat this procedure for any other Fargate profiles in the cluster and for any other clusters in your account. + + +[#slr-manual-delete-eks-fargate] +=== Manually delete the service-linked role + +Use the IAM console, the {aws} CLI, or the {aws} API to delete the AWSServiceRoleForAmazonEKSForFargate service-linked role. For more information, see link:IAM/latest/UserGuide/using-service-linked-roles.html#delete-service-linked-role[Deleting a service-linked role,type="documentation"] in the _IAM User Guide_. + +[#slr-regions-eks-fargate] +== Supported regions for Amazon EKS service-linked roles + +Amazon EKS supports using service-linked roles in all of the regions where the service is available. For more information, see link:general/latest/gr/eks.html[Amazon EKS endpoints and quotas,type="documentation"]. \ No newline at end of file diff --git a/latest/ug/security/iam-reference/using-service-linked-roles-eks-nodegroups.adoc b/latest/ug/security/iam-reference/using-service-linked-roles-eks-nodegroups.adoc new file mode 100644 index 000000000..50d9d1ff6 --- /dev/null +++ b/latest/ug/security/iam-reference/using-service-linked-roles-eks-nodegroups.adoc @@ -0,0 +1,92 @@ +include::../../attributes.txt[] + +[.topic] +[#using-service-linked-roles-eks-nodegroups] += Using roles for Amazon EKS node groups +:info_titleabbrev: Node groups role + +[abstract] +-- +How to use service-linked roles to give Amazon EKS access to resources in your {aws} account. +-- + +Amazon EKS uses {aws} Identity and Access Management (IAM) link:IAM/latest/UserGuide/id_roles.html#iam-term-service-linked-role[service-linked roles,type="documentation"]. A service-linked role is a unique type of IAM role that is linked directly to Amazon EKS. Service-linked roles are predefined by Amazon EKS and include all the permissions that the service requires to call other {aws} services on your behalf. + +A service-linked role makes setting up Amazon EKS easier because you don't have to manually add the necessary permissions. Amazon EKS defines the permissions of its service-linked roles, and unless defined otherwise, only Amazon EKS can assume its roles. The defined permissions include the trust policy and the permissions policy, and that permissions policy cannot be attached to any other IAM entity. + +You can delete a service-linked role only after first deleting their related resources. This protects your Amazon EKS resources because you can't inadvertently remove permission to access the resources. + +For information about other services that support service-linked roles, see link:IAM/latest/UserGuide/reference_aws-services-that-work-with-iam.html[{aws} services that work with IAM,type="documentation"] and look for the services that have *Yes* in the *Service-linked role* column. Choose a *Yes* with a link to view the service-linked role documentation for that service. + +[#service-linked-role-permissions-eks-nodegroups] +== Service-linked role permissions for Amazon EKS + +Amazon EKS uses the service-linked role named `AWSServiceRoleForAmazonEKSNodegroup`. The role allows Amazon EKS to manage node groups in your account. The attached `AWSServiceRoleForAmazonEKSNodegroup` policy allows the role to manage the following resources: Auto Scaling groups, security groups, launch templates, and IAM instance profiles. For more information, see <>. + +The `AWSServiceRoleForAmazonEKSNodegroup` service-linked role trusts the following services to assume the role: + +* `eks-nodegroup.amazonaws.com` + +The role permissions policy allows Amazon EKS to complete the following actions on the specified resources: + +* link:aws-managed-policy/latest/reference/AWSServiceRoleForAmazonEKSNodegroup.html[AWSServiceRoleForAmazonEKSNodegroup,type="documentation"] + +You must configure permissions to allow an IAM entity (such as a user, group, or role) to create, edit, or delete a service-linked role. For more information, see link:IAM/latest/UserGuide/using-service-linked-roles.html#service-linked-role-permissions[Service-linked role permissions,type="documentation"] in the _IAM User Guide_. + +[#create-service-linked-role-eks-nodegroups] +== Creating a service-linked role for Amazon EKS + +You don't need to manually create a service-linked role. When you CreateNodegroup in the {aws-management-console}, the {aws} CLI, or the {aws} API, Amazon EKS creates the service-linked role for you. + +[IMPORTANT] +==== + +This service-linked role can appear in your account if you completed an action in another service that uses the features supported by this role. If you were using the Amazon EKS service before January 1, 2017, when it began supporting service-linked roles, then Amazon EKS created the AWSServiceRoleForAmazonEKSNodegroup role in your account. To learn more, see link:IAM/latest/UserGuide/troubleshoot_roles.html#troubleshoot_roles_new-role-appeared[A new role appeared in my IAM account,type="documentation"]. + +==== + +[#create-service-linked-role-service-api-eks-nodegroups] +=== Creating a service-linked role in Amazon EKS ({aws} API) + +You don't need to manually create a service-linked role. When you create a managed node group in the {aws-management-console}, the {aws} CLI, or the {aws} API, Amazon EKS creates the service-linked role for you. + +If you delete this service-linked role, and then need to create it again, you can use the same process to recreate the role in your account. When you create another managed node group, Amazon EKS creates the service-linked role for you again. + +[#edit-service-linked-role-eks-nodegroups] +== Editing a service-linked role for Amazon EKS + +Amazon EKS does not allow you to edit the `AWSServiceRoleForAmazonEKSNodegroup` service-linked role. After you create a service-linked role, you cannot change the name of the role because various entities might reference the role. However, you can edit the description of the role using IAM. For more information, see link:IAM/latest/UserGuide/using-service-linked-roles.html#edit-service-linked-role[Editing a service-linked role,type="documentation"] in the _IAM User Guide_. + +[#delete-service-linked-role-eks-nodegroups] +== Deleting a service-linked role for Amazon EKS + +If you no longer need to use a feature or service that requires a service-linked role, we recommend that you delete that role. That way you don't have an unused entity that is not actively monitored or maintained. However, you must clean up your service-linked role before you can manually delete it. + +[#service-linked-role-review-before-delete-eks-nodegroups] +=== Cleaning up a service-linked role + +Before you can use IAM to delete a service-linked role, you must first delete any resources used by the role. + +[NOTE] +==== + +If the Amazon EKS service is using the role when you try to delete the resources, then the deletion might fail. If that happens, wait for a few minutes and try the operation again. + +==== +. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. +. In the left navigation pane, choose *Clusters*. +. Select the *Compute* tab. +. In the *Node groups* section, choose the node group to delete. +. Type the name of the node group in the deletion confirmation window, and then choose *Delete*. +. Repeat this procedure for any other node groups in the cluster. Wait for all of the delete operations to finish. + + +[#slr-manual-delete-eks-nodegroups] +=== Manually delete the service-linked role + +Use the IAM console, the {aws} CLI, or the {aws} API to delete the `AWSServiceRoleForAmazonEKSNodegroup` service-linked role. For more information, see link:IAM/latest/UserGuide/using-service-linked-roles.html#delete-service-linked-role[Deleting a service-linked role,type="documentation"] in the _IAM User Guide_. + +[#slr-regions-eks-nodegroups] +== Supported regions for Amazon EKS service-linked roles + +Amazon EKS supports using service-linked roles in all of the regions where the service is available. For more information, see link:general/latest/gr/eks.html[Amazon EKS endpoints and quotas,type="documentation"]. \ No newline at end of file diff --git a/latest/ug/security/iam-reference/using-service-linked-roles-eks-outpost.adoc b/latest/ug/security/iam-reference/using-service-linked-roles-eks-outpost.adoc new file mode 100644 index 000000000..4865db464 --- /dev/null +++ b/latest/ug/security/iam-reference/using-service-linked-roles-eks-outpost.adoc @@ -0,0 +1,91 @@ +include::../../attributes.txt[] + +[.topic] +[#using-service-linked-roles-eks-outpost] += Using roles for Amazon EKS local clusters on Outpost +:info_titleabbrev: Local cluster role + +[abstract] +-- +How to use service-linked roles to give Amazon EKS access to resources in your {aws} account. +-- + +Amazon Elastic Kubernetes Service uses {aws} Identity and Access Management (IAM) link:IAM/latest/UserGuide/id_roles.html#iam-term-service-linked-role[service-linked roles,type="documentation"]. A service-linked role is a unique type of IAM role that is linked directly to Amazon EKS. Service-linked roles are predefined by Amazon EKS and include all the permissions that the service requires to call other {aws} services on your behalf. + +A service-linked role makes setting up Amazon EKS easier because you don't have to manually add the necessary permissions. Amazon EKS defines the permissions of its service-linked roles, and unless defined otherwise, only Amazon EKS can assume its roles. The defined permissions include the trust policy and the permissions policy, and that permissions policy cannot be attached to any other IAM entity. + +You can delete a service-linked role only after first deleting their related resources. This protects your Amazon EKS resources because you can't inadvertently remove permission to access the resources. + +For information about other services that support service-linked roles, see link:IAM/latest/UserGuide/reference_aws-services-that-work-with-iam.html[{aws} services that work with IAM,type="documentation"] and look for the services that have *Yes* in the *Service-linked role* column. Choose a *Yes* with a link to view the service-linked role documentation for that service. + +[#service-linked-role-permissions] +== Service-linked role permissions for Amazon EKS + +Amazon EKS uses the service-linked role named `AWSServiceRoleForAmazonEKSLocalOutpost`. The role allows Amazon EKS to manage local clusters in your account. The attached policies allow the role to manage the following resources: network interfaces, security groups, logs, and Amazon EC2 instances. + +[NOTE] +==== + +The `AWSServiceRoleForAmazonEKSLocalOutpost` service-linked role is distinct from the role required for cluster creation. For more information, see <>. + +==== + +The `AWSServiceRoleForAmazonEKSLocalOutpost` service-linked role trusts the following services to assume the role: + + + +* `outposts.eks-local.amazonaws.com` + +The role permissions policy allows Amazon EKS to complete the following actions on the specified resources: + + + +* link:aws-managed-policy/latest/reference/AmazonEKSServiceRolePolicy.html[AmazonEKSServiceRolePolicy,type="documentation"] + +You must configure permissions to allow an IAM entity (such as a user, group, or role) to create, edit, or delete a service-linked role. For more information, see link:IAM/latest/UserGuide/using-service-linked-roles.html#service-linked-role-permissions[Service-linked role permissions,type="documentation"] in the _IAM User Guide_. + +[#create-service-linked-role-eks-outpost] +== Creating a service-linked role for Amazon EKS + +You don't need to manually create a service-linked role. When you create a cluster in the {aws-management-console}, the {aws} CLI, or the {aws} API, Amazon EKS creates the service-linked role for you. + +If you delete this service-linked role, and then need to create it again, you can use the same process to recreate the role in your account. When you create a cluster, Amazon EKS creates the service-linked role for you again. + +[#edit-service-linked-role-eks-outpost] +== Editing a service-linked role for Amazon EKS + +Amazon EKS does not allow you to edit the `AWSServiceRoleForAmazonEKSLocalOutpost` service-linked role. After you create a service-linked role, you can't change the name of the role because various entities might reference the role. However, you can edit the description of the role using IAM. For more information, see link:IAM/latest/UserGuide/using-service-linked-roles.html#edit-service-linked-role[Editing a service-linked role,type="documentation"] in the _IAM User Guide_. + +[#delete-service-linked-role-eks-outpost] +== Deleting a service-linked role for Amazon EKS + +If you no longer need to use a feature or service that requires a service-linked role, we recommend that you delete that role. That way you don't have an unused entity that is not actively monitored or maintained. However, you must clean up your service-linked role before you can manually delete it. + +[#service-linked-role-review-before-delete-eks-outpost] +=== Cleaning up a service-linked role + +Before you can use IAM to delete a service-linked role, you must first delete any resources used by the role. + +[NOTE] +==== + +If the Amazon EKS service is using the role when you try to delete the resources, then the deletion might fail. If that happens, wait for a few minutes and try the operation again. + +==== +. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. +. In the left navigation pane, choose Amazon EKS *Clusters*. +. If your cluster has any node groups or Fargate profiles, you must delete them before you can delete the cluster. For more information, see <> and <>. +. On the *Clusters* page, choose the cluster that you want to delete and choose *Delete*. +. Type the name of the cluster in the deletion confirmation window, and then choose *Delete*. +. Repeat this procedure for any other clusters in your account. Wait for all of the delete operations to finish. + + +[#slr-manual-delete-eks-outpost] +=== Manually delete the service-linked role + +Use the IAM console, the {aws} CLI, or the {aws} API to delete the `AWSServiceRoleForAmazonEKSLocalOutpost` service-linked role. For more information, see link:IAM/latest/UserGuide/using-service-linked-roles.html#delete-service-linked-role[Deleting a service-linked role,type="documentation"] in the _IAM User Guide_. + +[#slr-regions-eks-connector] +== Supported regions for Amazon EKS service-linked roles + +Amazon EKS supports using service-linked roles in all of the regions where the service is available. For more information, see link:general/latest/gr/eks.html[Amazon EKS endpoints and quotas,type="documentation"]. \ No newline at end of file diff --git a/latest/ug/security/iam-reference/using-service-linked-roles-eks.adoc b/latest/ug/security/iam-reference/using-service-linked-roles-eks.adoc new file mode 100644 index 000000000..47bafbf00 --- /dev/null +++ b/latest/ug/security/iam-reference/using-service-linked-roles-eks.adoc @@ -0,0 +1,87 @@ +include::../../attributes.txt[] + +[.topic] +[#using-service-linked-roles-eks] += Using roles for Amazon EKS clusters +:info_titleabbrev: Cluster role + +[abstract] +-- +How to use service-linked roles to give Amazon EKS access to resources in your {aws} account. +-- + +Amazon Elastic Kubernetes Service uses {aws} Identity and Access Management (IAM) link:IAM/latest/UserGuide/id_roles.html#iam-term-service-linked-role[service-linked roles,type="documentation"]. A service-linked role is a unique type of IAM role that is linked directly to Amazon EKS. Service-linked roles are predefined by Amazon EKS and include all the permissions that the service requires to call other {aws} services on your behalf. + +A service-linked role makes setting up Amazon EKS easier because you don't have to manually add the necessary permissions. Amazon EKS defines the permissions of its service-linked roles, and unless defined otherwise, only Amazon EKS can assume its roles. The defined permissions include the trust policy and the permissions policy, and that permissions policy cannot be attached to any other IAM entity. + +You can delete a service-linked role only after first deleting their related resources. This protects your Amazon EKS resources because you can't inadvertently remove permission to access the resources. + +For information about other services that support service-linked roles, see link:IAM/latest/UserGuide/reference_aws-services-that-work-with-iam.html[{aws} services that work with IAM,type="documentation"] and look for the services that have *Yes* in the *Service-linked role* column. Choose a *Yes* with a link to view the service-linked role documentation for that service. + +[#service-linked-role-permissions-eks] +== Service-linked role permissions for Amazon EKS + +Amazon EKS uses the service-linked role named `AWSServiceRoleForAmazonEKS`. The role allows Amazon EKS to manage clusters in your account. The attached policies allow the role to manage the following resources: network interfaces, security groups, logs, and VPCs. + +[NOTE] +==== + +The `AWSServiceRoleForAmazonEKS` service-linked role is distinct from the role required for cluster creation. For more information, see <>. + +==== + +The `AWSServiceRoleForAmazonEKS` service-linked role trusts the following services to assume the role: + +* `eks.amazonaws.com` + +The role permissions policy allows Amazon EKS to complete the following actions on the specified resources: + +* link:aws-managed-policy/latest/reference/AmazonEKSServiceRolePolicy.html[AmazonEKSServiceRolePolicy,type="documentation"] + +You must configure permissions to allow an IAM entity (such as a user, group, or role) to create, edit, or delete a service-linked role. For more information, see link:IAM/latest/UserGuide/using-service-linked-roles.html#service-linked-role-permissions[Service-linked role permissions,type="documentation"] in the _IAM User Guide_. + +[#create-service-linked-role-eks] +== Creating a service-linked role for Amazon EKS + +You don't need to manually create a service-linked role. When you create a cluster in the {aws-management-console}, the {aws} CLI, or the {aws} API, Amazon EKS creates the service-linked role for you. + +If you delete this service-linked role, and then need to create it again, you can use the same process to recreate the role in your account. When you create a cluster, Amazon EKS creates the service-linked role for you again. + +[#edit-service-linked-role-eks] +== Editing a service-linked role for Amazon EKS + +Amazon EKS does not allow you to edit the `AWSServiceRoleForAmazonEKS` service-linked role. After you create a service-linked role, you cannot change the name of the role because various entities might reference the role. However, you can edit the description of the role using IAM. For more information, see link:IAM/latest/UserGuide/using-service-linked-roles.html#edit-service-linked-role[Editing a service-linked role,type="documentation"] in the _IAM User Guide_. + +[#delete-service-linked-role-eks] +== Deleting a service-linked role for Amazon EKS + +If you no longer need to use a feature or service that requires a service-linked role, we recommend that you delete that role. That way you don't have an unused entity that is not actively monitored or maintained. However, you must clean up your service-linked role before you can manually delete it. + +[#service-linked-role-review-before-delete-eks] +=== Cleaning up a service-linked role + +Before you can use IAM to delete a service-linked role, you must first delete any resources used by the role. + +[NOTE] +==== + +If the Amazon EKS service is using the role when you try to delete the resources, then the deletion might fail. If that happens, wait for a few minutes and try the operation again. + +==== +. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. +. In the left navigation pane, choose *Clusters*. +. If your cluster has any node groups or Fargate profiles, you must delete them before you can delete the cluster. For more information, see <> and <>. +. On the *Clusters* page, choose the cluster that you want to delete and choose *Delete*. +. Type the name of the cluster in the deletion confirmation window, and then choose *Delete*. +. Repeat this procedure for any other clusters in your account. Wait for all of the delete operations to finish. + + +[#slr-manual-delete-eks] +=== Manually delete the service-linked role + +Use the IAM console, the {aws} CLI, or the {aws} API to delete the `AWSServiceRoleForAmazonEKS` service-linked role. For more information, see link:IAM/latest/UserGuide/using-service-linked-roles.html#delete-service-linked-role[Deleting a service-linked role,type="documentation"] in the _IAM User Guide_. + +[#slr-regions-eks] +== Supported regions for Amazon EKS service-linked roles + +Amazon EKS supports using service-linked roles in all of the regions where the service is available. For more information, see link:general/latest/gr/eks.html[Amazon EKS endpoints and quotas,type="documentation"]. \ No newline at end of file diff --git a/latest/ug/security/iam-reference/using-service-linked-roles.adoc b/latest/ug/security/iam-reference/using-service-linked-roles.adoc new file mode 100644 index 000000000..45c0df7ce --- /dev/null +++ b/latest/ug/security/iam-reference/using-service-linked-roles.adoc @@ -0,0 +1,29 @@ +include::../../attributes.txt[] + +[.topic] +[#using-service-linked-roles] += Using service-linked roles for Amazon EKS +:info_titleabbrev: Service-linked roles + +[abstract] +-- +How to use service-linked roles to give Amazon EKS access to resources in your {aws} account. +-- + +Amazon Elastic Kubernetes Service uses {aws} Identity and Access Management (IAM) link:IAM/latest/UserGuide/id_roles.html#iam-term-service-linked-role[service-linked roles,type="documentation"]. A service-linked role is a unique type of IAM role that is linked directly to Amazon EKS. Service-linked roles are predefined by Amazon EKS and include all the permissions that the service requires to call other {aws} services on your behalf. + +[.topiclist] +[[Topic List]] + +include::using-service-linked-roles-eks.adoc[leveloffset=+1] + +include::using-service-linked-roles-eks-nodegroups.adoc[leveloffset=+1] + +include::using-service-linked-roles-eks-fargate.adoc[leveloffset=+1] + +include::using-service-linked-roles-eks-connector.adoc[leveloffset=+1] + +include::using-service-linked-roles-eks-outpost.adoc[leveloffset=+1] + +include::using-service-linked-roles-eks-dashboard.adoc[leveloffset=+1] + diff --git a/latest/ug/security/infrastructure-security.adoc b/latest/ug/security/infrastructure-security.adoc new file mode 100644 index 000000000..743f45845 --- /dev/null +++ b/latest/ug/security/infrastructure-security.adoc @@ -0,0 +1,39 @@ +include::../attributes.txt[] + +[.topic] +[#infrastructure-security] += Infrastructure security in Amazon EKS +:info_titleabbrev: Infrastructure security + +[abstract] +-- +Learn how Amazon EKS isolates service traffic. +-- + +As a managed service, Amazon Elastic Kubernetes Service is protected by {aws} global network security. For information about {aws} security services and how {aws} protects infrastructure, see link:security/[{aws} Cloud Security,type="marketing"]. To design your {aws} environment using the best practices for infrastructure security, see link:wellarchitected/latest/security-pillar/infrastructure-protection.html[Infrastructure Protection,type="documentation"] in _Security Pillar {aws} Well‐Architected Framework_. + +You use {aws} published API calls to access Amazon EKS through the network. Clients must support the following: + +* Transport Layer Security (TLS). We require TLS 1.2 and recommend TLS 1.3. +* Cipher suites with perfect forward secrecy (PFS) such as DHE (Ephemeral Diffie-Hellman) or ECDHE (Elliptic Curve Ephemeral Diffie-Hellman). Most modern systems such as Java 7 and later support these modes. + +Additionally, requests must be signed by using an access key ID and a secret access key that is associated with an IAM principal. Or you can use the link:STS/latest/APIReference/welcome.html[{aws} Security Token Service,type="documentation"] ({aws} STS) to generate temporary security credentials to sign requests. + +When you create an Amazon EKS cluster, you specify the VPC subnets for your cluster to use. Amazon EKS requires subnets in at least two Availability Zones. We recommend a VPC with public and private subnets so that Kubernetes can create public load balancers in the public subnets that load balance traffic to Pods running on nodes that are in private subnets. + +For more information about VPC considerations, see <>. + +If you create your VPC and node groups with the {aws} CloudFormation templates provided in the <> walkthrough, then your control plane and node security groups are configured with our recommended settings. + +For more information about security group considerations, see <>. + +When you create a new cluster, Amazon EKS creates an endpoint for the managed Kubernetes API server that you use to communicate with your cluster (using Kubernetes management tools such as `kubectl`). By default, this API server endpoint is public to the internet, and access to the API server is secured using a combination of {aws} Identity and Access Management (IAM) and native Kubernetes https://kubernetes.io/docs/reference/access-authn-authz/rbac/[Role Based Access Control] (RBAC). + +You can enable private access to the Kubernetes API server so that all communication between your nodes and the API server stays within your VPC. You can limit the IP addresses that can access your API server from the internet, or completely disable internet access to the API server. + +For more information about modifying cluster endpoint access, see <>. + +You can implement Kubernetes _network policies_ with the Amazon VPC CNI or third-party tools such as https://docs.tigera.io/calico/latest/about/[Project Calico]. For more information about using the Amazon VPC CNI for network policies, see <>. Project Calico is a third party open source project. For more information, see the https://docs.tigera.io/calico/latest/getting-started/kubernetes/managed-public-cloud/eks/[Project Calico documentation]. + +include::vpc-interface-endpoints.adoc[leveloffset=+1] + diff --git a/latest/ug/security/manage-secrets.adoc b/latest/ug/security/manage-secrets.adoc new file mode 100644 index 000000000..b66e0fee4 --- /dev/null +++ b/latest/ug/security/manage-secrets.adoc @@ -0,0 +1,21 @@ +include::../attributes.txt[] + +[.topic] +[#manage-secrets] += Use {aws} Secrets Manager secrets with Amazon EKS Pods +:info_titleabbrev: {aws} Secrets Manager + +To show secrets from Secrets Manager and parameters from Parameter Store as files mounted in Amazon EKS Pods, you can use the {aws} Secrets and Configuration Provider (ASCP) for the https://secrets-store-csi-driver.sigs.k8s.io/[Kubernetes Secrets Store CSI Driver]. + +With the ASCP, you can store and manage your secrets in Secrets Manager and then retrieve them through your workloads running on Amazon EKS. You can use IAM roles and policies to limit access to your secrets to specific Kubernetes Pods in a cluster. The ASCP retrieves the Pod identity and exchanges the identity for an IAM role. ASCP assumes the IAM role of the Pod, and then it can retrieve secrets from Secrets Manager that are authorized for that role. + +If you use Secrets Manager automatic rotation for your secrets, you can also use the Secrets Store CSI Driver rotation reconciler feature to ensure you are retrieving the latest secret from Secrets Manager. + +[NOTE] +==== + +{aws} Fargate (Fargate) node groups are not supported. + +==== + +For more information, see link:secretsmanager/latest/userguide/integrating_csi_driver.html[Using Secrets Manager secrets in Amazon EKS,type="documentation"] in the {aws} Secrets Manager User Guide. \ No newline at end of file diff --git a/latest/ug/security/security-best-practices.adoc b/latest/ug/security/security-best-practices.adoc new file mode 100644 index 000000000..b5cf74080 --- /dev/null +++ b/latest/ug/security/security-best-practices.adoc @@ -0,0 +1,13 @@ +include::../attributes.txt[] + +[.topic] +[#security-best-practices] += Secure Amazon EKS clusters with best practices +:info_titleabbrev: Best practices + +[abstract] +-- +Learn how to secure your Amazon EKS clusters by following the best practices from the community. +-- + +The Amazon EKS security best practices are in the link:eks/latest/best-practices/security.html[Best Practices for Security,type="documentation"] in the _Amazon EKS Best Practices Guide_. \ No newline at end of file diff --git a/latest/ug/security/security-eks.adoc b/latest/ug/security/security-eks.adoc index 1211a1dfa..097e0e505 100644 --- a/latest/ug/security/security-eks.adoc +++ b/latest/ug/security/security-eks.adoc @@ -1,15 +1,9 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[security-eks,security-eks.title]] +[#security-eks] = Security considerations for Amazon Elastic Kubernetes Service -:info_doctype: section -:info_title: Security considerations for Amazon Elastic Kubernetes Service -:info_titleabbrev: Considerations for Amazon EKS -:info_abstract: Configure Amazon EKS clusters to meet your security and compliance objectives, and learn \ - how to use other {aws} services that help you to secure your Amazon EKS \ - clusters. - -include::../attributes.txt[] +:info_titleabbrev: Considerations for EKS [abstract] -- @@ -21,122 +15,8 @@ The following are considerations for security of the cloud, as they affect Amazo [.topiclist] [[Topic List]] -[.topic] -[[infrastructure-security,infrastructure-security.title]] -== Infrastructure security in Amazon EKS - -[abstract] --- -Learn how Amazon EKS isolates service traffic. --- - -As a managed service, Amazon Elastic Kubernetes Service is protected by {aws} global network security. For information about {aws} security services and how {aws} protects infrastructure, see link:security/[{aws} Cloud Security,type="marketing"]. To design your {aws} environment using the best practices for infrastructure security, see link:wellarchitected/latest/security-pillar/infrastructure-protection.html[Infrastructure Protection,type="documentation"] in _Security Pillar {aws} Well‐Architected Framework_. - -You use {aws} published API calls to access Amazon EKS through the network. Clients must support the following: - - - -* Transport Layer Security (TLS). We require TLS 1.2 and recommend TLS 1.3. -* Cipher suites with perfect forward secrecy (PFS) such as DHE (Ephemeral Diffie-Hellman) or ECDHE (Elliptic Curve Ephemeral Diffie-Hellman). Most modern systems such as Java 7 and later support these modes. - -Additionally, requests must be signed by using an access key ID and a secret access key that is associated with an IAM principal. Or you can use the link:STS/latest/APIReference/welcome.html[{aws} Security Token Service,type="documentation"] ({aws} STS) to generate temporary security credentials to sign requests. - -When you create an Amazon EKS cluster, you specify the VPC subnets for your cluster to use. Amazon EKS requires subnets in at least two Availability Zones. We recommend a VPC with public and private subnets so that [.noloc]`Kubernetes` can create public load balancers in the public subnets that load balance traffic to [.noloc]`Pods` running on nodes that are in private subnets. - -For more information about VPC considerations, see <>. - -If you create your VPC and node groups with the {aws} CloudFormation templates provided in the <> walkthrough, then your control plane and node security groups are configured with our recommended settings. - -For more information about security group considerations, see <>. - -When you create a new cluster, Amazon EKS creates an endpoint for the managed [.noloc]`Kubernetes` API server that you use to communicate with your cluster (using [.noloc]`Kubernetes` management tools such as `kubectl`). By default, this API server endpoint is public to the internet, and access to the API server is secured using a combination of {aws} Identity and Access Management (IAM) and native [.noloc]`Kubernetes` https://kubernetes.io/docs/reference/access-authn-authz/rbac/[Role Based Access Control] (RBAC). - -You can enable private access to the [.noloc]`Kubernetes` API server so that all communication between your nodes and the API server stays within your VPC. You can limit the IP addresses that can access your API server from the internet, or completely disable internet access to the API server. - -For more information about modifying cluster endpoint access, see <>. - -You can implement [.noloc]`Kubernetes` _network policies_ with the Amazon VPC CNI or third-party tools such as https://docs.tigera.io/calico/latest/about/[Project Calico]. For more information about using the Amazon VPC CNI for network policies, see <>. Project [.noloc]`Calico` is a third party open source project. For more information, see the https://docs.tigera.io/calico/latest/getting-started/kubernetes/managed-public-cloud/eks/[Project Calico documentation]. - -[.topic] -[[vpc-interface-endpoints,vpc-interface-endpoints.title]] -=== Access the Amazon EKS using {aws} PrivateLink - -[abstract] --- -Learn how to securely access Amazon Elastic Kubernetes Service (Amazon EKS) APIs from within your VPC using {aws} PrivateLink, avoiding public internet exposure while benefiting from private connectivity, routing optimization, and built-in security controls for enhanced cluster management. --- - -You can use {aws} PrivateLink to create a private connection between your VPC and Amazon Elastic Kubernetes Service. You can access Amazon EKS as if it were in your VPC, without the use of an internet gateway, NAT device, VPN connection, or {aws} Direct Connect connection. Instances in your VPC don't need public IP addresses to access Amazon EKS. - -You establish this private connection by creating an interface endpoint powered by {aws} PrivateLink. We create an endpoint network interface in each subnet that you enable for the interface endpoint. These are requester-managed network interfaces that serve as the entry point for traffic destined for Amazon EKS. - -For more information, see link:vpc/latest/privatelink/privatelink-access-aws-services.html[Access {aws} services through {aws} PrivateLink,type="documentation"] in the _{aws} PrivateLink Guide_. - -[[vpc-endpoint-considerations,vpc-endpoint-considerations.title]] -==== Considerations for Amazon EKS - -* Before you set up an interface endpoint for Amazon EKS, review link:vpc/latest/privatelink/create-interface-endpoint.html#considerations-interface-endpoints[Considerations,type="documentation"] in the _{aws} PrivateLink Guide_. -* Amazon EKS supports making calls to all of its API actions through the interface endpoint, but not to the [.noloc]`Kubernetes` APIs. The [.noloc]`Kubernetes` API server already supports a <>. The [.noloc]`Kubernetes` API server private endpoint creates a private endpoint for the [.noloc]`Kubernetes` API server that you use to communicate with your cluster (using [.noloc]`Kubernetes` management tools such as `kubectl`). You can enable <> to the [.noloc]`Kubernetes` API server so that all communication between your nodes and the API server stays within your VPC. {aws} PrivateLink for the Amazon EKS API helps you call the Amazon EKS APIs from your VPC without exposing traffic to the public internet. -* You can't configure Amazon EKS to only be accessed through an interface endpoint. -* Standard pricing for {aws} PrivateLink applies for interface endpoints for Amazon EKS. You are billed for every hour that an interface endpoint is provisioned in each Availability Zone and for data processed through the interface endpoint. For more information, see link:privatelink/pricing/[{aws} PrivateLink pricing,type="marketing"]. -* VPC endpoint policies are not supported for Amazon EKS. By default, full access to Amazon EKS is allowed through the interface endpoint. Alternatively, you can associate a security group with the endpoint network interfaces to control traffic to Amazon EKS through the interface endpoint. -* You can use VPC flow logs to capture information about IP traffic going to and from network interfaces, including interface endpoints. You can publish flow log data to Amazon CloudWatch or Amazon S3. For more information, see link:vpc/latest/userguide/flow-logs.html[Logging IP traffic using VPC Flow Logs,type="documentation"] in the Amazon VPC User Guide. -* You can access the Amazon EKS APIs from an on-premises data center by connecting it to a VPC that has an interface endpoint. You can use {aws} Direct Connect or {aws} Site-to-Site VPN to connect your on-premises sites to a VPC. -* You can connect other VPCs to the VPC with an interface endpoint using an {aws} Transit Gateway or VPC peering. VPC peering is a networking connection between two VPCs. You can establish a VPC peering connection between your VPCs, or with a VPC in another account. The VPCs can be in different {aws} Regions. Traffic between peered VPCs stays on the {aws} network. The traffic doesn't traverse the public internet. A Transit Gateway is a network transit hub that you can use to interconnect VPCs. Traffic between a VPC and a Transit Gateway remains on the {aws} global private network. The traffic isn't exposed to the public internet. -* Before August 2024, VPC interface endpoints for Amazon EKS were only accessible over `IPv4` using `eks.[.replaceable]``region``.amazonaws.com`. New VPC interface endpoints that are made after August 2024 use dual-stack of `IPv4` and `IPv6` IP addresses and both DNS names: `eks.[.replaceable]``region``.amazonaws.com` and `eks.[.replaceable]``region``.api.aws`. -* {aws} PrivateLink support for the EKS API isn't available in the Asia Pacific (Malaysia) {aws} Region. {aws} PrivateLink support for `eks-auth` for EKS Pod Identity is available in the Asia Pacific (Malaysia) {aws} Region. - - -[[vpc-endpoint-create,vpc-endpoint-create.title]] -==== Create an interface endpoint for Amazon EKS - -You can create an interface endpoint for Amazon EKS using either the Amazon VPC console or the {aws} Command Line Interface ({aws} CLI). For more information, see link:vpc/latest/privatelink/create-interface-endpoint.html#create-interface-endpoint-aws[Create a VPC endpoint,type="documentation"] in the _{aws} PrivateLink Guide_. - -Create an interface endpoint for Amazon EKS using the following service names: - - - -* -[source,none,subs="verbatim,attributes"] ----- -com.amazonaws.region-code.eks ----- -* -[source,none,subs="verbatim,attributes"] ----- -com.amazonaws.region-code.eks-auth ----- - -The private DNS feature is enabled by default when creating an interface endpoint for Amazon EKS and other {aws} services. To use the private DNS feature, you must ensure that the following VPC attributes are set to `true`: `enableDnsHostnames` and `enableDnsSupport`. For more information, see link:vpc/latest/userguide/vpc-dns.html#vpc-dns-updating[View and update DNS attributes for your VPC,type="documentation"] in the Amazon VPC User Guide. With the private DNS feature enabled for the interface endpoint: - - - -* You can make any API request to Amazon EKS using its default Regional DNS name. After August 2024, any new VPC interface endpoint for the Amazon EKS API have two default Regional DNS names and you can choose the `dualstack` for the IP address type. The first DNS name is `eks.[.replaceable]``region``.api.aws` which is dual-stack. It resolves to both `IPv4` addresses and `IPv6` addresses. Before August 2024, Amazon EKS only used `eks.[.replaceable]``region``.amazonaws.com` which resolved to `IPv4` addresses only. If you want to use `IPv6` and dual-stack IP addresses with an existing VPC interface endpoint, you can update the endpoint to use the `dualstack` type of IP address, but it will only have the `eks.[.replaceable]``region``.amazonaws.com` DNS name. In this configuration, the existing endpoint updates to point that name to both `IPv4` and `IPv6` IP addresses. For a list of APIs, see link:eks/latest/APIReference/API_Operations.html[Actions,type="documentation"] in the Amazon EKS API Reference. -* You don't need to make any changes to your applications that call the EKS APIs. -+ -However, To use the dual-stack endpoints with the {aws} CLI, see the link:sdkref/latest/guide/feature-endpoints.html[Dual-stack and FIPS endpoints,type="documentation"] configuration in the _{aws} SDKs and Tools Reference Guide_. -* Any call made to the Amazon EKS default service endpoint is automatically routed through the interface endpoint over the private {aws} network. - - -[.topic] -[[disaster-recovery-resiliency,disaster-recovery-resiliency.title]] -== Understand resilience in Amazon EKS clusters - -[abstract] --- -Learn how Amazon EKS ensures high availability, data resilience, and fault tolerance for your [.noloc]`Kubernetes` control plane by leveraging {aws} infrastructure across multiple Availability Zones . --- - -The {aws} global infrastructure is built around {aws} Regions and Availability Zones. {aws} Regions provide multiple physically separated and isolated Availability Zones, which are connected with low-latency, high-throughput, and highly redundant networking. With Availability Zones, you can design and operate applications and databases that automatically fail over between Availability Zones without interruption. Availability Zones are more highly available, fault tolerant, and scalable than traditional single or multiple data center infrastructures. - -Amazon EKS runs and scales the [.noloc]`Kubernetes` control plane across multiple {aws} Availability Zones to ensure high availability. Amazon EKS automatically scales control plane instances based on load, detects and replaces unhealthy control plane instances, and automatically patches the control plane. After you initiate a version update, Amazon EKS updates your control plane for you, maintaining high availability of the control plane during the update. - -This control plane consists of at least two API server instances and three `etcd` instances that run across three Availability Zones within an {aws} Region. Amazon EKS: - - +include::infrastructure-security.adoc[leveloffset=+1] -* Actively monitors the load on control plane instances and automatically scales them to ensure high performance. -* Automatically detects and replaces unhealthy control plane instances, restarting them across the Availability Zones within the {aws} Region as needed. -* Leverages the architecture of {aws} Regions in order to maintain high availability. Because of this, Amazon EKS is able to offer an link:eks/sla[SLA for API server endpoint availability,type="marketing"]. +include::disaster-recovery-resiliency.adoc[leveloffset=+1] -For more information about {aws} Regions and Availability Zones, see link:about-aws/global-infrastructure/[{aws} global infrastructure,type="marketing"]. +include::cross-service-confused-deputy-prevention.adoc[leveloffset=+1] diff --git a/latest/ug/security/security-k8s.adoc b/latest/ug/security/security-k8s.adoc index c663f6796..c1db689ae 100644 --- a/latest/ug/security/security-k8s.adoc +++ b/latest/ug/security/security-k8s.adoc @@ -1,857 +1,27 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[security-k8s,security-k8s.title]] -= Security considerations for [.noloc]`Kubernetes` -:info_doctype: section -:info_title: Security considerations for Kubernetes +[#security-k8s] += Security considerations for Kubernetes :info_titleabbrev: Considerations for Kubernetes -:info_abstract: Configure Kubernetes to meet your security and compliance objectives, and learn \ - how to use other {aws} services that help you to secure your Kubernetes \ - resources. - -include::../attributes.txt[] [abstract] -- -Configure [.noloc]`Kubernetes` to meet your security and compliance objectives, and learn how to use other {aws} services that help you to secure your [.noloc]`Kubernetes` resources. +Configure Kubernetes to meet your security and compliance objectives, and learn how to use other {aws} services that help you to secure your Kubernetes resources. -- -The following are considerations for security in the cloud, as they affect [.noloc]`Kubernetes` in Amazon EKS clusters. For an in-depth review of security controls and practices in [.noloc]`Kubernetes`, see https://kubernetes.io/docs/concepts/security/cloud-native-security/[Cloud Native Security and Kubernetes] in the [.noloc]`Kubernetes` documentation. +The following are considerations for security in the cloud, as they affect Kubernetes in Amazon EKS clusters. For an in-depth review of security controls and practices in Kubernetes, see https://kubernetes.io/docs/concepts/security/cloud-native-security/[Cloud Native Security and Kubernetes] in the Kubernetes documentation. [.topiclist] [[Topic List]] -[.topic] -[[cert-signing,cert-signing.title]] -== Secure workloads with [.noloc]`Kubernetes` certificates - -[abstract] --- -Learn how to request and obtain X.509 certificates from the Certificate Authority (CA) using Certificate Signing Requests (CSRs) in Amazon EKS, including details on migrating from legacy signers, generating CSRs, approving requests, and handling certificate signing considerations before upgrading to Kubernetes 1.24. --- - -The [.noloc]`Kubernetes` Certificates API automates https://www.itu.int/rec/T-REC-X.509[X.509] credential provisioning. The API features a command line interface for [.noloc]`Kubernetes` API clients to request and obtain https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/[X.509 certificates] from a Certificate Authority (CA). You can use the `CertificateSigningRequest` (CSR) resource to request that a denoted signer sign the certificate. Your requests are either approved or denied before they're signed. [.noloc]`Kubernetes` supports both built-in signers and custom signers with well-defined behaviors. This way, clients can predict what happens to their CSRs. To learn more about certificate signing, see https://kubernetes.io/docs/reference/access-authn-authz/certificate-signing-requests/[signing requests]. - -One of the built-in signers is `kubernetes.io/legacy-unknown`. The `v1beta1` API of CSR resource honored this legacy-unknown signer. However, the stable `v1` API of CSR doesn't allow the `signerName` to be set to `kubernetes.io/legacy-unknown`. - -Amazon EKS version `1.21` and earlier allowed the `legacy-unknown` value as the `signerName` in `v1beta1` CSR API. This API enables the Amazon EKS Certificate Authority (CA) to generate certificates. However, in [.noloc]`Kubernetes` version `1.22`, the `v1beta1` CSR API was replaced by the `v1` CSR API. This API doesn't support the signerName of "`legacy-unknown.`" If you want to use Amazon EKS CA for generating certificates on your clusters, you must use a custom signer. It was introduced in Amazon EKS version `1.22`. To use the CSR `v1` API version and generate a new certificate, you must migrate any existing manifests and API clients. Existing certificates that were created with the existing `v1beta1` API are valid and function until the certificate expires. This includes the following: - - - -* Trust distribution: None. There's no standard trust or distribution for this signer in a [.noloc]`Kubernetes` cluster. -* Permitted subjects: Any -* Permitted x509 extensions: Honors subjectAltName and key usage extensions and discards other extensions -* Permitted key usages: Must not include usages beyond ["key encipherment", "digital signature", "server auth"] -+ -NOTE: Client certificate signing is not supported. -* Expiration/certificate lifetime: 1 year (default and maximum) -* CA bit allowed/disallowed: Not allowed - - -[[csr-example,csr-example.title]] -=== Example CSR generation with signerName - -These steps shows how to generate a serving certificate for DNS name `myserver.default.svc` using `signerName: beta.eks.amazonaws.com/app-serving`. Use this as a guide for your own environment. - -. Run the `openssl genrsa -out myserver.key 2048` command to generate an RSA private key. -+ -[source,bash,subs="verbatim,attributes"] ----- -openssl genrsa -out myserver.key 2048 ----- -. Run the following command to generate a certificate request. -+ -[source,bash,subs="verbatim,attributes"] ----- -openssl req -new -key myserver.key -out myserver.csr -subj "/CN=myserver.default.svc" ----- -. Generate a `base64` value for the CSR request and store it in a variable for use in a later step. -+ -[source,bash,subs="verbatim,attributes"] ----- -base_64=$(cat myserver.csr | base64 -w 0 | tr -d "\n") ----- -. Run the following command to create a file named `mycsr.yaml`. In the following example, `beta.eks.amazonaws.com/app-serving` is the `signerName`. -+ -[source,yaml,subs="verbatim,attributes"] ----- -cat >mycsr.yaml < myserver.crt ----- - - -[[csr-considerations,csr-considerations.title]] -=== Certificate signing considerations before upgrading your cluster to [.noloc]`Kubernetes` 1.24 - -In [.noloc]`Kubernetes` `1.23` and earlier, `kubelet` serving certificates with unverifiable IP and DNS Subject Alternative Names (SANs) are automatically issued with unverifiable SANs. The SANs are omitted from the provisioned certificate. In `1.24` and later clusters, `kubelet` serving certificates aren't issued if a SAN can't be verified. This prevents the `kubectl exec` and `kubectl logs` commands from working. - -Before upgrading your cluster to `1.24`, determine whether your cluster has certificate signing requests (CSR) that haven't been approved by completing the following steps: - -. Run the following command. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl get csr -A ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -NAME AGE SIGNERNAME REQUESTOR REQUESTEDDURATION CONDITION -csr-7znmf 90m kubernetes.io/kubelet-serving system:node:ip-192-168-42-149.region.compute.internal Approved -csr-9xx5q 90m kubernetes.io/kubelet-serving system:node:ip-192-168-65-38.region.compute.internal Approved, Issued ----- -+ -If the returned output shows a CSR with a https://kubernetes.io/docs/reference/access-authn-authz/certificate-signing-requests/#kubernetes-signers[kubernetes.io/kubelet-serving] signer that's `Approved` but not `Issued` for a node, then you need to approve the request. -. Manually approve the CSR. Replace `csr-[.replaceable]``7znmf``` with your own value. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl certificate approve csr-7znmf ----- - -To auto-approve CSRs in the future, we recommend that you write an approving controller that can automatically validate and approve CSRs that contain IP or DNS SANs that Amazon EKS can't verify. - -[.topic] -[[default-roles-users,default-roles-users.title]] -== Understand Amazon EKS created RBAC roles and users - -[abstract] --- -Learn about the Kubernetes roles and users that Amazon EKS creates for cluster components and add-ons. Amazon EKS uses these role-based authorization control (RBAC) identities to operate the cluster. --- - -When you create a [.noloc]`Kubernetes` cluster, several default [.noloc]`Kubernetes` identities are created on that cluster for the proper functioning of [.noloc]`Kubernetes`. Amazon EKS creates [.noloc]`Kubernetes` identities for each of its default components. The identities provide [.noloc]`Kubernetes` role-based authorization control (RBAC) for the cluster components. For more information, see https://kubernetes.io/docs/reference/access-authn-authz/rbac/[Using RBAC Authorization] in the [.noloc]`Kubernetes` documentation. - -When you install optional <> to your cluster, additional [.noloc]`Kubernetes` identities might be added to your cluster. For more information about identities not addressed by this topic, see the documentation for the add-on. - -You can view the list of Amazon EKS created [.noloc]`Kubernetes` identities on your cluster using the {aws-management-console} or `kubectl` command line tool. All of the user identities appear in the `kube` audit logs available to you through Amazon CloudWatch. - - - -*{aws-management-console}*:: - -.Prerequisite -The link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] that you use must have the permissions described in <>. -+ -.. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. -.. In the *Clusters* list, choose the cluster that contains the identities that you want to view. -.. Choose the *Resources* tab. -.. Under *Resource types*, choose *Authorization*. -.. Choose, *ClusterRoles*, *ClusterRoleBindings*, *Roles*, or *RoleBindings*. All resources prefaced with *eks* are created by Amazon EKS. Additional Amazon EKS created identity resources are: -+ -*** The *ClusterRole* and *ClusterRoleBinding* named *aws-node*. The *aws-node* resources support the <>, which Amazon EKS installs on all clusters. -*** A *ClusterRole* named *vpc-resource-controller-role* and a *ClusterRoleBinding* named *vpc-resource-controller-rolebinding*. These resources support the https://github.com/aws/amazon-vpc-resource-controller-k8s[Amazon VPC resource controller], which Amazon EKS installs on all clusters. - -+ -In addition to the resources that you see in the console, the following special user identities exist on your cluster, though they're not visible in the cluster's configuration: -+ -*** *`eks:cluster-bootstrap`* – Used for `kubectl` operations during cluster bootstrap. -*** *`eks:support-engineer`* – Used for cluster management operations. -.. Choose a specific resource to view details about it. By default, you're shown information in *Structured view*. In the top-right corner of the details page you can choose *Raw view* to see all information for the resource. - - -*Kubectl*:: - -.Prerequisite -The entity that you use ({aws} Identity and Access Management (IAM) or [.noloc]`OpenID Connect` ([.noloc]`OIDC`)) to list the [.noloc]`Kubernetes` resources on the cluster must be authenticated by IAM or your [.noloc]`OIDC` identity provider. The entity must be granted permissions to use the [.noloc]`Kubernetes` `get` and `list` verbs for the `Role`, `ClusterRole`, `RoleBinding`, and `ClusterRoleBinding` resources on your cluster that you want the entity to work with. For more information about granting IAM entities access to your cluster, see <>. For more information about granting entities authenticated by your own [.noloc]`OIDC` provider access to your cluster, see <>. -.To view Amazon EKS created identities using `kubectl` -Run the command for the type of resource that you want to see. All returned resources that are prefaced with *eks* are created by Amazon EKS. In addition to the resources returned in the output from the commands, the following special user identities exist on your cluster, though they're not visible in the cluster's configuration: -+ -** *`eks:cluster-bootstrap`* – Used for `kubectl` operations during cluster bootstrap. -** *`eks:support-engineer`* – Used for cluster management operations. -+ -*ClusterRoles* – `ClusterRoles` are scoped to your cluster, so any permission granted to a role applies to resources in any [.noloc]`Kubernetes` namespace on the cluster. -+ -The following command returns all of the Amazon EKS created [.noloc]`Kubernetes` `ClusterRoles` on your cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl get clusterroles | grep eks ----- -+ -In addition to the `ClusterRoles` returned in the output that are prefaced with, the following `ClusterRoles` exist. -+ -** *`aws-node`* – This `ClusterRole` supports the <>, which Amazon EKS installs on all clusters. -** *`vpc-resource-controller-role`* – This `ClusterRole` supports the https://github.com/aws/amazon-vpc-resource-controller-k8s[Amazon VPC resource controller], which Amazon EKS installs on all clusters. - -+ -To see the specification for a `ClusterRole`, replace [.replaceable]`eks:k8s-metrics` in the following command with a `ClusterRole` returned in the output of the previous command. The following example returns the specification for the [.replaceable]`eks:k8s-metrics` `ClusterRole`. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl describe clusterrole eks:k8s-metrics ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -Name: eks:k8s-metrics -Labels: -Annotations: -PolicyRule: - Resources Non-Resource URLs Resource Names Verbs - --------- ----------------- -------------- ----- - [/metrics] [] [get] - endpoints [] [] [list] - nodes [] [] [list] - pods [] [] [list] - deployments.apps [] [] [list] ----- -+ -*ClusterRoleBindings* – `ClusterRoleBindings` are scoped to your cluster. -+ -The following command returns all of the Amazon EKS created [.noloc]`Kubernetes` `ClusterRoleBindings` on your cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl get clusterrolebindings | grep eks ----- -+ -In addition to the `ClusterRoleBindings` returned in the output, the following `ClusterRoleBindings` exist. -+ -** *`aws-node`* – This `ClusterRoleBinding` supports the <>, which Amazon EKS installs on all clusters. -** *`vpc-resource-controller-rolebinding`* – This `ClusterRoleBinding` supports the https://github.com/aws/amazon-vpc-resource-controller-k8s[Amazon VPC resource controller], which Amazon EKS installs on all clusters. - -+ -To see the specification for a `ClusterRoleBinding`, replace [.replaceable]`eks:k8s-metrics` in the following command with a `ClusterRoleBinding` returned in the output of the previous command. The following example returns the specification for the [.replaceable]`eks:k8s-metrics` `ClusterRoleBinding`. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl describe clusterrolebinding eks:k8s-metrics ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -Name: eks:k8s-metrics -Labels: -Annotations: -Role: - Kind: ClusterRole - Name: eks:k8s-metrics -Subjects: - Kind Name Namespace - ---- ---- --------- - User eks:k8s-metrics ----- -+ -*Roles* – `Roles` are scoped to a [.noloc]`Kubernetes` namespace. All Amazon EKS created `Roles` are scoped to the `kube-system` namespace. -+ -The following command returns all of the Amazon EKS created [.noloc]`Kubernetes` `Roles` on your cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl get roles -n kube-system | grep eks ----- -+ -To see the specification for a `Role`, replace [.replaceable]`eks:k8s-metrics` in the following command with the name of a `Role` returned in the output of the previous command. The following example returns the specification for the [.replaceable]`eks:k8s-metrics` `Role`. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl describe role eks:k8s-metrics -n kube-system ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -Name: eks:k8s-metrics -Labels: -Annotations: -PolicyRule: - Resources Non-Resource URLs Resource Names Verbs - --------- ----------------- -------------- ----- - daemonsets.apps [] [aws-node] [get] - deployments.apps [] [vpc-resource-controller] [get] ----- -+ -*RoleBindings* – `RoleBindings` are scoped to a [.noloc]`Kubernetes` namespace. All Amazon EKS created `RoleBindings` are scoped to the `kube-system` namespace. -+ -The following command returns all of the Amazon EKS created [.noloc]`Kubernetes` `RoleBindings` on your cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl get rolebindings -n kube-system | grep eks ----- -+ -To see the specification for a `RoleBinding`, replace [.replaceable]`eks:k8s-metrics` in the following command with a `RoleBinding` returned in the output of the previous command. The following example returns the specification for the [.replaceable]`eks:k8s-metrics` `RoleBinding`. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl describe rolebinding eks:k8s-metrics -n kube-system ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -Name: eks:k8s-metrics -Labels: -Annotations: -Role: - Kind: Role - Name: eks:k8s-metrics -Subjects: - Kind Name Namespace - ---- ---- --------- - User eks:k8s-metrics ----- - - -[.topic] -[[pod-security-policy,pod-security-policy.title]] -== Understand Amazon EKS created pod security policies [.noloc]`(PSP)` - -[abstract] --- -Learn about the Pod Security Policies [.noloc]`(PSP)` that Amazon EKS creates by default. PSP was deprecated in [.noloc]`Kubernetes` version `1.21` and removed in [.noloc]`Kubernetes` `1.25`. --- - -The [.noloc]`Kubernetes` [.noloc]`Pod` security policy admission controller validates [.noloc]`Pod` creation and update requests against a set of rules. By default, Amazon EKS clusters ship with a fully permissive security policy with no restrictions. For more information, see https://kubernetes.io/docs/concepts/policy/pod-security-policy/[Pod Security Policies] in the [.noloc]`Kubernetes` documentation. - -[NOTE] -==== - -The `PodSecurityPolicy` ([.noloc]`PSP`) was deprecated in [.noloc]`Kubernetes` version `1.21` and removed in [.noloc]`Kubernetes` `1.25`. [.noloc]`PSPs` are being replaced with https://kubernetes.io/docs/concepts/security/pod-security-admission/[Pod Security Admission (PSA)], a built-in admission controller that implements the security controls outlined in the https://kubernetes.io/docs/concepts/security/pod-security-standards/[Pod Security Standards (PSS)]. PSA and PSS have both reached beta feature states, and are enabled in Amazon EKS by default. To address [.noloc]`PSP` removal in `1.25`, we recommend that you implement PSS in Amazon EKS. For more information, see link:containers/implementing-pod-security-standards-in-amazon-eks[Implementing Pod Security Standards in Amazon EKS,type="blog"] on the {aws} blog. - -==== - -[[default-psp,default-psp.title]] -=== Amazon EKS default [.noloc]`Pod` security policy - -Amazon EKS clusters with [.noloc]`Kubernetes` version `1.13` or higher have a default [.noloc]`Pod` security policy named `eks.privileged`. This policy has no restriction on what kind of [.noloc]`Pod` can be accepted into the system, which is equivalent to running [.noloc]`Kubernetes` with the `PodSecurityPolicy` controller disabled. - -[NOTE] -==== - -This policy was created to maintain backwards compatibility with clusters that did not have the `PodSecurityPolicy` controller enabled. You can create more restrictive policies for your cluster and for individual namespaces and service accounts and then delete the default policy to enable the more restrictive policies. - -==== - -You can view the default policy with the following command. - -[source,bash,subs="verbatim,attributes"] ----- -kubectl get psp eks.privileged ----- - -An example output is as follows. - -[source,bash,subs="verbatim,attributes"] ----- -NAME PRIV CAPS SELINUX RUNASUSER FSGROUP SUPGROUP READONLYROOTFS VOLUMES -eks.privileged true * RunAsAny RunAsAny RunAsAny RunAsAny false * ----- - -For more details, you can describe the policy with the following command. - -[source,bash,subs="verbatim,attributes"] ----- -kubectl describe psp eks.privileged ----- - -An example output is as follows. - -[source,bash,subs="verbatim,attributes"] ----- -Name: eks.privileged - -Settings: - Allow Privileged: true - Allow Privilege Escalation: 0xc0004ce5f8 - Default Add Capabilities: - Required Drop Capabilities: - Allowed Capabilities: * - Allowed Volume Types: * - Allow Host Network: true - Allow Host Ports: 0-65535 - Allow Host PID: true - Allow Host IPC: true - Read Only Root Filesystem: false - SELinux Context Strategy: RunAsAny - User: - Role: - Type: - Level: - Run As User Strategy: RunAsAny - Ranges: - FSGroup Strategy: RunAsAny - Ranges: - Supplemental Groups Strategy: RunAsAny - Ranges: ----- - -You can view the full YAML file for the `eks.privileged` [.noloc]`Pod` security policy, its cluster role, and cluster role binding in <>. - -[[psp-delete-default,psp-delete-default.title]] -=== Delete the default Amazon EKS [.noloc]`Pod` security policy - -If you create more restrictive policies for your [.noloc]`Pods`, then after doing so, you can delete the default Amazon EKS `eks.privileged` [.noloc]`Pod` security policy to enable your custom policies. - -[IMPORTANT] -==== - -If you are using version `1.7.0` or later of the CNI plugin and you assign a custom [.noloc]`Pod` security policy to the `aws-node` [.noloc]`Kubernetes` service account used for the `aws-node` [.noloc]`Pods` deployed by the Daemonset, then the policy must have `NET_ADMIN` in its `allowedCapabilities` section along with `hostNetwork: true` and `privileged: true` in the policy's `spec`. - -==== -. Create a file named [.replaceable]`privileged-podsecuritypolicy.yaml` with the contents in the example file in <>. -. Delete the YAML with the following command. This deletes the default [.noloc]`Pod` security policy, the `ClusterRole`, and the `ClusterRoleBinding` associated with it. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl delete -f privileged-podsecuritypolicy.yaml ----- - - -[[psp-install-or-restore-default,psp-install-or-restore-default.title]] -=== Install or restore the default [.noloc]`Pod` security policy - -If you are upgrading from an earlier version of [.noloc]`Kubernetes`, or have modified or deleted the default Amazon EKS `eks.privileged` [.noloc]`Pod` security policy, you can restore it with the following steps. - -. Create a file called `privileged-podsecuritypolicy.yaml` with the following contents. -+ -[source,yaml,subs="verbatim,attributes"] ----- -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - name: eks.privileged - annotations: - kubernetes.io/description: 'privileged allows full unrestricted access to - Pod features, as if the PodSecurityPolicy controller was not enabled.' - seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' - labels: - kubernetes.io/cluster-service: "true" - eks.amazonaws.com/component: pod-security-policy -spec: - privileged: true - allowPrivilegeEscalation: true - allowedCapabilities: - - '*' - volumes: - - '*' - hostNetwork: true - hostPorts: - - min: 0 - max: 65535 - hostIPC: true - hostPID: true - runAsUser: - rule: 'RunAsAny' - seLinux: - rule: 'RunAsAny' - supplementalGroups: - rule: 'RunAsAny' - fsGroup: - rule: 'RunAsAny' - readOnlyRootFilesystem: false - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: eks:podsecuritypolicy:privileged - labels: - kubernetes.io/cluster-service: "true" - eks.amazonaws.com/component: pod-security-policy -rules: -- apiGroups: - - policy - resourceNames: - - eks.privileged - resources: - - podsecuritypolicies - verbs: - - use - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: eks:podsecuritypolicy:authenticated - annotations: - kubernetes.io/description: 'Allow all authenticated users to create privileged Pods.' - labels: - kubernetes.io/cluster-service: "true" - eks.amazonaws.com/component: pod-security-policy -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: eks:podsecuritypolicy:privileged -subjects: - - kind: Group - apiGroup: rbac.authorization.k8s.io - name: system:authenticated ----- -. Apply the YAML with the following command. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl apply -f privileged-podsecuritypolicy.yaml ----- - - -[.topic] -[[pod-security-policy-removal-faq,pod-security-policy-removal-faq.title]] -== Migrate from legacy pod security policies (PSP) - -[abstract] --- -Learn about the Pod Security Policy [.noloc]`(PSPs)` removal in [.noloc]`Kubernetes` `1.25`. Migrate to Pod Security Standards (PSS) or policy-as-code solutions before upgrading Amazon EKS clusters to [.noloc]`Kubernetes` 1.25 to avoid workload interruptions and maintain pod security controls. --- - -`PodSecurityPolicy` was https://kubernetes.io/blog/2021/04/06/podsecuritypolicy-deprecation-past-present-and-future/[deprecated in Kubernetes1.21], and has been removed in [.noloc]`Kubernetes` `1.25`. If you are using PodSecurityPolicy in your cluster, *then you must migrate to the built-in [.noloc]`Kubernetes` Pod Security Standards [.noloc]`(PSS)` or to a policy-as-code solution before upgrading your cluster to version `*1.25*` to avoid interruptions to your workloads.* Select any frequently asked question to learn more. - - -[[pod-security-policy-removal-what-is,pod-security-policy-removal-what-is.title]] -.What is a [.noloc]`PSP`? -[%collapsible] -==== - -https://kubernetes.io/docs/concepts/security/pod-security-policy/[PodSecurityPolicy] is a built-in admission controller that allows a cluster administrator to control security-sensitive aspects of [.noloc]`Pod` specification. If a [.noloc]`Pod` meets the requirements of its [.noloc]`PSP`, the [.noloc]`Pod` is admitted to the cluster as usual. If a [.noloc]`Pod` doesn't meet the [.noloc]`PSP` requirements, the [.noloc]`Pod` is rejected and can't run. -==== - -[[pod-security-policy-removal-specific,pod-security-policy-removal-specific.title]] -.Is the [.noloc]`PSP` removal specific to Amazon EKS or is it being removed in upstream [.noloc]`Kubernetes`? -[%collapsible] -==== - -This is an upstream change in the [.noloc]`Kubernetes` project, and not a change made in Amazon EKS. [.noloc]`PSP` was deprecated in [.noloc]`Kubernetes` `1.21` and removed in [.noloc]`Kubernetes` `1.25`. The [.noloc]`Kubernetes` community identified serious usability problems with [.noloc]`PSP`. These included accidentally granting broader permissions than intended and difficulty in inspecting which [.noloc]`PSPs` apply in a given situation. These issues couldn't be addressed without making breaking changes. This is the primary reason why the [.noloc]`Kubernetes` community https://kubernetes.io/blog/2021/04/06/podsecuritypolicy-deprecation-past-present-and-future/#why-is-podsecuritypolicy-going-away[decided to remove PSP]. -==== - -[[pod-security-policy-removal-check,pod-security-policy-removal-check.title]] -.How can I check if I'm using [.noloc]`PSPs` in my Amazon EKS clusters? -[%collapsible] -==== - -To check if you're using [.noloc]`PSPs` in your cluster, you can run the following command: - -[source,bash,subs="verbatim,attributes"] ----- -kubectl get psp ----- - -To see the [.noloc]`Pods` that the [.noloc]`PSPs` in your cluster are impacting, run the following command. This command outputs the [.noloc]`Pod` name, namespace, and [.noloc]`PSPs`: - -[source,bash,subs="verbatim,attributes"] ----- -kubectl get pod -A -o jsonpath='{range.items[?(@.metadata.annotations.kubernetes\.io/psp)]}{.metadata.name}{"\t"}{.metadata.namespace}{"\t"}{.metadata.annotations.kubernetes\.io/psp}{"\n"}' ----- -==== - -[[pod-security-policy-removal-what-can,pod-security-policy-removal-what-can.title]] -.If I'm using [.noloc]`PSPs` in my Amazon EKS cluster, what can I do? -[%collapsible] -==== - -Before upgrading your cluster to `1.25`, you must migrate your [.noloc]`PSPs` to either one of these alternatives: - - - -* [.noloc]`Kubernetes` [.noloc]`PSS`. - - -* Policy-as-code solutions from the [.noloc]`Kubernetes` environment. - -In response to the [.noloc]`PSP` deprecation and the ongoing need to control [.noloc]`Pod` security from the start, the [.noloc]`Kubernetes` community created a built-in solution with https://kubernetes.io/docs/concepts/security/pod-security-standards/[(PSS)] and https://kubernetes.io/docs/concepts/security/pod-security-admission/[Pod Security Admission (PSA)]. The PSA webhook implements the controls that are defined in the [.noloc]`PSS`. - -You can review best practices for migrating [.noloc]`PSPs` to the built-in [.noloc]`PSS` in the https://aws.github.io/aws-eks-best-practices/security/docs/pods/#pod-security-standards-pss-and-pod-security-admission-psa[EKS Best Practices Guide]. We also recommend reviewing our blog on link:containers/implementing-pod-security-standards-in-amazon-eks[Implementing Pod Security Standards in Amazon EKS,type="blog"]. Additional references include https://kubernetes.io/docs/tasks/configure-pod-container/migrate-from-psp/[Migrate from PodSecurityPolicy to the Built-In PodSecurity Admission Controller] and https://kubernetes.io/docs/reference/access-authn-authz/psp-to-pod-security-standards/[Mapping PodSecurityPolicies to Pod Security Standards]. - -Policy-as-code solutions provide guardrails to guide cluster users and prevents unwanted behaviors through prescribed automated controls. Policy-as-code solutions typically use https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/[Kubernetes Dynamic Admission Controllers] to intercept the [.noloc]`Kubernetes` API server request flow using a webhook call. Policy-as-code solutions mutate and validate request payloads based on policies written and stored as code. - -There are several open source policy-as-code solutions available for [.noloc]`Kubernetes`. To review best practices for migrating [.noloc]`PSPs` to a policy-as-code solution, see the https://aws.github.io/aws-eks-best-practices/security/docs/pods/#policy-as-code-pac[Policy-as-code] section of the Pod Security page on GitHub. -==== - -[[pod-security-policy-removal-privileged,pod-security-policy-removal-privileged.title]] -.I see a [.noloc]`PSP` called `eks.privileged` in my cluster. What is it and what can I do about it? -[%collapsible] -==== - -Amazon EKS clusters with [.noloc]`Kubernetes` version `1.13` or higher have a default [.noloc]`PSP` that's named `eks.privileged`. This policy is created in `1.24` and earlier clusters. It isn't used in `1.25` and later clusters. Amazon EKS automatically migrates this [.noloc]`PSP` to a [.noloc]`PSS`-based enforcement. No action is needed on your part. -==== - -[[pod-security-policy-removal-prevent,pod-security-policy-removal-prevent.title]] -.Will Amazon EKS make any changes to [.noloc]`PSPs` present in my existing cluster when I update my cluster to version `1.25`? -[%collapsible] -==== - -No. Besides `eks.privileged`, which is a [.noloc]`PSP` created by Amazon EKS, no changes are made to other [.noloc]`PSPs` in your cluster when you upgrade to `1.25`. -==== - -[[pod-security-policy-removal-migrate,pod-security-policy-removal-migrate.title]] -.Will Amazon EKS prevent a cluster update to version `1.25` if I haven't migrated off of [.noloc]`PSP`? -[%collapsible] -==== - -No. Amazon EKS won't prevent a cluster update to version `1.25` if you didn't migrate off of [.noloc]`PSP` yet. -==== - -[[pod-security-policy-removal-forget,pod-security-policy-removal-forget.title]] -.What if I forget to migrate my [.noloc]`PSPs` to [.noloc]`PSS/PSA` or to a policy-as-code solution before I update my cluster to version `1.25`? Can I migrate after updating my cluster? -[%collapsible] -==== - -When a cluster that contains a [.noloc]`PSP` is upgraded to [.noloc]`Kubernetes` version `1.25`, the API server doesn't recognize the [.noloc]`PSP` resource in `1.25`. This might result in [.noloc]`Pods` getting incorrect security scopes. For an exhaustive list of implications, see https://kubernetes.io/docs/tasks/configure-pod-container/migrate-from-psp/[Migrate from PodSecurityPolicy to the Built-In PodSecurity Admission Controller]. -==== - -[[pod-security-policy-removal-impact,pod-security-policy-removal-impact.title]] -.How does this change impact pod security for Windows workloads? -[%collapsible] -==== - -We don't expect any specific impact to Windows workloads. PodSecurityContext has a field called `windowsOptions` in the `PodSpec v1` API for Windows [.noloc]`Pods`. This uses [.noloc]`PSS` in [.noloc]`Kubernetes` `1.25`. For more information and best practices about enforcing [.noloc]`PSS` for Windows workloads, see the https://aws.github.io/aws-eks-best-practices/windows/docs/security/#pod-security-contexts[EKS Best Practices Guide] and [.noloc]`Kubernetes` https://kubernetes.io/docs/tasks/configure-pod-container/configure-runasusername/[documentation]. -==== - -[.topic] -[[enable-kms,enable-kms.title]] -== Encrypt Kubernetes secrets with {aws} KMS on existing clusters - -[abstract] --- -Learn how to enable Kubernetes secrets encryption with {aws} KMS on an existing Amazon EKS cluster, ensuring secure storage of sensitive data. --- - -If you enable https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/[secrets encryption], the [.noloc]`Kubernetes` secrets are encrypted using the {aws} KMS key that you select. The KMS key must meet the following conditions: - - - -* Symmetric -* Can encrypt and decrypt data -* Created in the same {aws} Region as the cluster -* If the KMS key was created in a different account, the link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] must have access to the KMS key. - -For more information, see link:kms/latest/developerguide/key-policy-modifying-external-accounts.html[Allowing IAM principals in other accounts to use a KMS key,type="documentation"] in the _link:kms/latest/developerguide/[{aws} Key Management Service Developer Guide,type="documentation"]_. - -[WARNING] -==== - -You can't disable secrets encryption after enabling it. This action is irreversible. - -==== - -eksctl :: - -You can enable encryption in two ways: - -** Add encryption to your cluster with a single command. -+ -To automatically re-encrypt your secrets, run the following command. -+ -[source,bash,subs="verbatim,attributes"] ----- -eksctl utils enable-secrets-encryption \ - --cluster my-cluster \ - --key-arn {arn-aws}kms:region-code:account:key/key ----- -+ -To opt-out of automatically re-encrypting your secrets, run the following command. -+ -[source,bash,subs="verbatim,attributes"] ----- -eksctl utils enable-secrets-encryption - --cluster my-cluster \ - --key-arn {arn-aws}kms:region-code:account:key/key \ - --encrypt-existing-secrets=false ----- -** Add encryption to your cluster with a `kms-cluster.yaml` file. -+ -[source,yaml,subs="verbatim,attributes"] ----- -apiVersion: eksctl.io/v1alpha5 -kind: ClusterConfig - -metadata: - name: my-cluster - region: region-code - -secretsEncryption: - keyARN: {arn-aws}kms:region-code:account:key/key ----- -+ -To have your secrets re-encrypt automatically, run the following command. -+ -[source,bash,subs="verbatim,attributes"] ----- -eksctl utils enable-secrets-encryption -f kms-cluster.yaml ----- -+ -To opt out of automatically re-encrypting your secrets, run the following command. -+ -[source,bash,subs="verbatim,attributes"] ----- -eksctl utils enable-secrets-encryption -f kms-cluster.yaml --encrypt-existing-secrets=false ----- - - -{aws-management-console}:: -.. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. -.. Choose the cluster that you want to add KMS encryption to. -.. Choose the *Overview* tab (this is selected by default). -.. Scroll down to the *Secrets encryption* section and choose *Enable*. -.. Select a key from the dropdown list and choose the *Enable* button. If no keys are listed, you must create one first. For more information, see link:kms/latest/developerguide/create-keys.html[Creating keys,type="documentation"] -.. Choose the *Confirm* button to use the chosen key. - - -{aws} CLI:: -.. Associate the https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/[secrets encryption] configuration with your cluster using the following {aws} CLI command. Replace the [.replaceable]`example values` with your own. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks associate-encryption-config \ - --cluster-name my-cluster \ - --encryption-config '[{"resources":["secrets"],"provider":{"keyArn":"{arn-aws}kms:region-code:account:key/key"}}]' ----- -+ -An example output is as follows. -+ -[source,json,subs="verbatim,attributes"] ----- -{ -  "update": { -    "id": "3141b835-8103-423a-8e68-12c2521ffa4d", -    "status": "InProgress", -    "type": "AssociateEncryptionConfig", -    "params": [ -      { -        "type": "EncryptionConfig", -        "value": "[{\"resources\":[\"secrets\"],\"provider\":{\"keyArn\":\"{arn-aws}kms:region-code:account:key/key\"}}]" -      } -    ], -    "createdAt": 1613754188.734, -    "errors": [] -  } -} ----- -.. You can monitor the status of your encryption update with the following command. Use the specific `cluster name` and `update ID` that was returned in the previous output. When a `Successful` status is displayed, the update is complete. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks describe-update \ - --region region-code \ - --name my-cluster \ - --update-id 3141b835-8103-423a-8e68-12c2521ffa4d ----- -+ -An example output is as follows. -+ -[source,json,subs="verbatim,attributes"] ----- -{ -  "update": { -    "id": "3141b835-8103-423a-8e68-12c2521ffa4d", -    "status": "Successful", -    "type": "AssociateEncryptionConfig", -    "params": [ -      { -        "type": "EncryptionConfig", -        "value": "[{\"resources\":[\"secrets\"],\"provider\":{\"keyArn\":\"{arn-aws}kms:region-code:account:key/key\"}}]" -      } -    ], -    "createdAt": 1613754188.734>, -    "errors": [] -  } -} ----- -.. To verify that encryption is enabled in your cluster, run the `describe-cluster` command. The response contains an `EncryptionConfig` string. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks describe-cluster --region region-code --name my-cluster ----- - -After you enabled encryption on your cluster, you must encrypt all existing secrets with the new key: - -[NOTE] -==== - -If you use `eksctl`, running the following command is necessary only if you opt out of re-encrypting your secrets automatically. - -==== - -[source,bash,subs="verbatim,attributes"] ----- -kubectl get secrets --all-namespaces -o json | kubectl annotate --overwrite -f - kms-encryption-timestamp="time value" ----- - -[WARNING] -==== - -If you enable https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/[secrets encryption] for an existing cluster and the KMS key that you use is ever deleted, then there's no way to recover the cluster. If you delete the KMS key, you permanently put the cluster in a degraded state. For more information, see link:kms/latest/developerguide/deleting-keys.html[Deleting {aws} KMS keys,type="documentation"]. - -==== - -[NOTE] -==== - -By default, the `create-key` command creates a link:kms/latest/developerguide/symmetric-asymmetric.html[symmetric encryption KMS key,type="documentation"] with a key policy that gives the account root admin access on {aws} KMS actions and resources. If you want to scope down the permissions, make sure that the `kms:DescribeKey` and `kms:CreateGrant` actions are permitted on the policy for the principal that calls the `create-cluster` API. - - -For clusters using KMS Envelope Encryption, `kms:CreateGrant` permissions are required. The condition `kms:GrantIsForAWSResource` is not supported for the CreateCluster action, and should not be used in KMS policies to control `kms:CreateGrant` permissions for users performing CreateCluster. - -==== - -[.topic] -[[manage-secrets,manage-secrets.title]] -== Use {aws} Secrets Manager secrets with Amazon EKS pods - -To show secrets from Secrets Manager and parameters from Parameter Store as files mounted in Amazon EKS [.noloc]`Pods`, you can use the {aws} Secrets and Configuration Provider (ASCP) for the https://secrets-store-csi-driver.sigs.k8s.io/[Kubernetes Secrets Store CSI Driver]. - -With the ASCP, you can store and manage your secrets in Secrets Manager and then retrieve them through your workloads running on Amazon EKS. You can use IAM roles and policies to limit access to your secrets to specific [.noloc]`Kubernetes` [.noloc]`Pods` in a cluster. The ASCP retrieves the [.noloc]`Pod` identity and exchanges the identity for an IAM role. ASCP assumes the IAM role of the [.noloc]`Pod`, and then it can retrieve secrets from Secrets Manager that are authorized for that role. +include::cert-signing.adoc[leveloffset=+1] -If you use Secrets Manager automatic rotation for your secrets, you can also use the Secrets Store CSI Driver rotation reconciler feature to ensure you are retrieving the latest secret from Secrets Manager. +include::default-roles-users.adoc[leveloffset=+1] -[NOTE] -==== +include::enable-kms.adoc[leveloffset=+1] -{aws} Fargate (Fargate) node groups are not supported. +include::manage-secrets.adoc[leveloffset=+1] -==== +include::envelope-encryption.adoc[leveloffset=+1] -For more information, see link:secretsmanager/latest/userguide/integrating_csi_driver.html[Using Secrets Manager secrets in Amazon EKS,type="documentation"] in the {aws} Secrets Manager User Guide. diff --git a/latest/ug/security/security.adoc b/latest/ug/security/security.adoc index b7db58015..e2e536162 100644 --- a/latest/ug/security/security.adoc +++ b/latest/ug/security/security.adoc @@ -1,36 +1,21 @@ -//!!NODE_ROOT +include::../attributes.txt[] + [.topic] -[[security,security.title]] +[#security] = Security in Amazon EKS -:doctype: book -:sectnums: -:toc: left -:icons: font -:experimental: -:idprefix: -:idseparator: - -:sourcedir: . -:info_doctype: chapter -:info_title: Security in Amazon EKS :info_titleabbrev: Security -:info_abstract: Configure Amazon EKS to meet your security and compliance objectives, and learn \ - how to use other {aws} services that help you to secure your Amazon EKS \ - resources. - -include::../attributes.txt[] [abstract] -- Configure Amazon EKS to meet your security and compliance objectives, and learn how to use other {aws} services that help you to secure your Amazon EKS resources. -- + Cloud security at {aws} is the highest priority. As an {aws} customer, you benefit from a data center and network architecture that is built to meet the requirements of the most security-sensitive organizations. Security is a shared responsibility between {aws} and you. The link:compliance/shared-responsibility-model/[shared responsibility model,type="marketing"] describes this as security _of_ the cloud and security _in_ the cloud: - - -* *Security of the cloud* – {aws} is responsible for protecting the infrastructure that runs {aws} services in the {aws} Cloud. For Amazon EKS, {aws} is responsible for the [.noloc]`Kubernetes` control plane, which includes the control plane nodes and `etcd` database. Third-party auditors regularly test and verify the effectiveness of our security as part of the link:compliance/programs/[{aws} compliance programs,type="marketing"]. To learn about the compliance programs that apply to Amazon EKS, see link:compliance/services-in-scope/[{aws} Services in Scope by Compliance Program,type="marketing"]. +* *Security of the cloud* – {aws} is responsible for protecting the infrastructure that runs {aws} services in the {aws} Cloud. For Amazon EKS, {aws} is responsible for the Kubernetes control plane, which includes the control plane nodes and `etcd` database. Third-party auditors regularly test and verify the effectiveness of our security as part of the link:compliance/programs/[{aws} compliance programs,type="marketing"]. To learn about the compliance programs that apply to Amazon EKS, see link:compliance/services-in-scope/[{aws} Services in Scope by Compliance Program,type="marketing"]. * *Security in the cloud* – Your responsibility includes the following areas. + ** The security configuration of the data plane, including the configuration of the security groups that allow traffic to pass from the Amazon EKS control plane into the customer VPC @@ -42,42 +27,30 @@ Security is a shared responsibility between {aws} and you. The link:compliance/s *** Managing platform-level identity and access management, either with or in addition to IAM ** The sensitivity of your data, your company's requirements, and applicable laws and regulations +Amazon EKS is certified by multiple compliance programs for regulated and sensitive applications. Amazon EKS is compliant with link:compliance/soc-faqs/[SOC,type="marketing"], link:compliance/pci-dss-level-1-faqs/[PCI,type="marketing"], link:compliance/iso-certified/[ISO,type="marketing"], link:compliance/fedramp/[FedRAMP-Moderate,type="marketing"], link:compliance/irap/[IRAP,type="marketing"], link:compliance/bsi-c5/[C5,type="marketing"], link:compliance/k-isms/[K-ISMS,type="marketing"], link:compliance/esquema-nacional-de-seguridad/[ENS High,type="marketing"], link:compliance/OSPAR/[OSPAR,type="marketing"], link:compliance/hitrust/[HITRUST CSF,type="marketing"], and is a link:compliance/hipaa-compliance/[HIPAA,type="marketing"] eligible service. For more information, see <>. + This documentation helps you understand how to apply the shared responsibility model when using Amazon EKS. The following topics show you how to configure Amazon EKS to meet your security and compliance objectives. You also learn how to use other {aws} services that help you to monitor and secure your Amazon EKS resources. [NOTE] ==== -[.noloc]`Linux` containers are made up of control groups (cgroups) and namespaces that help limit what a container can access, but all containers share the same [.noloc]`Linux` kernel as the host Amazon EC2 instance. Running a container as the root user (UID 0) or granting a container access to host resources or namespaces such as the host network or host PID namespace are strongly discouraged, because doing so reduces the effectiveness of the isolation that containers provide. +Linux containers are made up of control groups (cgroups) and namespaces that help limit what a container can access, but all containers share the same Linux kernel as the host Amazon EC2 instance. Running a container as the root user (UID 0) or granting a container access to host resources or namespaces such as the host network or host PID namespace are strongly discouraged, because doing so reduces the effectiveness of the isolation that containers provide. ==== [.topiclist] [[Topic List]] -include::configuration-vulnerability-analysis.adoc[leveloffset=+1] +include::security-best-practices.adoc[leveloffset=+1] +include::configuration-vulnerability-analysis.adoc[leveloffset=+1] include::compliance.adoc[leveloffset=+1] - include::security-eks.adoc[leveloffset=+1] - include::security-k8s.adoc[leveloffset=+1] - include::auto-security.adoc[leveloffset=+1] - -[.topic] -[[security-best-practices,security-best-practices.title]] -== Secure Amazon EKS clusters with best practices - -[abstract] --- -Learn how to secure your Amazon EKS clusters by following the best practices from the community. --- - -The Amazon EKS security best practices are in the link:eks/latest/best-practices/security.html[Best Practices for Security,type="documentation"] in the _Amazon EKS Best Practices Guide_. - include::iam-reference/security-iam.adoc[leveloffset=+1] diff --git a/latest/ug/security/vpc-interface-endpoints.adoc b/latest/ug/security/vpc-interface-endpoints.adoc new file mode 100644 index 000000000..b9d0917e0 --- /dev/null +++ b/latest/ug/security/vpc-interface-endpoints.adoc @@ -0,0 +1,82 @@ +include::../attributes.txt[] + +[.topic] +[#vpc-interface-endpoints] += Access Amazon EKS using {aws} PrivateLink +:info_titleabbrev: {aws} PrivateLink + +[abstract] +-- +Learn how to securely access Amazon Elastic Kubernetes Service (Amazon EKS) APIs from within your VPC using {aws} PrivateLink, avoiding public internet exposure while benefiting from private connectivity, routing optimization, and built-in security controls for enhanced cluster management. +-- + +You can use {aws} PrivateLink to create a private connection between your VPC and Amazon Elastic Kubernetes Service. You can access Amazon EKS as if it were in your VPC, without the use of an internet gateway, NAT device, VPN connection, or {aws} Direct Connect connection. Instances in your VPC don't need public IP addresses to access Amazon EKS. + +You establish this private connection by creating an interface endpoint powered by {aws} PrivateLink. We create an endpoint network interface in each subnet that you enable for the interface endpoint. These are requester-managed network interfaces that serve as the entry point for traffic destined for Amazon EKS. + +For more information, see link:vpc/latest/privatelink/privatelink-access-aws-services.html[Access {aws} services through {aws} PrivateLink,type="documentation"] in the _{aws} PrivateLink Guide_. + +[#vpc-endpoint-prerequisites] +== Before you begin + +Before you start, make sure you have performed the following tasks: + +* Review link:vpc/latest/privatelink/create-interface-endpoint.html#considerations-interface-endpoints[Access an {aws} service using an interface VPC endpoint,type="documentation"] in the _{aws} PrivateLink Guide_ + +[#vpc-endpoint-considerations] +== Considerations + +* *Support and Limitations*: Amazon EKS interface endpoints enable secure access to all Amazon EKS API actions from your VPC but come with specific limitations: they do not support access to Kubernetes APIs, as these have a separate private endpoint, you cannot configure Amazon EKS to be accessible only through the interface endpoint. +* *Pricing*: Using interface endpoints for Amazon EKS incurs standard {aws} PrivateLink charges: hourly charges for each endpoint provisioned in each Availability Zone, data processing charges for traffic through the endpoint. To learn more, see link:https://aws.amazon.com/privatelink/pricing/[{aws} PrivateLink pricing]. +* *Security and Access Control*: We recommend enhancing security and controlling access with these additional configurations—use VPC endpoint policies to control access to Amazon EKS through the interface endpoint, associate security groups with endpoint network interfaces to manage traffic, use VPC flow logs to capture and monitor IP traffic to and from the interface endpoints, with logs publishable to Amazon CloudWatch or Amazon S3. To learn more, see link:https://docs.aws.amazon.com/vpc/latest/privatelink/vpc-endpoints-access.html[Control access to VPC endpoints using endpoint policies] and link:https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs.html[Logging IP traffic using VPC Flow Logs]. +* *Connectivity Options*: Interface endpoints offer flexible connectivity options using *on-premises access* (connect your on-premises data center to a VPC with the interface endpoint using {aws} Direct Connect or {aws} Site-to-Site VPN) or via *inter-VPC connectivity* (use {aws} Transit Gateway or VPC peering to connect other VPCs to the VPC with the interface endpoint, keeping traffic within the {aws} network). +* *IP Version Support*: Endpoints created before August 2024 support only IPv4 using eks.region.amazonaws.com. New endpoints created after August 2024 support dual-stack IPv4 and IPv6 (e.g., eks.region.amazonaws.com, eks.region.api.aws). +* *Regional Availability*: {aws} PrivateLink for the EKS API is not available in Asia Pacific (Malaysia) (ap-southeast-5), Asia Pacific (Thailand) (ap-southeast-7), Mexico (Central) (mx-central-1), and Asia Pacific (Taipei) (ap-east-2) regions. {aws} PrivateLink support for eks-auth (EKS Pod Identity) is available in the Asia Pacific (Malaysia) (ap-southeast-5) region. + + +[#vpc-endpoint-create] +== Create an interface endpoint for Amazon EKS + +You can create an interface endpoint for Amazon EKS using either the Amazon VPC console or the {aws} Command Line Interface ({aws} CLI). For more information, see link:vpc/latest/privatelink/create-interface-endpoint.html#create-interface-endpoint-aws[Create a VPC endpoint,type="documentation"] in the _{aws} PrivateLink Guide_. + +Create an interface endpoint for Amazon EKS using the following service names: + +=== EKS API + +* com.amazonaws.region-code.eks +* com.amazonaws.region-code.eks-fips (for FIPS-compliant endpoints) + +=== EKS Auth API (EKS Pod Identity) + +* com.amazonaws.region-code.eks-auth + +[#vpc-endpoint-private-dns] +== Private DNS feature for Amazon EKS interface endpoints + +The private DNS feature, enabled by default for interface endpoints of Amazon EKS and other {aws} services, facilitates secure and private API requests using default Regional DNS names. This feature ensures that API calls are routed through the interface endpoint over the private {aws} network, enhancing security and performance. + +The private DNS feature activates automatically when you create an interface endpoint for Amazon EKS or other {aws} services. To enable, you need to configure your VPC correctly by setting specific attributes: + +* *enableDnsHostnames*: Allows instances within the VPC to have DNS hostnames. +* *enableDnsSupport*: Enables DNS resolution throughout the VPC. + +For step-by-step instructions to check or modify these settings, see link:https://docs.aws.amazon.com/vpc/latest/userguide/vpc-dns.html#vpc-dns-updating[View and update DNS attributes for your VPC]. + +=== DNS names and IP address types + +With the private DNS feature enabled, you can use specific DNS names to connect to Amazon EKS, and these options evolve over time: + +* *eks.region.amazonaws.com*: The traditional DNS name, resolving only to IPv4 addresses before August 2024. For existing endpoints updated to dual-stack, this name resolves to both IPv4 and IPv6 addresses. +* *eks.region.api.aws*: Available for new endpoints created after August 2024, this dual-stack DNS name resolves to both IPv4 and IPv6 addresses. + +After August 2024, new interface endpoints come with two DNS names, and you can opt for the dual-stack IP address type. For existing endpoints, updating to dual-stack modifies *eks.region.amazonaws.com* to support both IPv4 and IPv6. + +=== Using the Private DNS feature + +Once configured, the private DNS feature can be integrated into your workflows, offering the following capabilities: + +* *API Requests*: Use the default Regional DNS names, either `eks.region.amazonaws.com` or `eks.region.api.aws`, based on your endpoint's setup to make API requests to Amazon EKS. +* *Application Compatibility*: Your existing applications that call EKS APIs require no changes to leverage this feature. +* *{aws} CLI with Dual-Stack*: To use the dual-stack endpoints with the {aws} CLI, see the link:https://docs.aws.amazon.com/sdkref/latest/guide/feature-endpoints.html[Dual-stack and FIPS endpoints] configuration in the _{aws} SDKs and Tools Reference Guide_. +* *Automatic Routing*: Any call to the Amazon EKS default service endpoint is automatically directed through the interface endpoint, ensuring private and secure connectivity. + diff --git a/latest/ug/storage/csi-snapshot-controller.adoc b/latest/ug/storage/csi-snapshot-controller.adoc index 535880c62..8d5a6c1df 100644 --- a/latest/ug/storage/csi-snapshot-controller.adoc +++ b/latest/ug/storage/csi-snapshot-controller.adoc @@ -1,32 +1,23 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[csi-snapshot-controller,csi-snapshot-controller.title]] +[#csi-snapshot-controller] = Enable snapshot functionality for CSI volumes -:info_doctype: section -:info_title: Enable snapshot functionality for CSI volumes :info_titleabbrev: CSI snapshot controller -:keywords: CSI, snapshot, controller -:info_abstract: The Container Storage Interface (CSI) snapshot controller enables the use of \ - snapshot functionality in compatible CSI drivers, such as the Amazon EBS CSI \ - driver. [abstract] -- The Container Storage Interface (CSI) snapshot controller enables the use of snapshot functionality in compatible CSI drivers, such as the Amazon EBS CSI driver. -- -Snapshot functionality allows for point-in-time copies of your data. For this capability to work in [.noloc]`Kubernetes`, you need both a CSI driver with snapshot support (such as the Amazon EBS CSI driver) and a CSI snapshot controller. The snapshot controller is available either as an Amazon EKS managed add-on or as a self-managed installation. +Snapshot functionality allows for point-in-time copies of your data. For this capability to work in Kubernetes, you need both a CSI driver with snapshot support (such as the Amazon EBS CSI driver) and a CSI snapshot controller. The snapshot controller is available either as an Amazon EKS managed add-on or as a self-managed installation. Here are some things to consider when using the CSI snapshot controller. - - * The snapshot controller must be installed alongside a CSI driver with snapshot functionality. For installation instructions of the Amazon EBS CSI driver, see <>. -* [.noloc]`Kubernetes` doesn't support snapshots of volumes being served via CSI migration, such as Amazon EBS volumes using a `StorageClass` with provisioner `kubernetes.io/aws-ebs`. Volumes must be created with a `StorageClass` that references the CSI driver provisioner, `ebs.csi.aws.com`. For more information about CSI migration, see <>. +* Kubernetes doesn't support snapshots of volumes being served via CSI migration, such as Amazon EBS volumes using a `StorageClass` with provisioner `kubernetes.io/aws-ebs`. Volumes must be created with a `StorageClass` that references the CSI driver provisioner, `ebs.csi.aws.com`. * Amazon EKS Auto Mode does not include the snapshot controller. The storage capability of EKS Auto Mode is compatible with the snapshot controller. We recommend that you install the CSI snapshot controller through the Amazon EKS managed add-on. This add-on includes the custom resource definitions (CRDs) that are needed to create and manage snapshots on Amazon EKS. To add an Amazon EKS add-on to your cluster, see <>. For more information about add-ons, see <>. -Alternatively, if you want a self-managed installation of the CSI snapshot controller, see https://github.com/kubernetes-csi/external-snapshotter/blob/master/README.md#usage[Usage] in the upstream [.noloc]`Kubernetes` `external-snapshotter` on [.noloc]`GitHub`. +Alternatively, if you want a self-managed installation of the CSI snapshot controller, see https://github.com/kubernetes-csi/external-snapshotter/blob/master/README.md#usage[Usage] in the upstream Kubernetes `external-snapshotter` on GitHub. \ No newline at end of file diff --git a/latest/ug/storage/ebs-csi-migration-faq.adoc b/latest/ug/storage/ebs-csi-migration-faq.adoc deleted file mode 100644 index 892450906..000000000 --- a/latest/ug/storage/ebs-csi-migration-faq.adoc +++ /dev/null @@ -1,147 +0,0 @@ -//!!NODE_ROOT
-include::../attributes.txt[] - -[.topic] -[[ebs-csi-migration-faq,ebs-csi-migration-faq.title]] -= Amazon EBS CSI migration frequently asked questions -:info_doctype: section -:info_title: Amazon EBS CSI migration frequently asked \ - questions -:info_titleabbrev: EBS CSI migration FAQ -:keywords: Amazon EBS CSI driver, storage, CSI migration -:info_abstract: The Amazon EBS container storage interface migration feature is enabled by default on \ - Amazon EKS 1.23 and later clusters. Learn answers to frequently asked \ - questions about the feature and how it works with 1.23 and later \ - clusters. - -[abstract] --- -The Amazon EBS container storage interface migration feature is enabled by default on Amazon EKS `1.23` and later clusters. Learn answers to frequently asked questions about the feature and how it works with `1.23` and later clusters. --- - -[IMPORTANT] -==== - -If you have [.noloc]`Pods` running on a version `1.22` or earlier cluster, then you must install the Amazon EBS CSI driver (see <>) before updating your cluster to version `1.23` to avoid service interruption. - -==== - -The Amazon EBS container storage interface (CSI) migration feature moves responsibility for handling storage operations from the Amazon EBS in-tree EBS storage provisioner to the Amazon EBS CSI driver (see <>). - -[[csi-migration-faq-csi-drivers,csi-migration-faq-csi-drivers.title]] -== What are CSI drivers? - -CSI drivers: - - - -* Replace the [.noloc]`Kubernetes` "in-tree" storage drivers that exist in the [.noloc]`Kubernetes` project source code. -* Work with storage providers, such as Amazon EBS. -* Provide a simplified plugin model that make it easier for storage providers like {aws} to release features and maintain support without depending on the [.noloc]`Kubernetes` release cycle. - -For more information, see https://kubernetes-csi.github.io/docs/introduction.html[Introduction] in the [.noloc]`Kubernetes` CSI documentation. - -[[csi-migration-faq-what-is,csi-migration-faq-what-is.title]] -== What is CSI migration? - -The [.noloc]`Kubernetes` CSI Migration feature moves responsibility for handling storage operations from the existing in-tree storage plugins, such as `kubernetes.io/aws-ebs`, to corresponding CSI drivers. Existing `StorageClass`, `PersistentVolume` and `PersistentVolumeClaim` (PVC) objects continue to work, as long as the corresponding CSI driver is installed. When the feature is enabled: - - - -* Existing workloads that utilize PVCs continue to function as they always have. -* [.noloc]`Kubernetes` passes control of all storage management operations to CSI drivers. - -For more information, see https://kubernetes.io/blog/2021/12/10/storage-in-tree-to-csi-migration-status-update/[Kubernetes1.23: Kubernetes In-Tree to CSI Volume Migration Status Update] on the [.noloc]`Kubernetes` blog. - -To help you migrate from the in-tree plugin to CSI drivers, the `CSIMigration` and `CSIMigration{aws}` flags are enabled by default on Amazon EKS version `1.23` and later clusters. These flags enable your cluster to translate the in-tree APIs to their equivalent CSI APIs. These flags are set on the [.noloc]`Kubernetes` control plane managed by Amazon EKS and in the `kubelet` settings configured in Amazon EKS optimized AMIs. *If you have [.noloc]`Pods` using Amazon EBS volumes in your cluster, you must install the Amazon EBS CSI driver before updating your cluster to version `1.23`.* -If you don't, volume operations such as provisioning and mounting might not work as expected. For more information, see <>. - -[NOTE] -==== - -The in-tree `StorageClass` provisioner is named `kubernetes.io/aws-ebs`. The Amazon EBS CSI `StorageClass` provisioner is named `ebs.csi.aws.com`. - -==== - -[[csi-migration-faq-mounting-volumes,csi-migration-faq-mounting-volumes.title]] -== Can I mount `kubernetes.io/aws-ebs StorageClass` volumes in version `1.23` and later clusters? - -Yes, as long as the <> is installed. For newly created version `1.23` and later clusters, we recommend installing the Amazon EBS CSI driver as part of your cluster creation process. We also recommend only using `StorageClasses` based on the `ebs.csi.aws.com` provisioner. - -If you've updated your cluster control plane to version `1.23` and haven't yet updated your nodes to `1.23`, then the `CSIMigration` and `CSIMigration{aws}` kubelet flags aren't enabled. In this case, the in-tree driver is used to mount `kubernetes.io/aws-ebs` based volumes. The Amazon EBS CSI driver must still be installed however, to ensure that [.noloc]`Pods` using `kubernetes.io/aws-ebs` based volumes can be scheduled. The driver is also required for other volume operations to succeed. - -[[csi-migration-faq-aws-ebs-volumes,csi-migration-faq-aws-ebs-volumes.title]] -== Can I provision `kubernetes.io/aws-ebs StorageClass` volumes on Amazon EKS `1.23` and later clusters? - -Yes, as long as the <> is installed. - -[[csi-migration-faq-aws-ebs-provisioner,csi-migration-faq-aws-ebs-provisioner.title]] -== Will the `kubernetes.io/aws-ebs StorageClass` provisioner ever be removed from Amazon EKS? - -The `kubernetes.io/aws-ebs` `StorageClass` provisioner and `awsElasticBlockStore` volume type are no longer supported, but there are no plans to remove them. These resources are treated as a part of the [.noloc]`Kubernetes` API. - -[[csi-migration-faq-ebs-csi-driver,csi-migration-faq-ebs-csi-driver.title]] -== How do I install the Amazon EBS CSI driver? - -We recommend installing the <>. When an update is required to the Amazon EKS add-on, you initiate the update and Amazon EKS updates the add-on for you. If you want to manage the driver yourself, you can install it using the open source https://github.com/kubernetes-sigs/aws-ebs-csi-driver/tree/master/charts/aws-ebs-csi-driver[Helm chart]. - -[IMPORTANT] -==== - -The [.noloc]`Kubernetes` in-tree Amazon EBS driver runs on the [.noloc]`Kubernetes` control plane. It uses IAM permissions assigned to the <> to provision Amazon EBS volumes. The Amazon EBS CSI driver runs on nodes. The driver needs IAM permissions to provision volumes. For more information, see <>. - -==== - -[[csi-migration-faq-check-driver,csi-migration-faq-check-driver.title]] -== How can I check whether the Amazon EBS CSI driver is installed in my cluster? - -To determine whether the driver is installed on your cluster, run the following command: - -[source,bash,subs="verbatim,attributes"] ----- -kubectl get csidriver ebs.csi.aws.com ----- - -To check if that installation is managed by Amazon EKS, run the following command: - -[source,bash,subs="verbatim,attributes"] ----- -aws eks list-addons --cluster-name my-cluster ----- - - -[[csi-migration-faq-update-prevention,csi-migration-faq-update-prevention.title]] -== Will Amazon EKS prevent a cluster update to version `1.23` if I haven't already installed the Amazon EBS CSI driver? - -No. - -[[csi-migration-faq-driver-after-cluster-update,csi-migration-faq-driver-after-cluster-update.title]] -== What if I forget to install the Amazon EBS CSI driver before I update my cluster to version 1.23? Can I install the driver after updating my cluster? - -Yes, but volume operations requiring the Amazon EBS CSI driver will fail after your cluster update until the driver is installed. - -[[csi-migration-faq-default-storageclass,csi-migration-faq-default-storageclass.title]] -== What is the default `StorageClass` applied in newly created Amazon EKS version `1.23` and later clusters? - -The default `StorageClass` behavior remains unchanged. With each new cluster, Amazon EKS applies a `kubernetes.io/aws-ebs` based `StorageClass` named `gp2`. We don't plan to ever remove this `StorageClass` from newly created clusters. Separate from the cluster default `StorageClass`, if you create an `ebs.csi.aws.com` based `StorageClass` without specifying a volume type, the Amazon EBS CSI driver will default to using `gp3`. - -[[csi-migration-faq-existing-storageclasses,csi-migration-faq-existing-storageclasses.title]] -== Will Amazon EKS make any changes to `StorageClasses` already present in my existing cluster when I update my cluster to version `1.23`? - -No. - -[[csi-migration-faq-migrate-using-snapshots,csi-migration-faq-migrate-using-snapshots.title]] -== How do I migrate a persistent volume from the `kubernetes.io/aws-ebs` `StorageClass` to `ebs.csi.aws.com` using snapshots? - -To migrate a persistent volume, see link:containers/migrating-amazon-eks-clusters-from-gp2-to-gp3-ebs-volumes[Migrating Amazon EKS clusters from gp2 to gp3 EBS volumes,type="blog"] on the {aws} blog. - -[[csi-migration-faq-migrate-using-annotations,csi-migration-faq-migrate-using-annotations.title]] -== How do I modify an Amazon EBS volume using annotations? - -Starting with `aws-ebs-csi-driver` `v1.19.0-eksbuild.2`, you can modify Amazon EBS volumes using annotations within each `PersistentVolumeClaim` (PVC). -The new https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/docs/modify-volume.md[volume modification] feature is implemented as an additional sidecar, called `volumemodifier`. For more information, see link:storage/simplifying-amazon-ebs-volume-migration-and-modification-using-the-ebs-csi-driver[Simplifying Amazon EBS volume migration and modification on Kubernetes using the EBS CSI Driver,type="blog"] on the {aws} blog. - -[[csi-migration-faq-windows,csi-migration-faq-windows.title]] -== Is migration supported for Windows workloads? - -Yes. If you're installing the Amazon EBS CSI driver using the open source Helm chart, set `node.enableWindows` to `true`. This is set by default if installing the Amazon EBS CSI driver as an Amazon EKS add-on. When creating `StorageClasses`, set the `fsType` to a Windows file system, such as `ntfs`. Volume operations for Windows workloads are then migrated to the Amazon EBS CSI driver the same as they are for Linux workloads. diff --git a/latest/ug/storage/ebs-csi.adoc b/latest/ug/storage/ebs-csi.adoc index f7506a783..9dc0c0e0d 100644 --- a/latest/ug/storage/ebs-csi.adoc +++ b/latest/ug/storage/ebs-csi.adoc @@ -1,15 +1,9 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[ebs-csi,ebs-csi.title]] -= Store [.noloc]`Kubernetes` volumes with Amazon EBS -:info_doctype: section -:info_title: Store Kubernetes volumes with Amazon EBS +[#ebs-csi] += Use Kubernetes volume storage with Amazon EBS :info_titleabbrev: Amazon EBS -:keywords: Amazon EBS CSI driver, storage -:info_abstract: The Amazon Elastic Block Store (Amazon EBS) Container Storage Interface (CSI) driver manages the \ - lifecycle of Amazon EBS volumes as storage for Kubernetes Volumes. [abstract] -- @@ -22,17 +16,17 @@ The Amazon Elastic Block Store (Amazon EBS) Container Storage Interface (CSI) dr ==== -The https://github.com/kubernetes-sigs/aws-ebs-csi-driver/[Amazon Elastic Block Store (Amazon EBS) Container Storage Interface (CSI) driver] manages the lifecycle of Amazon EBS volumes as storage for the Kubernetes Volumes that you create. The Amazon EBS CSI driver makes Amazon EBS volumes for these types of [.noloc]`Kubernetes` volumes: generic https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/[ephemeral volumes] and https://kubernetes.io/docs/concepts/storage/persistent-volumes/[persistent volumes]. +The https://github.com/kubernetes-sigs/aws-ebs-csi-driver/[Amazon Elastic Block Store (Amazon EBS) Container Storage Interface (CSI) driver] manages the lifecycle of Amazon EBS volumes as storage for the Kubernetes Volumes that you create. The Amazon EBS CSI driver makes Amazon EBS volumes for these types of Kubernetes volumes: generic https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/[ephemeral volumes] and https://kubernetes.io/docs/concepts/storage/persistent-volumes/[persistent volumes]. -[[ebs-csi-considerations,ebs-csi-considerations.title]] +[#ebs-csi-considerations] == Considerations * You do not need to install the Amazon EBS CSI controller on EKS Auto Mode clusters. -* You can't mount Amazon EBS volumes to Fargate [.noloc]`Pods`. -* You can run the Amazon EBS CSI controller on Fargate nodes, but the Amazon EBS CSI node [.noloc]`DaemonSet` can only run on Amazon EC2 instances. +* You can't mount Amazon EBS volumes to Fargate Pods. +* You can run the Amazon EBS CSI controller on Fargate nodes, but the Amazon EBS CSI node `DaemonSet` can only run on Amazon EC2 instances. * Amazon EBS volumes and the Amazon EBS CSI driver are not compatible with Amazon EKS Hybrid Nodes. * Support will be provided for the latest add-on version and one prior version. Bugs or vulnerabilities found in the latest version will be backported to the previous release in a new minor version. - +* EKS Auto Mode requires storage classes to use `ebs.csi.eks.amazonaws.com` as the provisioner. The standard Amazon EBS CSI Driver (`ebs.csi.aws.com`) manages its own volumes separately. To use existing volumes with EKS Auto Mode, migrate them using volume snapshots to a storage class that uses the Auto Mode provisioner. [IMPORTANT] ==== @@ -41,7 +35,7 @@ To use the snapshot functionality of the Amazon EBS CSI driver, you must first i ==== -[[ebs-csi-prereqs,ebs-csi-prereqs.title]] +[#ebs-csi-prereqs] == Prerequisites * An existing cluster. To see the required platform version, run the following command. @@ -50,19 +44,19 @@ To use the snapshot functionality of the Amazon EBS CSI driver, you must first i ---- aws eks describe-addon-versions --addon-name aws-ebs-csi-driver ---- -* An existing {aws} Identity and Access Management (IAM) [.noloc]`OpenID Connect` ([.noloc]`OIDC`) provider for your cluster. To determine whether you already have one, or to create one, see <>. -* If you're using a cluster wide restricted <>, make sure that the add-on is granted sufficient permissions to be deployed. For the permissions required by each add-on [.noloc]`Pod`, see the https://github.com/kubernetes-sigs/aws-ebs-csi-driver/tree/master/deploy/kubernetes/base[relevant add-on manifest definition] on GitHub. - +* The EBS CSI driver needs {aws} IAM Permissions. +** {aws} suggests using EKS Pod Identities. For more information, see <>. +** For information about IAM Roles for Service Accounts, see <>. -[[csi-iam-role,csi-iam-role.title]] +[#csi-iam-role] == Step 1: Create an IAM role -The Amazon EBS CSI plugin requires IAM permissions to make calls to {aws} APIs on your behalf. If you don't do these steps, attempting to install the add-on and running `kubectl describe pvc` will show `failed to provision volume with StorageClass` along with a `could not create volume in EC2: UnauthorizedOperation` error. For more information, see https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/docs/install.md#set-up-driver-permissions[Set up driver permission] on [.noloc]`GitHub`. +The Amazon EBS CSI plugin requires IAM permissions to make calls to {aws} APIs on your behalf. If you don't do these steps, attempting to install the add-on and running `kubectl describe pvc` will show `failed to provision volume with StorageClass` along with a `could not create volume in EC2: UnauthorizedOperation` error. For more information, see https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/docs/install.md#set-up-driver-permissions[Set up driver permission] on GitHub. [NOTE] ==== -[.noloc]`Pods` will have access to the permissions that are assigned to the IAM role unless you block access to IMDS. For more information, see <>. +Pods will have access to the permissions that are assigned to the IAM role unless you block access to IMDS. For more information, see <>. ==== @@ -72,16 +66,17 @@ The following procedure shows you how to create an IAM role and attach the {aws} * <> * <> +NOTE: You can create a self-managed policy with scoped-down permissions. Review link:aws-managed-policy/latest/reference/AmazonEBSCSIDriverPolicy.html[`AmazonEBSCSIDriverPolicy`,type="documentation"] and create a custom IAM Policy with reduced permissions. [NOTE] ==== -The specific steps in this procedure are written for using the driver as an Amazon EKS add-on. Different steps are needed to use the driver as a self-managed add-on. For more information, see https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/docs/install.md#set-up-driver-permissions[Set up driver permissions] on [.noloc]`GitHub`. +The specific steps in this procedure are written for using the driver as an Amazon EKS add-on. Different steps are needed to use the driver as a self-managed add-on. For more information, see https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/docs/install.md#set-up-driver-permissions[Set up driver permissions] on GitHub. ==== === `eksctl` [[eksctl_store_app_data]] -. Create an IAM role and attach a policy. {aws} maintains an {aws} managed policy or you can create your own custom policy. You can create an IAM role and attach the {aws} managed policy with the following command. Replace [.replaceable]`my-cluster` with the name of your cluster. The command deploys an {aws} CloudFormation stack that creates an IAM role and attaches the IAM policy to it. If your cluster is in the {aws} GovCloud (US-East) or {aws} GovCloud (US-West) {aws} Regions, then replace `arn:aws:` with `arn:aws-us-gov:`. +. Create an IAM role and attach a policy. {aws} maintains an {aws} managed policy or you can create your own custom policy. You can create an IAM role and attach the {aws} managed policy with the following command. Replace [.replaceable]`my-cluster` with the name of your cluster. The command deploys an {aws} CloudFormation stack that creates an IAM role and attaches the IAM policy to it. + [source,bash,subs="verbatim,attributes"] ---- @@ -94,42 +89,13 @@ eksctl create iamserviceaccount \ --attach-policy-arn {arn-aws}iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy \ --approve ---- -. If you use a custom link:kms/[KMS key,type="marketing"] for encryption on your Amazon EBS volumes, customize the IAM role as needed. For example, do the following: +. You can skip this step if you do not use a custom link:kms/[KMS key,type="marketing"]. If you use one for encryption on your Amazon EBS volumes, customize the IAM role as needed. For example, do the following: + -.. Copy and paste the following code into a new `kms-key-for-encryption-on-ebs.json` file. Replace [.replaceable]`custom-key-arn` with the custom link:service-authorization/latest/reference/list_awskeymanagementservice.html#awskeymanagementservice-key[KMS key ARN,type="documentation"]. +.. Copy and paste the following code into a new `kms-key-for-encryption-on-ebs.json` file. Replace [.replaceable]`custom-key-arn` with the custom link:service-authorization/latest/reference/list_awskeymanagementservice.html#awskeymanagementservice-key[KMS key ARN,type="documentation"]. + [source,json,subs="verbatim,attributes"] ---- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "kms:CreateGrant", - "kms:ListGrants", - "kms:RevokeGrant" - ], - "Resource": ["custom-key-arn"], - "Condition": { - "Bool": { - "kms:GrantIsForAWSResource": "true" - } - } - }, - { - "Effect": "Allow", - "Action": [ - "kms:Encrypt", - "kms:Decrypt", - "kms:ReEncrypt*", - "kms:GenerateDataKey*", - "kms:DescribeKey" - ], - "Resource": ["custom-key-arn"] - } - ] - } +include::iam_snippet:11e5f21f-2972-429c-98c3-1b7f7389c7b2[] ---- .. Create the policy. You can change [.replaceable]`KMS_Key_For_Encryption_On_EBS_Policy` to a different name. However, if you do, make sure to change it in later steps, too. + @@ -139,7 +105,7 @@ aws iam create-policy \ --policy-name KMS_Key_For_Encryption_On_EBS_Policy \ --policy-document file://kms-key-for-encryption-on-ebs.json ---- -.. Attach the IAM policy to the role with the following command. Replace [.replaceable]`111122223333` with your account ID. If your cluster is in the {aws} GovCloud (US-East) or {aws} GovCloud (US-West) {aws} Regions, then replace `arn:aws:` with `arn:aws-us-gov:`. +.. Attach the IAM policy to the role with the following command. Replace [.replaceable]`111122223333` with your account ID. + [source,bash,subs="verbatim,attributes"] ---- @@ -151,11 +117,11 @@ aws iam attach-role-policy \ === {aws-management-console} [[console_store_app_data]] . Open the IAM console at https://console.aws.amazon.com/iam/. . In the left navigation pane, choose *Roles*. -. On the *Roles* page, choose *Create role*. +. On the *Roles* page, choose *Create role*. . On the *Select trusted entity* page, do the following: + -.. In the *Trusted entity type* section, choose *Web identity*. -.. For *Identity provider*, choose the *[.noloc]`OpenID Connect` provider URL* for your cluster (as shown under *Overview* in Amazon EKS). +.. In the *Trusted entity type* section, choose *Web identity*. +.. For *Identity provider*, choose the *OpenID Connect provider URL* for your cluster (as shown under *Overview* in Amazon EKS). .. For *Audience*, choose `sts.amazonaws.com`. .. Choose *Next*. . On the *Add permissions* page, do the following: @@ -166,10 +132,10 @@ aws iam attach-role-policy \ . On the *Name, review, and create* page, do the following: + .. For *Role name*, enter a unique name for your role, such as [.replaceable]`AmazonEKS_EBS_CSI_DriverRole`. -.. Under *Add tags (Optional)*, add metadata to the role by attaching tags as key-value pairs. For more information about using tags in IAM, see link:IAM/latest/UserGuide/id_tags.html[Tagging IAM resources,type="documentation"] in the _IAM User Guide_. +.. Under *Add tags (Optional)*, add metadata to the role by attaching tags as key-value pairs. For more information about using tags in IAM, see link:IAM/latest/UserGuide/id_tags.html[Tagging IAM resources,type="documentation"] in the _IAM User Guide_. .. Choose *Create role*. . After the role is created, choose the role in the console to open it for editing. -. Choose the *Trust relationships* tab, and then choose *Edit trust policy*. +. Choose the *Trust relationships* tab, and then choose *Edit trust policy*. . Find the line that looks similar to the following line: + [source,json,subs="verbatim,attributes"] @@ -187,50 +153,21 @@ Add a comma to the end of the previous line, and then add the following line aft . If you use a custom link:kms/[KMS key,type="marketing"] for encryption on your Amazon EBS volumes, customize the IAM role as needed. For example, do the following: + .. In the left navigation pane, choose *Policies*. -.. On the *Policies* page, choose *Create Policy*. -.. On the *Create policy* page, choose the *JSON* tab. -.. Copy and paste the following code into the editor, replacing [.replaceable]`custom-key-arn` with the custom link:service-authorization/latest/reference/list_awskeymanagementservice.html#awskeymanagementservice-key[KMS key ARN,type="documentation"]. +.. On the *Policies* page, choose *Create Policy*. +.. On the *Create policy* page, choose the *JSON* tab. +.. Copy and paste the following code into the editor, replacing [.replaceable]`custom-key-arn` with the custom link:service-authorization/latest/reference/list_awskeymanagementservice.html#awskeymanagementservice-key[KMS key ARN,type="documentation"]. + [source,json,subs="verbatim,attributes"] ---- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "kms:CreateGrant", - "kms:ListGrants", - "kms:RevokeGrant" - ], - "Resource": ["custom-key-arn"], - "Condition": { - "Bool": { - "kms:GrantIsForAWSResource": "true" - } - } - }, - { - "Effect": "Allow", - "Action": [ - "kms:Encrypt", - "kms:Decrypt", - "kms:ReEncrypt*", - "kms:GenerateDataKey*", - "kms:DescribeKey" - ], - "Resource": ["custom-key-arn"] - } - ] - } +include::iam_snippet:4bc92ef3-131e-44d3-ab0d-f3f1275e885a[] ---- .. Choose *Next: Tags*. -.. On the *Add tags (Optional)* page, choose *Next: Review*. +.. On the *Add tags (Optional)* page, choose *Next: Review*. .. For *Name*, enter a unique name for your policy (for example, [.replaceable]`KMS_Key_For_Encryption_On_EBS_Policy`). .. Choose *Create policy*. .. In the left navigation pane, choose *Roles*. .. Choose the *[.replaceable]`AmazonEKS_EBS_CSI_DriverRole`* in the console to open it for editing. -.. From the *Add permissions* dropdown list, choose *Attach policies*. +.. From the *Add permissions* dropdown list, choose *Attach policies*. .. In the *Filter policies* box, enter [.replaceable]`KMS_Key_For_Encryption_On_EBS_Policy`. .. Select the check box to the left of the [.replaceable]`KMS_Key_For_Encryption_On_EBS_Policy` that was returned in the search. .. Choose *Attach policies*. @@ -251,28 +188,11 @@ https://oidc.eks.region-code.amazonaws.com/id/EXAMPLED539D4633E53DE1B71EXAMPLE ---- . Create the IAM role, granting the `AssumeRoleWithWebIdentity` action. + -.. Copy the following contents to a file that's named `aws-ebs-csi-driver-trust-policy.json`. Replace [.replaceable]`111122223333` with your account ID. Replace [.replaceable]`EXAMPLED539D4633E53DE1B71EXAMPLE` and [.replaceable]`region-code` with the values returned in the previous step. If your cluster is in the {aws} GovCloud (US-East) or {aws} GovCloud (US-West) {aws} Regions, then replace `arn:aws:` with `arn:aws-us-gov:`. +.. Copy the following contents to a file that's named `aws-ebs-csi-driver-trust-policy.json`. Replace [.replaceable]`111122223333` with your account ID. Replace [.replaceable]`EXAMPLED539D4633E53DE1B71EXAMPLE` and [.replaceable]`region-code` with the values returned in the previous step. + [source,json,subs="verbatim,attributes"] ---- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "{arn-aws}iam::111122223333:oidc-provider/oidc.eks.region-code.amazonaws.com/id/EXAMPLED539D4633E53DE1B71EXAMPLE" - }, - "Action": "sts:AssumeRoleWithWebIdentity", - "Condition": { - "StringEquals": { - "oidc.eks.region-code.amazonaws.com/id/EXAMPLED539D4633E53DE1B71EXAMPLE:aud": "sts.amazonaws.com", - "oidc.eks.region-code.amazonaws.com/id/EXAMPLED539D4633E53DE1B71EXAMPLE:sub": "system:serviceaccount:kube-system:ebs-csi-controller-sa" - } - } - } - ] - } +include::iam_snippet:f27fa2c8-e511-47ab-8b50-3c357f34de4e[] ---- .. Create the role. You can change [.replaceable]`AmazonEKS_EBS_CSI_DriverRole` to a different name. If you change it, make sure to change it in later steps. + @@ -282,7 +202,7 @@ aws iam create-role \ --role-name AmazonEKS_EBS_CSI_DriverRole \ --assume-role-policy-document file://"aws-ebs-csi-driver-trust-policy.json" ---- -. Attach a policy. {aws} maintains an {aws} managed policy or you can create your own custom policy. Attach the {aws} managed policy to the role with the following command. If your cluster is in the {aws} GovCloud (US-East) or {aws} GovCloud (US-West) {aws} Regions, then replace `arn:aws:` with `arn:aws-us-gov:`. +. Attach a policy. {aws} maintains an {aws} managed policy or you can create your own custom policy. Attach the {aws} managed policy to the role with the following command. + [source,bash,subs="verbatim,attributes"] ---- @@ -292,40 +212,11 @@ aws iam attach-role-policy \ ---- . If you use a custom link:kms/[KMS key,type="marketing"] for encryption on your Amazon EBS volumes, customize the IAM role as needed. For example, do the following: + -.. Copy and paste the following code into a new `kms-key-for-encryption-on-ebs.json` file. Replace [.replaceable]`custom-key-arn` with the custom link:service-authorization/latest/reference/list_awskeymanagementservice.html#awskeymanagementservice-key[KMS key ARN,type="documentation"]. +.. Copy and paste the following code into a new `kms-key-for-encryption-on-ebs.json` file. Replace [.replaceable]`custom-key-arn` with the custom link:service-authorization/latest/reference/list_awskeymanagementservice.html#awskeymanagementservice-key[KMS key ARN,type="documentation"]. + [source,json,subs="verbatim,attributes"] ---- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "kms:CreateGrant", - "kms:ListGrants", - "kms:RevokeGrant" - ], - "Resource": ["custom-key-arn"], - "Condition": { - "Bool": { - "kms:GrantIsForAWSResource": "true" - } - } - }, - { - "Effect": "Allow", - "Action": [ - "kms:Encrypt", - "kms:Decrypt", - "kms:ReEncrypt*", - "kms:GenerateDataKey*", - "kms:DescribeKey" - ], - "Resource": ["custom-key-arn"] - } - ] - } +include::iam_snippet:c3dc1e73-97cf-493e-b828-eb4236aa3d0b[] ---- .. Create the policy. You can change [.replaceable]`KMS_Key_For_Encryption_On_EBS_Policy` to a different name. However, if you do, make sure to change it in later steps, too. + @@ -335,7 +226,7 @@ aws iam create-policy \ --policy-name KMS_Key_For_Encryption_On_EBS_Policy \ --policy-document file://kms-key-for-encryption-on-ebs.json ---- -.. Attach the IAM policy to the role with the following command. Replace [.replaceable]`111122223333` with your account ID. If your cluster is in the {aws} GovCloud (US-East) or {aws} GovCloud (US-West) {aws} Regions, then replace `arn:aws:` with `arn:aws-us-gov:`. +.. Attach the IAM policy to the role with the following command. Replace [.replaceable]`111122223333` with your account ID. + [source,bash,subs="verbatim,attributes"] ---- @@ -344,9 +235,9 @@ aws iam attach-role-policy \ --role-name AmazonEKS_EBS_CSI_DriverRole ---- -Now that you have created the Amazon EBS CSI driver IAM role, you can continue to the next section. When you deploy the add-on with this IAM role, it creates and is configured to use a service account that's named `ebs-csi-controller-sa`. The service account is bound to a [.noloc]`Kubernetes` `clusterrole` that's assigned the required [.noloc]`Kubernetes` permissions. +Now that you have created the Amazon EBS CSI driver IAM role, you can continue to the next section. When you deploy the add-on with this IAM role, it creates and is configured to use a service account that's named `ebs-csi-controller-sa`. The service account is bound to a Kubernetes `clusterrole` that's assigned the required Kubernetes permissions. -[[managing-ebs-csi,managing-ebs-csi.title]] +[#managing-ebs-csi] == Step 2: Get the Amazon EBS CSI driver We recommend that you install the Amazon EBS CSI driver through the Amazon EKS add-on to improve security and reduce the amount of work. To add an Amazon EKS add-on to your cluster, see <>. For more information about add-ons, see <>. @@ -354,13 +245,22 @@ We recommend that you install the Amazon EBS CSI driver through the Amazon EKS a [IMPORTANT] ==== -Before adding the Amazon EBS driver as an Amazon EKS add-on, confirm that you don't have a self-managed version of the driver installed on your cluster. If so, see https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/docs/install.md#uninstalling-the-ebs-csi-driver[Uninstalling a self-managed Amazon EBS CSI driver] on [.noloc]`GitHub`. +Before adding the Amazon EBS driver as an Amazon EKS add-on, confirm that you don't have a self-managed version of the driver installed on your cluster. If so, see https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/docs/install.md#uninstalling-the-ebs-csi-driver[Uninstalling a self-managed Amazon EBS CSI driver] on GitHub. + +==== + +[NOTE] +==== + +By default, the RBAC role used by the EBS CSI has permissions to mutate nodes to support its taint removal feature. Due to limitations of Kubernetes RBAC, this also allows it to mutate any other Node in the cluster. +The Helm chart has a parameter (`node.serviceAccount.disableMutation`) that disables mutating Node RBAC permissions for the ebs-csi-node service account. When enabled, driver features such as taint removal will not function. ==== -Alternatively, if you want a self-managed installation of the Amazon EBS CSI driver, see https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/docs/install.md[Installation] on [.noloc]`GitHub`. -[[ebs-sample-app,ebs-sample-app.title]] +Alternatively, if you want a self-managed installation of the Amazon EBS CSI driver, see https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/docs/install.md[Installation] on GitHub. + +[#ebs-sample-app] == Step 3: Deploy a sample application -You can deploy a variety of sample apps and modify them as needed. For more information, see https://github.com/kubernetes-sigs/aws-ebs-csi-driver/tree/master/examples/kubernetes[Kubernetes Examples] on [.noloc]`GitHub`. +You can deploy a variety of sample apps and modify them as needed. For more information, see https://github.com/kubernetes-sigs/aws-ebs-csi-driver/tree/master/examples/kubernetes[Kubernetes Examples] on GitHub. diff --git a/latest/ug/storage/efs-csi.adoc b/latest/ug/storage/efs-csi.adoc index 50dcbd07d..d7101d923 100644 --- a/latest/ug/storage/efs-csi.adoc +++ b/latest/ug/storage/efs-csi.adoc @@ -1,46 +1,47 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[efs-csi,efs-csi.title]] -= Store an elastic file system with Amazon EFS -:info_doctype: section -:info_title: Store an elastic file system with Amazon EFS +[#efs-csi] += Use elastic file system storage with Amazon EFS :info_titleabbrev: Amazon EFS -:keywords: Amazon EFS CSI driver, storage -:info_abstract: The Amazon EFS Container Storage Interface (CSI) driver provides a CSI interface that \ - allows Kubernetes clusters running on {aws} to manage the lifecycle of Amazon EFS file \ - systems. [abstract] -- -The Amazon EFS Container Storage Interface (CSI) driver provides a CSI interface that allows [.noloc]`Kubernetes` clusters running on {aws} to manage the lifecycle of Amazon EFS file systems. +The Amazon EFS Container Storage Interface (CSI) driver provides a CSI interface that allows Kubernetes clusters running on {aws} to manage the lifecycle of Amazon EFS file systems. -- -link:efs/latest/ug/whatisefs.html[Amazon Elastic File System,type="documentation"] (Amazon EFS) provides serverless, fully elastic file storage so that you can share file data without provisioning or managing storage capacity and performance. The https://github.com/kubernetes-sigs/aws-efs-csi-driver[Amazon EFS Container Storage Interface (CSI) driver] provides a CSI interface that allows [.noloc]`Kubernetes` clusters running on {aws} to manage the lifecycle of Amazon EFS file systems. This topic shows you how to deploy the Amazon EFS CSI driver to your Amazon EKS cluster. +link:efs/latest/ug/whatisefs.html[Amazon Elastic File System,type="documentation"] (Amazon EFS) provides serverless, fully elastic file storage so that you can share file data without provisioning or managing storage capacity and performance. The https://github.com/kubernetes-sigs/aws-efs-csi-driver[Amazon EFS Container Storage Interface (CSI) driver] provides a CSI interface that allows Kubernetes clusters running on {aws} to manage the lifecycle of Amazon EFS file systems. This topic shows you how to deploy the Amazon EFS CSI driver to your Amazon EKS cluster. -[[efs-csi.considerations,efs-csi.considerations.title]] +[#efs-csi-considerations] == Considerations * The Amazon EFS CSI driver isn't compatible with Windows-based container images. * You can't use https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/examples/kubernetes/dynamic_provisioning/README.md[dynamic provisioning] for persistent volumes with Fargate nodes, but you can use https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/examples/kubernetes/static_provisioning/README.md[static provisioning]. -* https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/examples/kubernetes/dynamic_provisioning/README.md[Dynamic provisioning] requires https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/CHANGELOG-1.x.md#v12[1.2] or later of the driver. You can use https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/examples/kubernetes/static_provisioning/README.md[static provisioning] for persistent volumes using version `1.1` of the driver on any supported Amazon EKS cluster version (see <>). +* https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/examples/kubernetes/dynamic_provisioning/README.md[Dynamic provisioning] requires https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/CHANGELOG-1.x.md#v12[1.2] or later of the driver. You can use https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/examples/kubernetes/static_provisioning/README.md[static provisioning] for persistent volumes using version `1.1` of the driver on any supported Amazon EKS cluster version (see link:eks/latest/userguide/kubernetes-versions.html[Amazon EKS supported versions,type="documentation"]). * Version https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/CHANGELOG-1.x.md#v132[1.3.2] or later of this driver supports the Arm64 architecture, including Amazon EC2 Graviton-based instances. * Version https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/CHANGELOG-1.x.md#v142[1.4.2] or later of this driver supports using FIPS for mounting file systems. -* Take note of the resource quotas for Amazon EFS. For example, there's a quota of 1000 access points that can be created for each Amazon EFS file system. For more information, see link:efs/latest/ug/limits.html#limits-efs-resources-per-account-per-region[Amazon EFS resource quotas that you cannot change,type="documentation"]. +* Take note of the resource quotas for Amazon EFS. For example, there's a quota of 1000 access points that can be created for each Amazon EFS file system. For more information, see link:efs/latest/ug/limits.html#limits-efs-resources-per-account-per-region[Amazon EFS resource quotas that you cannot change,type="documentation"]. * Starting in version https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/CHANGELOG-2.x.md#v200[2.0.0], this driver switched from using `stunnel` to `efs-proxy` for TLS connections. When `efs-proxy` is used, it will open a number of threads equal to one plus the number of cores for the node it's running on. * The Amazon EFS CSI driver isn't compatible with Amazon EKS Hybrid Nodes. -[[efs-csi.prereqs,efs-csi.prereqs.title]] +[#efs-csi-prereqs] == Prerequisites -* An existing {aws} Identity and Access Management (IAM) [.noloc]`OpenID Connect` ([.noloc]`OIDC`) provider for your cluster. To determine whether you already have one, or to create one, see <>. -* Version `2.12.3` or later or version `1.27.160` or later of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device or {aws} CloudShell. To check your current version, use `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or [.noloc]`Homebrew` for [.noloc]`macOS` are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the _{aws} Command Line Interface User Guide_. The {aws} CLI version that is installed in {aws} CloudShell might also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the _{aws} CloudShell User Guide_. -* The `kubectl` command line tool is installed on your device or {aws} CloudShell. The version can be the same as or up to one minor version earlier or later than the [.noloc]`Kubernetes` version of your cluster. For example, if your cluster version is `1.29`, you can use `kubectl` version `1.28`, `1.29`, or `1.30` with it. To install or upgrade `kubectl`, see <>. +* The Amazon EFS CSI driver needs {aws} Identity and Access Management (IAM) permissions. +** {aws} suggests using EKS Pod Identities. For more information, see <>. +** For information about IAM roles for service accounts and setting up an IAM OpenID Connect (OIDC) provider for your cluster, see <>. +* Version `2.12.3` or later or version `1.27.160` or later of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device or {aws} CloudShell. To check your current version, use `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or Homebrew for macOS are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the _{aws} Command Line Interface User Guide_. The {aws} CLI version that is installed in {aws} CloudShell might also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the _{aws} CloudShell User Guide_. +* The `kubectl` command line tool is installed on your device or {aws} CloudShell. The version can be the same as or up to one minor version earlier or later than the Kubernetes version of your cluster. For example, if your cluster version is `1.29`, you can use `kubectl` version `1.28`, `1.29`, or `1.30` with it. To install or upgrade `kubectl`, see <>. +[NOTE] +==== + +A Pod running on Fargate automatically mounts an Amazon EFS file system, without needing manual driver installation steps. + +==== -[[efs-create-iam-resources,efs-create-iam-resources.title]] +[#efs-create-iam-resources] == Step 1: Create an IAM role The Amazon EFS CSI driver requires IAM permissions to interact with your file system. Create an IAM role and attach the required {aws} managed policy to it. To implement this procedure, you can use one of these tools: @@ -52,12 +53,34 @@ The Amazon EFS CSI driver requires IAM permissions to interact with your file sy [NOTE] ==== -The specific steps in this procedure are written for using the driver as an Amazon EKS add-on. For details on self-managed installations, see https://github.com/kubernetes-sigs/aws-efs-csi-driver#set-up-driver-permission[Set up driver permission] on [.noloc]`GitHub`. +The specific steps in this procedure are written for using the driver as an Amazon EKS add-on. For details on self-managed installations, see https://github.com/kubernetes-sigs/aws-efs-csi-driver#set-up-driver-permission[Set up driver permission] on GitHub. ==== -=== `eksctl` [[eksctl_efs_store_app_data]] -Run the following commands to create an IAM role with `eksctl`. Replace [.replaceable]`my-cluster` with your cluster name and [.replaceable]`AmazonEKS_EFS_CSI_DriverRole` with the name for your role. +[#eksctl_efs_store_app_data] +=== `eksctl` + +[#efs-eksctl-pod-identities] +==== If using Pod Identities + +Run the following commands to create an IAM role and Pod Identity association with `eksctl`. Replace `my-cluster` with your cluster name. You can also replace `AmazonEKS_EFS_CSI_DriverRole` with a different name. + +[source,bash,subs="verbatim,attributes"] +---- +export cluster_name=my-cluster +export role_name=AmazonEKS_EFS_CSI_DriverRole +eksctl create podidentityassociation \ + --service-account-name efs-csi-controller-sa \ + --namespace kube-system \ + --cluster $cluster_name \ + --role-name $role_name \ + --permission-policy-arns {arn-aws}iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy +---- + +[#efs-eksctl-irsa] +==== If using IAM roles for service accounts + +Run the following commands to create an IAM role with `eksctl`. Replace `my-cluster` with your cluster name. You can also replace `AmazonEKS_EFS_CSI_DriverRole` with a different name. [source,bash,subs="verbatim,attributes"] ---- @@ -71,131 +94,173 @@ eksctl create iamserviceaccount \ --role-only \ --attach-policy-arn {arn-aws}iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy \ --approve -TRUST_POLICY=$(aws iam get-role --role-name $role_name --query 'Role.AssumeRolePolicyDocument' | \ +TRUST_POLICY=$(aws iam get-role --output json --role-name $role_name --query 'Role.AssumeRolePolicyDocument' | \ sed -e 's/efs-csi-controller-sa/efs-csi-*/' -e 's/StringEquals/StringLike/') aws iam update-assume-role-policy --role-name $role_name --policy-document "$TRUST_POLICY" ---- -=== {aws-management-console} [[console_efs_store_app_data]] +[#console_efs_store_app_data] +=== {aws-management-console} Run the following to create an IAM role with {aws-management-console}. . Open the IAM console at https://console.aws.amazon.com/iam/. . In the left navigation pane, choose *Roles*. -. On the *Roles* page, choose *Create role*. +. On the *Roles* page, choose *Create role*. . On the *Select trusted entity* page, do the following: -+ -.. In the *Trusted entity type* section, choose *Web identity*. -.. For *Identity provider*, choose the *[.noloc]`OpenID Connect` provider URL* for your cluster (as shown under *Overview* in Amazon EKS). -.. For *Audience*, choose `sts.amazonaws.com`. -.. Choose *Next*. +.. If using EKS Pod Identities: +... In the *Trusted entity type* section, choose *{aws} service*. +... In the *Service or use case* drop down, choose *EKS*. +... In the *Use case* section, choose *EKS - Pod Identity*. +... Choose *Next*. +.. If using IAM roles for service accounts: +... In the *Trusted entity type* section, choose *Web identity*. +... For *Identity provider*, choose the *OpenID Connect provider URL* for your cluster (as shown under *Overview* in Amazon EKS). +... For *Audience*, choose `sts.amazonaws.com`. +... Choose *Next*. . On the *Add permissions* page, do the following: -+ -.. In the *Filter policies* box, enter [.replaceable]`AmazonEFSCSIDriverPolicy`. -.. Select the check box to the left of the [.replaceable]`AmazonEFSCSIDriverPolicy` returned in the search. +.. In the *Filter policies* box, enter `AmazonEFSCSIDriverPolicy`. +.. Select the check box to the left of the `AmazonEFSCSIDriverPolicy` returned in the search. .. Choose *Next*. . On the *Name, review, and create* page, do the following: -+ -.. For *Role name*, enter a unique name for your role, such as [.replaceable]`AmazonEKS_EFS_CSI_DriverRole`. -.. Under *Add tags (Optional)*, add metadata to the role by attaching tags as key-value pairs. For more information about using tags in IAM, see link:IAM/latest/UserGuide/id_tags.html[Tagging IAM resources,type="documentation"] in the _IAM User Guide_. +.. For *Role name*, enter a unique name for your role, such as `AmazonEKS_EFS_CSI_DriverRole`. +.. Under *Add tags (Optional)*, add metadata to the role by attaching tags as key-value pairs. For more information about using tags in IAM, see link:IAM/latest/UserGuide/id_tags.html[Tagging IAM resources,type="documentation"] in the _IAM User Guide_. .. Choose *Create role*. -. After the role is created, choose the role in the console to open it for editing. -. Choose the *Trust relationships* tab, and then choose *Edit trust policy*. -. Find the line that looks similar to the following line: +. After the role is created: +.. If using EKS Pod Identities: +... Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. +... In the left navigation pane, select *Clusters*, and then select the name of the cluster that you want to configure the EKS Pod Identity association for. +... Choose the *Access* tab. +... In *Pod Identity associations*, choose *Create*. +... Choose the *IAM role* dropdown and select your newly created role. +... Choose the *Kubernetes namespace* field and input `kube-system`. +... Choose the *Kubernetes service account* field and input `efs-csi-controller-sa`. +... Choose *Create*. +... For more information on creating Pod Identity associations, see <>. +.. If using IAM roles for service accounts: +... Choose the role to open it for editing. +... Choose the *Trust relationships* tab, and then choose *Edit trust policy*. +... Find the line that looks similar to the following line: + [source,json,subs="verbatim,attributes"] ---- -"oidc.eks.region-code.amazonaws.com/id/EXAMPLED539D4633E53DE1B71EXAMPLE:aud": "sts.amazonaws.com" +"oidc.eks.region-code.amazonaws.com/id/:aud": "sts.amazonaws.com" ---- + -Add the following line above the previous line. Replace [.replaceable]`region-code` with the {aws} Region that your cluster is in. Replace [.replaceable]`EXAMPLED539D4633E53DE1B71EXAMPLE` with your cluster's OIDC provider ID. +Add the following line above the previous line. Replace `` with the {aws} Region that your cluster is in. Replace `` with your cluster's OIDC provider ID. + [source,json,subs="verbatim,attributes"] ---- -"oidc.eks.region-code.amazonaws.com/id/EXAMPLED539D4633E53DE1B71EXAMPLE:sub": "system:serviceaccount:kube-system:efs-csi-*", +"oidc.eks..amazonaws.com/id/:sub": "system:serviceaccount:kube-system:efs-csi-*", ---- -. Modify the `Condition` operator from `"StringEquals"` to `"StringLike"`. -. Choose *Update policy* to finish. +... Modify the `Condition` operator from `"StringEquals"` to `"StringLike"`. +... Choose *Update policy* to finish. -=== {aws} CLI [[awscli_efs_store_app_data]] +[#awscli_efs_store_app_data] +=== {aws} CLI Run the following commands to create an IAM role with {aws} CLI. -. View your cluster's OIDC provider URL. Replace [.replaceable]`my-cluster` with your cluster name. If the output from the command is `None`, review the *Prerequisites*. +[#efs-cli-pod-identities] +==== If using Pod Identities + +. Create the IAM role that grants the `AssumeRole` and `TagSession` actions to the `pods.eks.amazonaws.com` service. ++ +.. Copy the following contents to a file named `aws-efs-csi-driver-trust-policy-pod-identity.json`. ++ +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:f594a908-9316-4427-9fc5-eb97fb8984e1[] +---- +.. Create the role. Replace `my-cluster` with your cluster name. You can also replace `AmazonEKS_EFS_CSI_DriverRole` with a different name. ++ +[source,bash,subs="verbatim,attributes"] +---- +export cluster_name=my-cluster +export role_name=AmazonEKS_EFS_CSI_DriverRole +aws iam create-role \ + --role-name $role_name \ + --assume-role-policy-document file://"aws-efs-csi-driver-trust-policy-pod-identity.json" +---- +. Attach the required {aws} managed policy to the role with the following command. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam attach-role-policy \ + --policy-arn {arn-aws}iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy \ + --role-name $role_name +---- + +. Run the following command to create the Pod Identity association. Replace `{arn-aws}iam::<111122223333>:role/my-role` with the role created in previous steps. ++ +---- +aws eks create-pod-identity-association --cluster-name $cluster_name --role-arn {arn-aws}iam::<111122223333>:role/my-role --namespace kube-system --service-account efs-csi-controller-sa +---- +. For more information on creating Pod Identity associations, see <>. + +[#efs-cli-irsa] +==== If using IAM roles for service accounts + +. View your cluster's OIDC provider URL. Replace `my-cluster` with your cluster name. You can also replace `AmazonEKS_EFS_CSI_DriverRole` with a different name. + [source,bash,subs="verbatim,attributes"] ---- -aws eks describe-cluster --name my-cluster --query "cluster.identity.oidc.issuer" --output text +export cluster_name=my-cluster +export role_name=AmazonEKS_EFS_CSI_DriverRole +aws eks describe-cluster --name $cluster_name --query "cluster.identity.oidc.issuer" --output text ---- + An example output is as follows. + [source,bash,subs="verbatim,attributes"] ---- -https://oidc.eks.region-code.amazonaws.com/id/EXAMPLED539D4633E53DE1B71EXAMPLE +https://oidc.eks..amazonaws.com/id/ ---- ++ +If the output from the command is `None`, review the *Prerequisites*. . Create the IAM role that grants the `AssumeRoleWithWebIdentity` action. + -.. Copy the following contents to a file named [.replaceable]`aws-efs-csi-driver-trust-policy`.json``. Replace [.replaceable]`111122223333` with your account ID. Replace [.replaceable]`EXAMPLED539D4633E53DE1B71EXAMPLE` and [.replaceable]`region-code` with the values returned in the previous step. If your cluster is in the {aws} GovCloud (US-East) or {aws} GovCloud (US-West) {aws} Regions, then replace `{arn-aws}` with `arn:aws-us-gov:`. +.. Copy the following contents to a file named `aws-efs-csi-driver-trust-policy.json`. Replace `<111122223333>` with your account ID. Replace `` and `` with the values returned in the previous step. + [source,json,subs="verbatim,attributes"] ---- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "{arn-aws}iam::111122223333:oidc-provider/oidc.eks.region-code.amazonaws.com/id/EXAMPLED539D4633E53DE1B71EXAMPLE" - }, - "Action": "sts:AssumeRoleWithWebIdentity", - "Condition": { - "StringLike": { - "oidc.eks.region-code.amazonaws.com/id/EXAMPLED539D4633E53DE1B71EXAMPLE:sub": "system:serviceaccount:kube-system:efs-csi-*", - "oidc.eks.region-code.amazonaws.com/id/EXAMPLED539D4633E53DE1B71EXAMPLE:aud": "sts.amazonaws.com" - } - } - } - ] -} ----- -.. Create the role. You can change [.replaceable]`AmazonEKS_EFS_CSI_DriverRole` to a different name, but if you do, make sure to change it in later steps too. +include::iam_snippet:e34d5873-4f45-4b88-941f-aa6d605a5e16[] +---- +.. Create the role. + [source,bash,subs="verbatim,attributes"] ---- aws iam create-role \ - --role-name AmazonEKS_EFS_CSI_DriverRole \ + --role-name $role_name \ --assume-role-policy-document file://"aws-efs-csi-driver-trust-policy.json" ---- -. Attach the required {aws} managed policy to the role with the following command. If your cluster is in the {aws} GovCloud (US-East) or {aws} GovCloud (US-West) {aws} Regions, then replace `{arn-aws}` with `arn:aws-us-gov:`. +. Attach the required {aws} managed policy to the role with the following command. + [source,bash,subs="verbatim,attributes"] ---- aws iam attach-role-policy \ --policy-arn {arn-aws}iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy \ - --role-name AmazonEKS_EFS_CSI_DriverRole + --role-name $role_name ---- -[[efs-install-driver,efs-install-driver.title]] +[#efs-install-driver] == Step 2: Get the Amazon EFS CSI driver We recommend that you install the Amazon EFS CSI driver through the Amazon EKS add-on. To add an Amazon EKS add-on to your cluster, see <>. For more information about add-ons, see <>. If you're unable to use the Amazon EKS add-on, we encourage you to submit an issue about why you can't to the https://github.com/aws/containers-roadmap/issues[Containers roadmap GitHub repository]. -Alternatively, if you want a self-managed installation of the Amazon EFS CSI driver, see https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/docs/README.md#installation[Installation] on [.noloc]`GitHub`. - -[[efs-create-filesystem,efs-create-filesystem.title]] -== Step 3: Create an Amazon EFS file system - -[NOTE] +[IMPORTANT] +==== +Before adding the Amazon EFS driver as an Amazon EKS add-on, confirm that you don't have a self-managed version of the driver installed on your cluster. If so, see https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/docs/README.md#uninstalling-the-amazon-efs-csi-driver[Uninstalling the Amazon EFS CSI Driver] on GitHub. ==== -This step isn't needed for {aws} Fargate. A [.noloc]`Pod` running on Fargate automatically mounts an Amazon EFS file system, without needing manual driver installation steps. +Alternatively, if you want a self-managed installation of the Amazon EFS CSI driver, see https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/docs/README.md#installation[Installation] on GitHub. -==== +[#efs-create-filesystem] +== Step 3: Create an Amazon EFS file system -To create an Amazon EFS file system, see https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/docs/efs-create-filesystem.md[Create an Amazon EFS file system for Amazon EKS] on [.noloc]`GitHub`. +To create an Amazon EFS file system, see https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/docs/efs-create-filesystem.md[Create an Amazon EFS file system for Amazon EKS] on GitHub. -[[efs-sample-app,efs-sample-app.title]] +[#efs-sample-app] == Step 4: Deploy a sample application -You can deploy a variety of sample apps and modify them as needed. For more information, see https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/docs/README.md#examples[Examples] on [.noloc]`GitHub`. +You can deploy a variety of sample apps and modify them as needed. For more information, see https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/docs/README.md#examples[Examples] on GitHub. diff --git a/latest/ug/storage/file-cache-csi.adoc b/latest/ug/storage/file-cache-csi.adoc index fcffabe03..42027699c 100644 --- a/latest/ug/storage/file-cache-csi.adoc +++ b/latest/ug/storage/file-cache-csi.adoc @@ -1,16 +1,9 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[file-cache-csi,file-cache-csi.title]] +[#file-cache-csi] = Minimize latency with Amazon File Cache -:info_doctype: section -:info_title: Minimize latency with Amazon File Cache :info_titleabbrev: Amazon File Cache -:keywords: Amazon File Cache CSI driver, storage -:info_abstract: The Amazon File Cache Container Storage Interface (CSI) driver provides a CSI \ - interface that allows Amazon EKS clusters to manage the life cycle of Amazon file \ - caches. [abstract] -- @@ -19,4 +12,4 @@ The Amazon File Cache Container Storage Interface (CSI) driver provides a CSI in Amazon File Cache is a fully managed, high-speed cache on {aws} that's used to process file data, regardless of where the data is stored. Amazon File Cache automatically loads data into the cache when it's accessed for the first time and releases data when it's not used. For more information, see the link:fsx/latest/FileCacheGuide/what-is.html[Amazon File Cache User Guide,type="documentation"]. -The Amazon File Cache Container Storage Interface (CSI) driver provides a CSI interface that allows Amazon EKS clusters to manage the life cycle of Amazon file caches. Note that the Amazon File Cache CSI driver is not compatible with Amazon EKS Hybrid Nodes. To deploy the Amazon File Cache CSI driver to your Amazon EKS cluster, see https://github.com/kubernetes-sigs/aws-file-cache-csi-driver[aws-file-cache-csi-driver] on GitHub. +The Amazon File Cache Container Storage Interface (CSI) driver provides a CSI interface that allows Amazon EKS clusters to manage the life cycle of Amazon file caches. Note that the Amazon File Cache CSI driver is not compatible with Amazon EKS Hybrid Nodes. To deploy the Amazon File Cache CSI driver to your Amazon EKS cluster, see https://github.com/kubernetes-sigs/aws-file-cache-csi-driver[aws-file-cache-csi-driver] on GitHub. \ No newline at end of file diff --git a/latest/ug/storage/fsx-csi-create.adoc b/latest/ug/storage/fsx-csi-create.adoc new file mode 100644 index 000000000..648a8094e --- /dev/null +++ b/latest/ug/storage/fsx-csi-create.adoc @@ -0,0 +1,280 @@ +include::../attributes.txt[] + +[.topic] +[#fsx-csi-create] += Deploy the FSx for Lustre driver +:info_titleabbrev: Deploy the driver + +[abstract] +-- +This topic shows you how to deploy the FSx for Lustre CSI driver to your Amazon EKS cluster and verify that it works. +-- + +This topic shows you how to deploy the <> to your Amazon EKS cluster and verify that it works. We recommend using the latest version of the driver. For available versions, see https://github.com/kubernetes-sigs/aws-fsx-csi-driver/blob/master/docs/README.md#csi-specification-compatibility-matrix[CSI Specification Compatibility Matrix] on GitHub. + +[NOTE] +==== + +The driver isn't supported on Fargate or Amazon EKS Hybrid Nodes. + +==== + +For detailed descriptions of the available parameters and complete examples that demonstrate the driver's features, see the https://github.com/kubernetes-sigs/aws-fsx-csi-driver[FSx for Lustre Container Storage Interface (CSI) driver] project on GitHub. + +[#fsx-csi-prereqs] +== Prerequisites + +* An existing cluster. +* The Amazon FSx CSI Driver EKS add-on requires the EKS Pod Identity agent for authentication. Without this component, the add-on will fail with the error `Amazon EKS Pod Identity agent is not installed in the cluster`, preventing volume operations. Install the Pod Identity agent before or after deploying the FSx CSI Driver add-on. For more information, see <>. +* Version `2.12.3` or later or version `1.27.160` or later of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device or {aws} CloudShell. To check your current version, use `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or Homebrew for macOS are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the _{aws} Command Line Interface User Guide_. The {aws} CLI version that is installed in {aws} CloudShell might also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the _{aws} CloudShell User Guide_. +* Version `{eksctl-min-version}` or later of the `eksctl` command line tool installed on your device or {aws} CloudShell. To install or update `eksctl`, see https://eksctl.io/installation[Installation] in the `eksctl` documentation. +* The `kubectl` command line tool is installed on your device or {aws} CloudShell. The version can be the same as or up to one minor version earlier or later than the Kubernetes version of your cluster. For example, if your cluster version is `1.29`, you can use `kubectl` version `1.28`, `1.29`, or `1.30` with it. To install or upgrade `kubectl`, see <>. + +[#fsx-create-iam-role] +== Step 1: Create an IAM role + +The Amazon FSx CSI plugin requires IAM permissions to make calls to {aws} APIs on your behalf. + +[NOTE] +==== +Pods will have access to the permissions that are assigned to the IAM role unless you block access to IMDS. For more information, see <>. +==== + +The following procedure shows you how to create an IAM role and attach the {aws} managed policy to it. + +. Create an IAM role and attach the {aws} managed policy with the following command. Replace `my-cluster` with the name of the cluster you want to use. The command deploys an {aws} CloudFormation stack that creates an IAM role and attaches the IAM policy to it. ++ +[source,bash,subs="verbatim,attributes"] +---- +eksctl create iamserviceaccount \ + --name fsx-csi-controller-sa \ + --namespace kube-system \ + --cluster my-cluster \ + --role-name AmazonEKS_FSx_CSI_DriverRole \ + --role-only \ + --attach-policy-arn {arn-aws}iam::aws:policy/AmazonFSxFullAccess \ + --approve +---- ++ +You'll see several lines of output as the service account is created. The last lines of output are similar to the following. ++ +[source,bash,subs="verbatim,attributes"] +---- +[ℹ] 1 task: { + 2 sequential sub-tasks: { + create IAM role for serviceaccount "kube-system/fsx-csi-controller-sa", + create serviceaccount "kube-system/fsx-csi-controller-sa", + } } +[ℹ] building iamserviceaccount stack "eksctl-my-cluster-addon-iamserviceaccount-kube-system-fsx-csi-controller-sa" +[ℹ] deploying stack "eksctl-my-cluster-addon-iamserviceaccount-kube-system-fsx-csi-controller-sa" +[ℹ] waiting for CloudFormation stack "eksctl-my-cluster-addon-iamserviceaccount-kube-system-fsx-csi-controller-sa" +[ℹ] created serviceaccount "kube-system/fsx-csi-controller-sa" +---- ++ +Note the name of the {aws} CloudFormation stack that was deployed. In the previous example output, the stack is named `eksctl-my-cluster-addon-iamserviceaccount-kube-system-fsx-csi-controller-sa`. + +Now that you have created the Amazon FSx CSI driver IAM role, you can continue to the next section. When you deploy the add-on with this IAM role, it creates and is configured to use a service account that's named `fsx-csi-controller-sa`. The service account is bound to a Kubernetes `clusterrole` that's assigned the required Kubernetes permissions. + +[#fsx-csi-deploy-driver] +== Step 2: Install the Amazon FSx CSI driver + +We recommend that you install the Amazon FSx CSI driver through the Amazon EKS add-on to improve security and reduce the amount of work. To add an Amazon EKS add-on to your cluster, see <>. For more information about add-ons, see <>. + +[IMPORTANT] +==== + +Pre-existing Amazon FSx CSI driver installations in the cluster can cause add-on installation failures. When you attempt to install the Amazon EKS add-on version while a non-EKS FSx CSI Driver exists, the installation will fail due to resource conflicts. Use the `OVERWRITE` flag during installation to resolve this issue. + +[source,bash] +---- +aws eks create-addon --addon-name aws-fsx-csi-driver --cluster-name my-cluster --resolve-conflicts OVERWRITE +---- + +==== + +Alternatively, if you want a self-managed installation of the Amazon FSx CSI driver, see https://github.com/kubernetes-sigs/aws-fsx-csi-driver/blob/master/docs/install.md[Installation] on GitHub. + +[#fsx-csi-deploy-storage-class] +== Step 3: Deploy a storage class, persistent volume claim, and sample app + +This procedure uses the https://github.com/kubernetes-sigs/aws-fsx-csi-driver[FSx for Lustre Container Storage Interface (CSI) driver] GitHub repository to consume a dynamically-provisioned FSx for Lustre volume. + +. Note the security group for your cluster. You can see it in the {aws-management-console} under the *Networking* section or by using the following {aws} CLI command. Replace `my-cluster` with the name of the cluster you want to use. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks describe-cluster --name my-cluster --query cluster.resourcesVpcConfig.clusterSecurityGroupId +---- +. Create a security group for your Amazon FSx file system according to the criteria shown in link:fsx/latest/LustreGuide/limit-access-security-groups.html#fsx-vpc-security-groups[Amazon VPC Security Groups,type="documentation"] in the Amazon FSx for Lustre User Guide. For the *VPC*, select the VPC of your cluster as shown under the *Networking* section. For "the security groups associated with your Lustre clients", use your cluster security group. You can leave the outbound rules alone to allow *All traffic*. +. Download the storage class manifest with the following command. ++ +[source,bash,subs="verbatim,attributes"] +---- +curl -O https://raw.githubusercontent.com/kubernetes-sigs/aws-fsx-csi-driver/master/examples/kubernetes/dynamic_provisioning/specs/storageclass.yaml +---- +. Edit the parameters section of the `storageclass.yaml` file. Replace every example value with your own values. ++ +[source,yaml,subs="verbatim,attributes"] +---- +parameters: + subnetId: subnet-0eabfaa81fb22bcaf + securityGroupIds: sg-068000ccf82dfba88 + deploymentType: PERSISTENT_1 + automaticBackupRetentionDays: "1" + dailyAutomaticBackupStartTime: "00:00" + copyTagsToBackups: "true" + perUnitStorageThroughput: "200" + dataCompressionType: "NONE" + weeklyMaintenanceStartTime: "7:09:00" + fileSystemTypeVersion: "2.12" +---- ++ +** *`subnetId`* – The subnet ID that the Amazon FSx for Lustre file system should be created in. Amazon FSx for Lustre isn't supported in all Availability Zones. Open the Amazon FSx for Lustre console at https://console.aws.amazon.com/fsx/ to confirm that the subnet that you want to use is in a supported Availability Zone. The subnet can include your nodes, or can be a different subnet or VPC: ++ +*** You can check for the node subnets in the {aws-management-console} by selecting the node group under the *Compute* section. +*** If the subnet that you specify isn't the same subnet that you have nodes in, then your VPCs must be link:whitepapers/latest/aws-vpc-connectivity-options/amazon-vpc-to-amazon-vpc-connectivity-options.html[connected,type="documentation"], and you must ensure that you have the necessary ports open in your security groups. +** *`securityGroupIds`* – The ID of the security group you created for the file system. +** *`deploymentType` (optional)* – The file system deployment type. Valid values are `SCRATCH_1`, `SCRATCH_2`, `PERSISTENT_1`, and `PERSISTENT_2`. For more information about deployment types, see link:fsx/latest/LustreGuide/getting-started-step1.html[Create your Amazon FSx for Lustre file system,type="documentation"]. +** *other parameters (optional)* – For information about the other parameters, see https://github.com/kubernetes-sigs/aws-fsx-csi-driver/tree/master/examples/kubernetes/dynamic_provisioning#edit-storageclass[Edit StorageClass] on GitHub. +. Create the storage class manifest. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl apply -f storageclass.yaml +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +storageclass.storage.k8s.io/fsx-sc created +---- +. Download the persistent volume claim manifest. ++ +[source,bash,subs="verbatim,attributes"] +---- +curl -O https://raw.githubusercontent.com/kubernetes-sigs/aws-fsx-csi-driver/master/examples/kubernetes/dynamic_provisioning/specs/claim.yaml +---- +. (Optional) Edit the `claim.yaml` file. Change `1200Gi` to one of the following increment values, based on your storage requirements and the `deploymentType` that you selected in a previous step. ++ +[source,yaml,subs="verbatim,attributes"] +---- +storage: 1200Gi +---- ++ +** `SCRATCH_2` and `PERSISTENT` – `1.2 TiB`, `2.4 TiB`, or increments of 2.4 TiB over 2.4 TiB. +** `SCRATCH_1` – `1.2 TiB`, `2.4 TiB`, `3.6 TiB`, or increments of 3.6 TiB over 3.6 TiB. +. Create the persistent volume claim. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl apply -f claim.yaml +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +persistentvolumeclaim/fsx-claim created +---- +. Confirm that the file system is provisioned. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl describe pvc +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +Name: fsx-claim +Namespace: default +StorageClass: fsx-sc +Status: Bound +[...] +---- ++ +NOTE: The `Status` may show as `Pending` for 5-10 minutes, before changing to `Bound`. Don't continue with the next step until the `Status` is `Bound`. If the `Status` shows `Pending` for more than 10 minutes, use warning messages in the `Events` as reference for addressing any problems. +. Deploy the sample application. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/aws-fsx-csi-driver/master/examples/kubernetes/dynamic_provisioning/specs/pod.yaml +---- +. Verify that the sample application is running. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get pods +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +NAME READY STATUS RESTARTS AGE +fsx-app 1/1 Running 0 8s +---- +. Verify that the file system is mounted correctly by the application. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl exec -ti fsx-app -- df -h +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +Filesystem Size Used Avail Use% Mounted on +overlay 80G 4.0G 77G 5% / +tmpfs 64M 0 64M 0% /dev +tmpfs 3.8G 0 3.8G 0% /sys/fs/cgroup +192.0.2.0@tcp:/abcdef01 1.1T 7.8M 1.1T 1% /data +/dev/nvme0n1p1 80G 4.0G 77G 5% /etc/hosts +shm 64M 0 64M 0% /dev/shm +tmpfs 6.9G 12K 6.9G 1% /run/secrets/kubernetes.io/serviceaccount +tmpfs 3.8G 0 3.8G 0% /proc/acpi +tmpfs 3.8G 0 3.8G 0% /sys/firmware +---- +. Verify that data was written to the FSx for Lustre file system by the sample app. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl exec -it fsx-app -- ls /data +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +out.txt +---- ++ +This example output shows that the sample app successfully wrote the `out.txt` file to the file system. + + +[NOTE] +==== + +Before deleting the cluster, make sure to delete the FSx for Lustre file system. For more information, see link:fsx/latest/LustreGuide/getting-started-step4.html[Clean up resources,type="documentation"] in the _FSx for Lustre User Guide_. + +==== + +== Performance tuning for FSx for Lustre + +When using FSx for Lustre with Amazon EKS, you can optimize performance by applying Lustre tunings during node initialization. The recommended approach is to use launch template user data to ensure consistent configuration across all nodes. + +These tunings include: + +* Network and RPC optimizations +* Lustre module management +* LRU (Lock Resource Unit) tunings +* Client cache control settings +* RPC controls for OST and MDC + +For detailed instructions on implementing these performance tunings: + +* For optimizing performance for standard nodes (non-EFA), see <> for a complete script that can be added to your launch template user data. +* For optimizing performance for EFA-enabled nodes, see <>. \ No newline at end of file diff --git a/latest/ug/storage/fsx-csi-tuning-efa.adoc b/latest/ug/storage/fsx-csi-tuning-efa.adoc new file mode 100644 index 000000000..dc1460d65 --- /dev/null +++ b/latest/ug/storage/fsx-csi-tuning-efa.adoc @@ -0,0 +1,744 @@ +include::../attributes.txt[] + +[.topic] +[#fsx-csi-tuning-efa] += Optimize Amazon FSx for Lustre performance on nodes (EFA) +:info_titleabbrev: Optimize (EFA) + +[abstract] +-- +Learn how to set up Elastic Fabric Adapter (EFA) tuning with Amazon EKS and Amazon FSx for Lustre. +-- + +This topic describes how to set up Elastic Fabric Adapter (EFA) tuning with Amazon EKS and Amazon FSx for Lustre. + +[NOTE] +==== + +* For information on creating and deploying the FSx for Lustre CSI driver, see <>. +* For optimizing standard nodes without EFA, see <>. + +==== + +[#create-eks-cluster] +== Step 1. Create EKS cluster + +Create a cluster using the provided configuration file: + +[source,bash] +---- +# Create cluster using efa-cluster.yaml +eksctl create cluster -f efa-cluster.yaml +---- + +Example `efa-cluster.yaml`: + +[source,yaml] +---- +#efa-cluster.yaml + +apiVersion: eksctl.io/v1alpha5 +kind: ClusterConfig + +metadata: + name: csi-fsx + region: us-east-1 + version: "1.30" + +iam: + withOIDC: true + +availabilityZones: ["us-east-1a", "us-east-1d"] + +managedNodeGroups: + - name: my-efa-ng + instanceType: c6gn.16xlarge + minSize: 1 + desiredCapacity: 1 + maxSize: 1 + availabilityZones: ["us-east-1b"] + volumeSize: 300 + privateNetworking: true + amiFamily: Ubuntu2204 + efaEnabled: true + preBootstrapCommands: + - | + #!/bin/bash + eth_intf="$(/sbin/ip -br -4 a sh | grep $(hostname -i)/ | awk '{print $1}')" + efa_version=$(modinfo efa | awk '/^version:/ {print $2}' | sed 's/[^0-9.]//g') + min_efa_version="2.12.1" + + if [[ "$(printf '%s\n' "$min_efa_version" "$efa_version" | sort -V | head -n1)" != "$min_efa_version" ]]; then + sudo curl -O https://efa-installer.amazonaws.com/aws-efa-installer-1.37.0.tar.gz + tar -xf aws-efa-installer-1.37.0.tar.gz && cd aws-efa-installer + echo "Installing EFA driver" + sudo apt-get update && apt-get upgrade -y + sudo apt install -y pciutils environment-modules libnl-3-dev libnl-route-3-200 libnl-route-3-dev dkms + sudo ./efa_installer.sh -y + modinfo efa + else + echo "Using EFA driver version $efa_version" + fi + + echo "Installing Lustre client" + sudo wget -O - https://fsx-lustre-client-repo-public-keys.s3.amazonaws.com/fsx-ubuntu-public-key.asc | gpg --dearmor | sudo tee /usr/share/keyrings/fsx-ubuntu-public-key.gpg > /dev/null + sudo echo "deb [signed-by=/usr/share/keyrings/fsx-ubuntu-public-key.gpg] https://fsx-lustre-client-repo.s3.amazonaws.com/ubuntu jammy main" > /etc/apt/sources.list.d/fsxlustreclientrepo.list + sudo apt update | tail + sudo apt install -y lustre-client-modules-$(uname -r) amazon-ec2-utils | tail + modinfo lustre + + echo "Loading Lustre/EFA modules..." + sudo /sbin/modprobe lnet + sudo /sbin/modprobe kefalnd ipif_name="$eth_intf" + sudo /sbin/modprobe ksocklnd + sudo lnetctl lnet configure + + echo "Configuring TCP interface..." + sudo lnetctl net del --net tcp 2> /dev/null + sudo lnetctl net add --net tcp --if $eth_intf + + # For P5 instance type which supports 32 network cards, + # by default add 8 EFA interfaces selecting every 4th device (1 per PCI bus) + echo "Configuring EFA interface(s)..." + instance_type="$(ec2-metadata --instance-type | awk '{ print $2 }')" + num_efa_devices="$(ls -1 /sys/class/infiniband | wc -l)" + echo "Found $num_efa_devices available EFA device(s)" + + if [[ "$instance_type" == "p5.48xlarge" || "$instance_type" == "p5e.48xlarge" ]]; then + for intf in $(ls -1 /sys/class/infiniband | awk 'NR % 4 == 1'); do + sudo lnetctl net add --net efa --if $intf --peer-credits 32 + done + else + # Other instances: Configure 2 EFA interfaces by default if the instance supports multiple network cards, + # or 1 interface for single network card instances + # Can be modified to add more interfaces if instance type supports it + sudo lnetctl net add --net efa --if $(ls -1 /sys/class/infiniband | head -n1) --peer-credits 32 + if [[ $num_efa_devices -gt 1 ]]; then + sudo lnetctl net add --net efa --if $(ls -1 /sys/class/infiniband | tail -n1) --peer-credits 32 + fi + fi + + echo "Setting discovery and UDSP rule" + sudo lnetctl set discovery 1 + sudo lnetctl udsp add --src efa --priority 0 + sudo /sbin/modprobe lustre + + sudo lnetctl net show + echo "Added $(sudo lnetctl net show | grep -c '@efa') EFA interface(s)" +---- + +[#create-node-group] +== Step 2. Create node group + +Create an EFA-enabled node group: + +[source,bash] +---- +# Create node group using efa-ng.yaml +eksctl create nodegroup -f efa-ng.yaml +---- + +[IMPORTANT] +=== +Adjust these values for your environment in section `# 5. Mount FSx filesystem`. + +[source,bash] +---- +FSX_DNS="" # Needs to be adjusted. +MOUNT_NAME="" # Needs to be adjusted. +MOUNT_POINT="" # Needs to be adjusted. +---- +=== + +Example `efa-ng.yaml`: + +[source,yaml] +---- +apiVersion: eksctl.io/v1alpha5 +kind: ClusterConfig + +metadata: + name: final-efa + region: us-east-1 + +managedNodeGroups: + - name: ng-1 + instanceType: c6gn.16xlarge + minSize: 1 + desiredCapacity: 1 + maxSize: 1 + availabilityZones: ["us-east-1a"] + volumeSize: 300 + privateNetworking: true + amiFamily: Ubuntu2204 + efaEnabled: true + preBootstrapCommands: + - | + #!/bin/bash + exec 1> >(logger -s -t $(basename $0)) 2>&1 + + ######################################################################################### + # Configuration Section # + ######################################################################################### + + # File System Configuration + FSX_DNS="" # Needs to be adjusted. + MOUNT_NAME="" # Needs to be adjusted. + MOUNT_POINT="" # Needs to be adjusted. + + # Lustre Tuning Parameters + LUSTRE_LRU_MAX_AGE=600000 + LUSTRE_MAX_CACHED_MB=64 + LUSTRE_OST_MAX_RPC=32 + LUSTRE_MDC_MAX_RPC=64 + LUSTRE_MDC_MOD_RPC=50 + + # File paths + FUNCTIONS_SCRIPT="/usr/local/bin/lustre_functions.sh" + TUNINGS_SCRIPT="/usr/local/bin/apply_lustre_tunings.sh" + SERVICE_FILE="/etc/systemd/system/lustre-tunings.service" + + #EFA + MIN_EFA_VERSION="2.12.1" + + # Function to check if a command was successful + check_success() { + if [ $? -eq 0 ]; then + echo "SUCCESS: $1" + else + echo "FAILED: $1" + return 1 + fi + } + + echo "********Starting FSx for Lustre configuration********" + + # 1. Install Lustre client + if grep -q '^ID=ubuntu' /etc/os-release; then + echo "Detected Ubuntu, proceeding with Lustre setup..." + # Add Lustre repository + sudo wget -O - https://fsx-lustre-client-repo-public-keys.s3.amazonaws.com/fsx-ubuntu-public-key.asc | sudo gpg --dearmor | sudo tee /usr/share/keyrings/fsx-ubuntu-public-key.gpg > /dev/null + + echo "deb [signed-by=/usr/share/keyrings/fsx-ubuntu-public-key.gpg] https://fsx-lustre-client-repo.s3.amazonaws.com/ubuntu jammy main" | sudo tee /etc/apt/sources.list.d/fsxlustreclientrepo.list + + sudo apt-get update + sudo apt-get install -y lustre-client-modules-$(uname -r) + sudo apt-get install -y lustre-client + else + echo "Not Ubuntu, exiting" + exit 1 + fi + + check_success "Install Lustre client" + + # Ensure Lustre tools are in the PATH + export PATH=$PATH:/usr/sbin + + # 2. Apply network and RPC tunings + echo "********Applying network and RPC tunings********" + if ! grep -q "options ptlrpc ptlrpcd_per_cpt_max" /etc/modprobe.d/modprobe.conf; then + echo "options ptlrpc ptlrpcd_per_cpt_max=64" | sudo tee -a /etc/modprobe.d/modprobe.conf + check_success "Set ptlrpcd_per_cpt_max" + else + echo "ptlrpcd_per_cpt_max already set in modprobe.conf" + fi + + if ! grep -q "options ksocklnd credits" /etc/modprobe.d/modprobe.conf; then + echo "options ksocklnd credits=2560" | sudo tee -a /etc/modprobe.d/modprobe.conf + check_success "Set ksocklnd credits" + else + echo "ksocklnd credits already set in modprobe.conf" + fi + + # 3. Load Lustre modules + manage_lustre_modules() { + echo "Checking for existing Lustre modules..." + if lsmod | grep -q lustre; then + echo "Existing Lustre modules found." + + # Check for mounted Lustre filesystems + echo "Checking for mounted Lustre filesystems..." + if mount | grep -q "type lustre"; then + echo "Found mounted Lustre filesystems. Attempting to unmount..." + mounted_fs=$(mount | grep "type lustre" | awk '{print $3}') + for fs in $mounted_fs; do + echo "Unmounting $fs" + sudo umount $fs + check_success "Unmount filesystem $fs" + done + else + echo "No Lustre filesystems mounted." + fi + + # After unmounting, try to remove modules + echo "Attempting to remove Lustre modules..." + sudo lustre_rmmod + if [ $? -eq 0 ]; then + echo "SUCCESS: Removed existing Lustre modules" + else + echo "WARNING: Could not remove Lustre modules. They may still be in use." + echo "Please check for any remaining Lustre processes or mounts." + return 1 + fi + else + echo "No existing Lustre modules found." + fi + + echo "Loading Lustre modules..." + sudo modprobe lustre + check_success "Load Lustre modules" || exit 1 + + echo "Checking loaded Lustre modules:" + lsmod | grep lustre + } + + # Managing Lustre kernel modules + echo "********Managing Lustre kernel modules********" + manage_lustre_modules + + # 4. Initializing Lustre networking + echo "********Initializing Lustre networking********" + sudo lctl network up + check_success "Initialize Lustre networking" || exit 1 + + # 4.5 EFA Setup and Configuration + setup_efa() { + echo "********Starting EFA Setup********" + + # Get interface and version information + eth_intf="$(/sbin/ip -br -4 a sh | grep $(hostname -i)/ | awk '{print $1}')" + efa_version=$(modinfo efa | awk '/^version:/ {print $2}' | sed 's/[^0-9.]//g') + min_efa_version=$MIN_EFA_VERSION + + # Install or verify EFA driver + if [[ "$(printf '%s\n' "$min_efa_version" "$efa_version" | sort -V | head -n1)" != "$min_efa_version" ]]; then + echo "Installing EFA driver..." + sudo curl -O https://efa-installer.amazonaws.com/aws-efa-installer-1.37.0.tar.gz + tar -xf aws-efa-installer-1.37.0.tar.gz && cd aws-efa-installer + + # Install dependencies + sudo apt-get update && apt-get upgrade -y + sudo apt install -y pciutils environment-modules libnl-3-dev libnl-route-3-200 libnl-route-3-dev dkms + + # Install EFA + sudo ./efa_installer.sh -y + modinfo efa + else + echo "Using existing EFA driver version $efa_version" + fi + } + + configure_efa_network() { + echo "********Configuring EFA Network********" + + # Load required modules + echo "Loading network modules..." + sudo /sbin/modprobe lnet + sudo /sbin/modprobe kefalnd ipif_name="$eth_intf" + sudo /sbin/modprobe ksocklnd + + # Initialize LNet + echo "Initializing LNet..." + sudo lnetctl lnet configure + + # Configure TCP interface + echo "Configuring TCP interface..." + sudo lnetctl net del --net tcp 2> /dev/null + sudo lnetctl net add --net tcp --if $eth_intf + + # For P5 instance type which supports 32 network cards, + # by default add 8 EFA interfaces selecting every 4th device (1 per PCI bus) + echo "Configuring EFA interface(s)..." + instance_type="$(ec2-metadata --instance-type | awk '{ print $2 }')" + num_efa_devices="$(ls -1 /sys/class/infiniband | wc -l)" + echo "Found $num_efa_devices available EFA device(s)" + + if [[ "$instance_type" == "p5.48xlarge" || "$instance_type" == "p5e.48xlarge" ]]; then + # P5 instance configuration + for intf in $(ls -1 /sys/class/infiniband | awk 'NR % 4 == 1'); do + sudo lnetctl net add --net efa --if $intf --peer-credits 32 + done + else + # Standard configuration + # Other instances: Configure 2 EFA interfaces by default if the instance supports multiple network cards, + # or 1 interface for single network card instances + # Can be modified to add more interfaces if instance type supports it + sudo lnetctl net add --net efa --if $(ls -1 /sys/class/infiniband | head -n1) --peer-credits 32 + if [[ $num_efa_devices -gt 1 ]]; then + sudo lnetctl net add --net efa --if $(ls -1 /sys/class/infiniband | tail -n1) --peer-credits 32 + fi + fi + + # Configure discovery and UDSP + echo "Setting up discovery and UDSP rules..." + sudo lnetctl set discovery 1 + sudo lnetctl udsp add --src efa --priority 0 + sudo /sbin/modprobe lustre + + # Verify configuration + echo "Verifying EFA network configuration..." + sudo lnetctl net show + echo "Added $(sudo lnetctl net show | grep -c '@efa') EFA interface(s)" + } + + # Main execution + setup_efa + configure_efa_network + + # 5. Mount FSx filesystem + if [ ! -z "$FSX_DNS" ] && [ ! -z "$MOUNT_NAME" ]; then + echo "********Creating mount point********" + sudo mkdir -p $MOUNT_POINT + check_success "Create mount point" + + echo "Mounting FSx filesystem..." + sudo mount -t lustre ${FSX_DNS}@tcp:/${MOUNT_NAME} ${MOUNT_POINT} + check_success "Mount FSx filesystem" + else + echo "Skipping FSx mount as DNS or mount name is not provided" + fi + + # 6. Applying Lustre performance tunings + echo "********Applying Lustre performance tunings********" + + # Get number of CPUs + NUM_CPUS=$(nproc) + + # Calculate LRU size (100 * number of CPUs) + LRU_SIZE=$((100 * NUM_CPUS)) + + #Apply LRU tunings + echo "Apply LRU tunings" + sudo lctl set_param ldlm.namespaces.*.lru_max_age=${LUSTRE_LRU_MAX_AGE} + check_success "Set lru_max_age" + sudo lctl set_param ldlm.namespaces.*.lru_size=$LRU_SIZE + check_success "Set lru_size" + + # Client Cache Control + sudo lctl set_param llite.*.max_cached_mb=${LUSTRE_MAX_CACHED_MB} + check_success "Set max_cached_mb" + + # RPC Controls + sudo lctl set_param osc.*OST*.max_rpcs_in_flight=${LUSTRE_OST_MAX_RPC} + check_success "Set OST max_rpcs_in_flight" + + sudo lctl set_param mdc.*.max_rpcs_in_flight=${LUSTRE_MDC_MAX_RPC} + check_success "Set MDC max_rpcs_in_flight" + + sudo lctl set_param mdc.*.max_mod_rpcs_in_flight=${LUSTRE_MDC_MOD_RPC} + check_success "Set MDC max_mod_rpcs_in_flight" + + # 7. Verify all tunings + echo "********Verifying all tunings********" + + # Function to verify parameter value + verify_param() { + local param=$1 + local expected=$2 + local actual=$3 + + if [ "$actual" == "$expected" ]; then + echo "SUCCESS: $param is correctly set to $expected" + else + echo "WARNING: $param is set to $actual (expected $expected)" + fi + } + + echo "Verifying all parameters:" + + # LRU tunings + actual_lru_max_age=$(lctl get_param -n ldlm.namespaces.*.lru_max_age | head -1) + verify_param "lru_max_age" "600000" "$actual_lru_max_age" + + actual_lru_size=$(lctl get_param -n ldlm.namespaces.*.lru_size | head -1) + verify_param "lru_size" "$LRU_SIZE" "$actual_lru_size" + + # Client Cache + actual_max_cached_mb=$(lctl get_param -n llite.*.max_cached_mb | grep "max_cached_mb:" | awk '{print $2}') + verify_param "max_cached_mb" "64" "$actual_max_cached_mb" + + # RPC Controls + actual_ost_rpcs=$(lctl get_param -n osc.*OST*.max_rpcs_in_flight | head -1) + verify_param "OST max_rpcs_in_flight" "32" "$actual_ost_rpcs" + + actual_mdc_rpcs=$(lctl get_param -n mdc.*.max_rpcs_in_flight | head -1) + verify_param "MDC max_rpcs_in_flight" "64" "$actual_mdc_rpcs" + + actual_mdc_mod_rpcs=$(lctl get_param -n mdc.*.max_mod_rpcs_in_flight | head -1) + verify_param "MDC max_mod_rpcs_in_flight" "50" "$actual_mdc_mod_rpcs" + + # Network and RPC configurations from modprobe.conf + actual_ptlrpc=$(grep "ptlrpc ptlrpcd_per_cpt_max" /etc/modprobe.d/modprobe.conf | awk '{print $3}') + verify_param "ptlrpcd_per_cpt_max" "ptlrpcd_per_cpt_max=64" "$actual_ptlrpc" + + actual_ksocklnd=$(grep "ksocklnd credits" /etc/modprobe.d/modprobe.conf | awk '{print $3}') + verify_param "ksocklnd credits" "credits=2560" "$actual_ksocklnd" + + # 8. Setup persistence + setup_persistence() { + # Create functions file + cat << EOF > $FUNCTIONS_SCRIPT + #!/bin/bash + + apply_lustre_tunings() { + local NUM_CPUS=\$(nproc) + local LRU_SIZE=\$((100 * NUM_CPUS)) + + echo "Applying Lustre performance tunings..." + lctl set_param ldlm.namespaces.*.lru_max_age=$LUSTRE_LRU_MAX_AGE + lctl set_param ldlm.namespaces.*.lru_size=\$LRU_SIZE + lctl set_param llite.*.max_cached_mb=$LUSTRE_MAX_CACHED_MB + lctl set_param osc.*OST*.max_rpcs_in_flight=$LUSTRE_OST_MAX_RPC + lctl set_param mdc.*.max_rpcs_in_flight=$LUSTRE_MDC_MAX_RPC + lctl set_param mdc.*.max_mod_rpcs_in_flight=$LUSTRE_MDC_MOD_RPC + } + EOF + + # Create tuning script + cat << EOF > $TUNINGS_SCRIPT + #!/bin/bash + exec 1> >(logger -s -t \$(basename \$0)) 2>&1 + + source $FUNCTIONS_SCRIPT + + # Function to check if Lustre is mounted + is_lustre_mounted() { + mount | grep -q "type lustre" + } + + # Function to mount Lustre + mount_lustre() { + echo "Mounting Lustre filesystem..." + mkdir -p $MOUNT_POINT + mount -t lustre ${FSX_DNS}@tcp:/${MOUNT_NAME} $MOUNT_POINT + return \$? + } + + # Main execution + # Try to mount if not already mounted + if ! is_lustre_mounted; then + echo "Lustre filesystem not mounted, attempting to mount..." + mount_lustre + fi + + # Wait for successful mount (up to 5 minutes) + for i in {1..30}; do + if is_lustre_mounted; then + echo "Lustre filesystem mounted, applying tunings..." + apply_lustre_tunings + exit 0 + fi + echo "Waiting for Lustre filesystem to be mounted... (attempt $i/30)" + sleep 10 + done + + echo "Timeout waiting for Lustre filesystem mount" + exit 1 + EOF + + # Create systemd service + + # Create systemd directory if it doesn't exist + sudo mkdir -p /etc/systemd/system/ + + # Create service file directly for Ubuntu + cat << EOF > $SERVICE_FILE + [Unit] + Description=Apply Lustre Performance Tunings + After=network.target remote-fs.target + + [Service] + Type=oneshot + ExecStart=/bin/bash -c 'source $FUNCTIONS_SCRIPT && $TUNINGS_SCRIPT' + RemainAfterExit=yes + + [Install] + WantedBy=multi-user.target + EOF + + + # Make scripts executable and enable service + sudo chmod +x $FUNCTIONS_SCRIPT + sudo chmod +x $TUNINGS_SCRIPT + systemctl enable lustre-tunings.service + systemctl start lustre-tunings.service + } + + echo "********Setting up persistent tuning********" + setup_persistence + + echo "FSx for Lustre configuration completed." +---- + +[#verify-efa-setup] +== (Optional) Step 3. Verify EFA setup + +SSH into node: + +[source,bash,subs="verbatim,quotes"] +---- +# Get instance ID from EKS console or {aws} CLI +ssh -i /path/to/your-key.pem ec2-user@ +---- + +Verify EFA configuration: + +[source,bash] +---- +sudo lnetctl net show +---- + +Check setup logs: + +[source,bash] +---- +sudo cat /var/log/cloud-init-output.log +---- + +Here's example expected output for `lnetctl net show`: + +[source,text] +---- +net: + - net type: tcp + ... + - net type: efa + local NI(s): + - nid: xxx.xxx.xxx.xxx@efa + status: up +---- + +[#example-deployments] +== Example deployments + +=== a. Create claim.yaml + +[source,yaml] +---- +#claim.yaml + +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: fsx-claim-efa +spec: + accessModes: + - ReadWriteMany + storageClassName: "" + resources: + requests: + storage: 4800Gi + volumeName: fsx-pv +---- + +Apply the claim: + +[source,bash] +---- +kubectl apply -f claim.yaml +---- + +=== b. Create pv.yaml + +Update the ``: + +[source,yaml] +---- +#pv.yaml + +apiVersion: v1 +kind: PersistentVolume +metadata: + name: fsx-pv +spec: + capacity: + storage: 4800Gi + volumeMode: Filesystem + accessModes: + - ReadWriteMany + mountOptions: + - flock + persistentVolumeReclaimPolicy: Recycle + csi: + driver: fsx.csi.aws.com + volumeHandle: fs-<1234567890abcdef0> + volumeAttributes: + dnsname: fs-<1234567890abcdef0>.fsx.us-east-1.amazonaws.com + mountname: +---- + +Apply the persistent volume: + +[source,bash] +---- +kubectl apply -f pv.yaml +---- + +=== c. Create pod.yaml + +[source,yaml] +---- +#pod.yaml + +apiVersion: v1 +kind: Pod +metadata: + name: fsx-efa-app +spec: + containers: + - name: app + image: amazonlinux:2 + command: ["/bin/sh"] + args: ["-c", "while true; do dd if=/dev/urandom bs=100M count=20 > data/test_file; sleep 10; done"] + resources: + requests: + vpc.amazonaws.com/efa: 1 + limits: + vpc.amazonaws.com/efa: 1 + volumeMounts: + - name: persistent-storage + mountPath: /data + volumes: + - name: persistent-storage + persistentVolumeClaim: + claimName: fsx-claim-efa +---- + +Apply the Pod: + +[source,bash] +---- +kubectl apply -f pod.yaml +---- + +[#verification-commands] +== Additional verification commands + +Verify Pod mounts and writes to filesystem: + +[source,bash] +---- +kubectl exec -ti fsx-efa-app -- df -h | grep data +# Expected output: +# <192.0.2.0>@tcp:/ 4.5T 1.2G 4.5T 1% /data + +kubectl exec -ti fsx-efa-app -- ls /data +# Expected output: +# test_file +---- + +SSH onto the node to verify traffic is going over EFA: + +[source,bash] +---- +sudo lnetctl net show -v +---- + +The expected output will show EFA interfaces with traffic statistics. + +== Related information + +* <> +* <> +* link:fsx/latest/LustreGuide/performance.html[Amazon FSx for Lustre Performance,type="documentation"] +* link:ec2/latest/userguide/efa.html[Elastic Fabric Adapter,type="documentation"] diff --git a/latest/ug/storage/fsx-csi-tuning-non-efa.adoc b/latest/ug/storage/fsx-csi-tuning-non-efa.adoc new file mode 100644 index 000000000..ec9904420 --- /dev/null +++ b/latest/ug/storage/fsx-csi-tuning-non-efa.adoc @@ -0,0 +1,297 @@ +include::../attributes.txt[] + +[.topic] +[#fsx-csi-tuning-non-efa] += Optimize Amazon FSx for Lustre performance on nodes (non-EFA) +:info_titleabbrev: Optimize (non-EFA) + +[abstract] +-- +Learn how to optimize Amazon FSx for Lustre performance on your Amazon EKS nodes by applying tuning parameters during node initialization. +-- + +You can optimize Amazon FSx for Lustre performance by applying tuning parameters during node initialization using launch template user data. + +[NOTE] +==== + +* For information on creating and deploying the FSx for Lustre CSI driver, see <>. +For optimizing performance with EFA-enabled nodes, see <>. + +==== + +== Why use launch template user data? + +* Applies tunings automatically during node initialization. +* Ensures consistent configuration across all nodes. +* Eliminates the need for manual node configuration. + +== Example script overview + +The example script defined in this topic performs these operations: + +=== `# 1. Install Lustre client` + +* Automatically detects your OS version: Amazon Linux 2 (AL2) or Amazon Linux 2023 (AL2023). +* Installs the appropriate Lustre client package. + +=== `# 2. Apply network and RPC tunings` + +* Sets `ptlrpcd_per_cpt_max=64` for parallel RPC processing. +* Configures `ksocklnd credits=2560` to optimize network buffers. + +=== `# 3. Load Lustre modules` + +* Safely removes existing Lustre modules if present. +* Handles unmounting of existing filesystems. +* Loads fresh Lustre modules. + +=== `# 4. Lustre Network Initialization` + +* Initializes Lustre networking configuration. +* Sets up required network parameters. + +=== `# 5. Mount FSx filesystem` + +* You must adjust the values for your environment in this section. + +=== `# 6. Apply tunings` + +* LRU (Lock Resource Unit) tunings: +** `lru_max_age=600000` +** `lru_size` calculated based on CPU count +* Client Cache Control: `max_cached_mb=64` +* RPC Controls: +** OST `max_rpcs_in_flight=32` +** MDC `max_rpcs_in_flight=64` +** MDC `max_mod_rpcs_in_flight=50` + +=== `# 7. Verify tunings` + +* Verifies all applied tunings. +* Reports success or warning for each parameter. + +=== `# 8. Setup persistence` + +* You must adjust the values for your environment in this section as well. +* Automatically detects your OS version (AL2 or AL2023) to determine which `Systemd` service to apply. +* System starts. +* `Systemd` starts `lustre-tunings` service (due to `WantedBy=multi-user.target`). +* Service runs `apply_lustre_tunings.sh` which: +** Checks if filesystem is mounted. +** Mounts filesystem if not mounted. +** Waits for successful mount (up to five minutes). +** Applies tuning parameters after successful mount. +* Settings remain active until reboot. +* Service exits after script completion. +** Systemd marks service as "active (exited)". +* Process repeats on next reboot. + +== Create a launch template + +. Open the Amazon EC2 console at https://console.aws.amazon.com/ec2/. +. Choose *Launch Templates*. +. Choose *Create launch template*. +. In *Advanced details*, locate the *User data* section. +. Paste the script below, updating anything as needed. ++ +[IMPORTANT] +==== +Adjust these values for your environment in both section `# 5. Mount FSx filesystem` and the `setup_persistence()` function of `apply_lustre_tunings.sh` in section `# 8. Setup persistence`: + +[source,bash] +---- +FSX_DNS="" # Needs to be adjusted. +MOUNT_NAME="" # Needs to be adjusted. +MOUNT_POINT="" # Needs to be adjusted. +---- +==== ++ +[source,bash] +---- +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="==MYBOUNDARY==" +--==MYBOUNDARY== +Content-Type: text/x-shellscript; charset="us-ascii" +#!/bin/bash +exec 1> >(logger -s -t $(basename $0)) 2>&1 +# Function definitions +check_success() { + if [ $? -eq 0 ]; then + echo "SUCCESS: $1" + else + echo "FAILED: $1" + return 1 + fi +} +apply_tunings() { + local NUM_CPUS=$(nproc) + local LRU_SIZE=$((100 * NUM_CPUS)) + local params=( + "ldlm.namespaces.*.lru_max_age=600000" + "ldlm.namespaces.*.lru_size=$LRU_SIZE" + "llite.*.max_cached_mb=64" + "osc.*OST*.max_rpcs_in_flight=32" + "mdc.*.max_rpcs_in_flight=64" + "mdc.*.max_mod_rpcs_in_flight=50" + ) + for param in "${params[@]}"; do + lctl set_param $param + check_success "Set ${param%%=*}" + done +} +verify_param() { + local param=$1 + local expected=$2 + local actual=$3 + + if [ "$actual" == "$expected" ]; then + echo "SUCCESS: $param is correctly set to $expected" + else + echo "WARNING: $param is set to $actual (expected $expected)" + fi +} +verify_tunings() { + local NUM_CPUS=$(nproc) + local LRU_SIZE=$((100 * NUM_CPUS)) + local params=( + "ldlm.namespaces.*.lru_max_age:600000" + "ldlm.namespaces.*.lru_size:$LRU_SIZE" + "llite.*.max_cached_mb:64" + "osc.*OST*.max_rpcs_in_flight:32" + "mdc.*.max_rpcs_in_flight:64" + "mdc.*.max_mod_rpcs_in_flight:50" + ) + echo "Verifying all parameters:" + for param in "${params[@]}"; do + name="${param%%:*}" + expected="${param#*:}" + actual=$(lctl get_param -n $name | head -1) + verify_param "${name##*.}" "$expected" "$actual" + done +} +setup_persistence() { + # Create functions file + cat << 'EOF' > /usr/local/bin/lustre_functions.sh +#!/bin/bash +apply_lustre_tunings() { + local NUM_CPUS=$(nproc) + local LRU_SIZE=$((100 * NUM_CPUS)) + + echo "Applying Lustre performance tunings..." + lctl set_param ldlm.namespaces.*.lru_max_age=600000 + lctl set_param ldlm.namespaces.*.lru_size=$LRU_SIZE + lctl set_param llite.*.max_cached_mb=64 + lctl set_param osc.*OST*.max_rpcs_in_flight=32 + lctl set_param mdc.*.max_rpcs_in_flight=64 + lctl set_param mdc.*.max_mod_rpcs_in_flight=50 +} +EOF + # Create tuning script + cat << 'EOF' > /usr/local/bin/apply_lustre_tunings.sh +#!/bin/bash +exec 1> >(logger -s -t $(basename $0)) 2>&1 +# Source the functions +source /usr/local/bin/lustre_functions.sh +# FSx details +FSX_DNS="" # Needs to be adjusted. +MOUNT_NAME="" # Needs to be adjusted. +MOUNT_POINT="" # Needs to be adjusted. +# Function to check if Lustre is mounted +is_lustre_mounted() { + mount | grep -q "type lustre" +} +# Function to mount Lustre +mount_lustre() { + echo "Mounting Lustre filesystem..." + mkdir -p ${MOUNT_POINT} + mount -t lustre ${FSX_DNS}@tcp:/${MOUNT_NAME} ${MOUNT_POINT} + return $? +} +# Main execution +# Try to mount if not already mounted +if ! is_lustre_mounted; then + echo "Lustre filesystem not mounted, attempting to mount..." + mount_lustre +fi +# Wait for successful mount (up to 5 minutes) +for i in {1..30}; do + if is_lustre_mounted; then + echo "Lustre filesystem mounted, applying tunings..." + apply_lustre_tunings + exit 0 + fi + echo "Waiting for Lustre filesystem to be mounted... (attempt $i/30)" + sleep 10 +done +echo "Timeout waiting for Lustre filesystem mount" +exit 1 +EOF + # Create systemd service + cat << 'EOF' > /etc/systemd/system/lustre-tunings.service +[Unit] +Description=Apply Lustre Performance Tunings +After=network.target remote-fs.target +StartLimitIntervalSec=0 +[Service] +Type=oneshot +ExecStart=/usr/local/bin/apply_lustre_tunings.sh +RemainAfterExit=yes +Restart=on-failure +RestartSec=30 +[Install] +WantedBy=multi-user.target +EOF + chmod +x /usr/local/bin/lustre_functions.sh + chmod +x /usr/local/bin/apply_lustre_tunings.sh + systemctl enable lustre-tunings.service + systemctl start lustre-tunings.service +} +echo "Starting FSx for Lustre configuration..." +# 1. Install Lustre client +if grep -q 'VERSION="2"' /etc/os-release; then + amazon-linux-extras install -y lustre +elif grep -q 'VERSION="2023"' /etc/os-release; then + dnf install -y lustre-client +fi +check_success "Install Lustre client" +# 2. Apply network and RPC tunings +export PATH=$PATH:/usr/sbin +echo "Applying network and RPC tunings..." +if ! grep -q "options ptlrpc ptlrpcd_per_cpt_max" /etc/modprobe.d/modprobe.conf; then + echo "options ptlrpc ptlrpcd_per_cpt_max=64" | tee -a /etc/modprobe.d/modprobe.conf + echo "options ksocklnd credits=2560" | tee -a /etc/modprobe.d/modprobe.conf +fi +# 3. Load Lustre modules +modprobe lustre +check_success "Load Lustre modules" || exit 1 +# 4. Lustre Network Initialization +lctl network up +check_success "Initialize Lustre networking" || exit 1 +# 5. Mount FSx filesystem +FSX_DNS="" # Needs to be adjusted. +MOUNT_NAME="" # Needs to be adjusted. +MOUNT_POINT="" # Needs to be adjusted. +if [ ! -z "$FSX_DNS" ] && [ ! -z "$MOUNT_NAME" ]; then + mkdir -p $MOUNT_POINT + mount -t lustre ${FSX_DNS}@tcp:/${MOUNT_NAME} ${MOUNT_POINT} + check_success "Mount FSx filesystem" +fi +# 6. Apply tunings +apply_tunings +# 7. Verify tunings +verify_tunings +# 8. Setup persistence +setup_persistence +echo "FSx for Lustre configuration completed." +--==MYBOUNDARY==-- +---- + +. When creating Amazon EKS node groups, select this launch template. For more information, see <>. + +== Related information + +* <> +* <> +* link:fsx/latest/LustreGuide/performance.html[Amazon FSx for Lustre Performance,type="documentation"] + diff --git a/latest/ug/storage/fsx-csi.adoc b/latest/ug/storage/fsx-csi.adoc index 85f70213b..3f8a35c44 100644 --- a/latest/ug/storage/fsx-csi.adoc +++ b/latest/ug/storage/fsx-csi.adoc @@ -1,307 +1,21 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[fsx-csi,fsx-csi.title]] -= Store high-performance apps with FSx for Lustre -:info_doctype: section -:info_title: Store high-performance apps with FSx for Lustre +[#fsx-csi] += Use high-performance app storage with Amazon FSx for Lustre :info_titleabbrev: Amazon FSx for Lustre -:keywords: Amazon FSx for Lustre CSI driver, storage -:info_abstract: The FSx for Lustre Container Storage Interface (CSI) driver provides a CSI interface \ - that allows Amazon EKS clusters to manage the lifecycle of FSx for Lustre file \ - systems. [abstract] -- -The FSx for Lustre Container Storage Interface (CSI) driver provides a CSI interface that allows Amazon EKS clusters to manage the lifecycle of FSx for Lustre file systems. +The Amazon FSx for Lustre Container Storage Interface (CSI) driver provides a CSI interface that allows Amazon EKS clusters to manage the lifecycle of Amazon FSx for Lustre file systems. -- -The https://github.com/kubernetes-sigs/aws-fsx-csi-driver[FSx for Lustre Container Storage Interface (CSI) driver] provides a CSI interface that allows Amazon EKS clusters to manage the lifecycle of FSx for Lustre file systems. For more information, see the link:fsx/latest/LustreGuide/what-is.html[FSx for Lustre User Guide,type="documentation"]. +The https://github.com/kubernetes-sigs/aws-fsx-csi-driver[Amazon FSx for Lustre Container Storage Interface (CSI) driver] provides a CSI interface that allows Amazon EKS clusters to manage the lifecycle of Amazon FSx for Lustre file systems. For more information, see the link:fsx/latest/LustreGuide/what-is.html[Amazon FSx for Lustre User Guide,type="documentation"]. -This topic shows you how to deploy the FSx for Lustre CSI driver to your Amazon EKS cluster and verify that it works. We recommend using the latest version of the driver. For available versions, see https://github.com/kubernetes-sigs/aws-fsx-csi-driver/blob/master/docs/README.md#csi-specification-compatibility-matrix[CSI Specification Compatibility Matrix] on GitHub. +For details on how to deploy the Amazon FSx for Lustre CSI driver to your Amazon EKS cluster and verify that it works, see <>. -[NOTE] -==== +include::fsx-csi-create.adoc[leveloffset=+1] -The driver isn't supported on Fargate or Amazon EKS Hybrid Nodes. +include::fsx-csi-tuning-efa.adoc[leveloffset=+1] -==== - -For detailed descriptions of the available parameters and complete examples that demonstrate the driver's features, see the https://github.com/kubernetes-sigs/aws-fsx-csi-driver[FSx for Lustre Container Storage Interface (CSI) driver] project on [.noloc]`GitHub`. - - - - -You must have: - -* Version `2.12.3` or later or version `1.27.160` or later of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device or {aws} CloudShell. To check your current version, use `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or [.noloc]`Homebrew` for [.noloc]`macOS` are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the _{aws} Command Line Interface User Guide_. The {aws} CLI version that is installed in {aws} CloudShell might also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the _{aws} CloudShell User Guide_. -* Version `{eksctl-min-version}` or later of the `eksctl` command line tool installed on your device or {aws} CloudShell. To install or update `eksctl`, see https://eksctl.io/installation[Installation] in the `eksctl` documentation. -* The `kubectl` command line tool is installed on your device or {aws} CloudShell. The version can be the same as or up to one minor version earlier or later than the [.noloc]`Kubernetes` version of your cluster. For example, if your cluster version is `1.29`, you can use `kubectl` version `1.28`, `1.29`, or `1.30` with it. To install or upgrade `kubectl`, see <>. - -The following procedures help you create a simple test cluster with the FSx for Lustre CSI driver so that you can see how it works. We don't recommend using the testing cluster for production workloads. For this tutorial, we recommend using the [.replaceable]`example values`, except where it's noted to replace them. You can replace any [.replaceable]`example value` when completing the steps for your production cluster. We recommend completing all steps in the same terminal because variables are set and used throughout the steps and won't exist in different terminals. - -. Set a few variables to use in the remaining steps. Replace [.replaceable]`my-csi-fsx-cluster` with the name of the test cluster you want to createand [.replaceable]`region-code` with the {aws} Region that you want to create your test cluster in. -+ -[source,bash,subs="verbatim,attributes"] ----- -export cluster_name=my-csi-fsx-cluster -export region_code=region-code ----- -. Create a test cluster. -+ -[source,bash,subs="verbatim,attributes"] ----- -eksctl create cluster \ - --name $cluster_name \ - --region $region_code \ - --with-oidc \ - --ssh-access \ - --ssh-public-key my-key ----- -+ -Cluster provisioning takes several minutes. During cluster creation, you'll see several lines of output. The last line of output is similar to the following example line. -+ -[source,bash,subs="verbatim,attributes"] ----- -[✓] EKS cluster "my-csi-fsx-cluster" in "region-code" region is ready ----- -. Create a [.noloc]`Kubernetes` service account for the driver and attach the `AmazonFSxFullAccess` {aws}-managed policy to the service account with the following command. If your cluster is in the {aws} GovCloud (US-East) or {aws} GovCloud (US-West) {aws} Regions, then replace `{arn-aws}` with `arn:aws-us-gov:`. -+ -[source,bash,subs="verbatim,attributes"] ----- -eksctl create iamserviceaccount \ - --name fsx-csi-controller-sa \ - --namespace kube-system \ - --cluster $cluster_name \ - --attach-policy-arn {arn-aws}iam::aws:policy/AmazonFSxFullAccess \ - --approve \ - --role-name AmazonEKSFSxLustreCSIDriverFullAccess \ - --region $region_code ----- -+ -You'll see several lines of output as the service account is created. The last lines of output are similar to the following. -+ -[source,bash,subs="verbatim,attributes"] ----- -[ℹ] 1 task: { - 2 sequential sub-tasks: { - create IAM role for serviceaccount "kube-system/fsx-csi-controller-sa", - create serviceaccount "kube-system/fsx-csi-controller-sa", - } } -[ℹ] building iamserviceaccount stack "eksctl-my-csi-fsx-cluster-addon-iamserviceaccount-kube-system-fsx-csi-controller-sa" -[ℹ] deploying stack "eksctl-my-csi-fsx-cluster-addon-iamserviceaccount-kube-system-fsx-csi-controller-sa" -[ℹ] waiting for CloudFormation stack "eksctl-my-csi-fsx-cluster-addon-iamserviceaccount-kube-system-fsx-csi-controller-sa" -[ℹ] created serviceaccount "kube-system/fsx-csi-controller-sa" ----- -+ -Note the name of the {aws} CloudFormation stack that was deployed. In the previous example output, the stack is named `eksctl-my-csi-fsx-cluster-addon-iamserviceaccount-kube-system-fsx-csi-controller-sa`. -. Deploy the driver with the following command. Replace [.replaceable]`release-X.XX` with your desired branch. The master branch isn't supported because it may contain upcoming features incompatible with the currently released stable version of the driver. We recommend using the latest released version. For a list of branches, see `aws-fsx-csi-driver` https://github.com/kubernetes-sigs/aws-fsx-csi-driver/branches/all[Branches] on GitHub. - -+ -NOTE: You can view the content being applied in https://github.com/kubernetes-sigs/aws-fsx-csi-driver/tree/master/deploy/kubernetes/overlays/stable[aws-fsx-csi-driver/deploy/kubernetes/overlays/stable] on [.noloc]`GitHub`. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl apply -k "github.com/kubernetes-sigs/aws-fsx-csi-driver/deploy/kubernetes/overlays/stable/?ref=release-X.XX" ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -serviceaccount/fsx-csi-controller-sa created -serviceaccount/fsx-csi-node-sa created -clusterrole.rbac.authorization.k8s.io/fsx-csi-external-provisioner-role created -clusterrole.rbac.authorization.k8s.io/fsx-external-resizer-role created -clusterrolebinding.rbac.authorization.k8s.io/fsx-csi-external-provisioner-binding created -clusterrolebinding.rbac.authorization.k8s.io/fsx-csi-resizer-binding created -deployment.apps/fsx-csi-controller created -daemonset.apps/fsx-csi-node created -csidriver.storage.k8s.io/fsx.csi.aws.com created ----- -. Note the ARN for the role that was created. If you didn't note it earlier and don't have it available anymore in the {aws} CLI output, you can do the following to see it in the {aws-management-console}. -+ -.. Open the link:cloudformation/[{aws} CloudFormation console,type="console"]. -.. Ensure that the console is set to the {aws} Region that you created your IAM role in and then select *Stacks*. -.. Select the stack named `eksctl-my-csi-fsx-cluster-addon-iamserviceaccount-kube-system-fsx-csi-controller-sa`. -.. Select the *Outputs* tab. The *Role1* ARN is listed on the *Outputs (1)* page. -. Patch the driver deployment to add the service account that you created earlier with the following command. Replace the ARN with the ARN that you noted. Replace [.replaceable]`111122223333` with your account ID. If your cluster is in the {aws} GovCloud (US-East) or {aws} GovCloud (US-West) {aws} Regions, then replace `{arn-aws}` with `arn:aws-us-gov:`. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl annotate serviceaccount -n kube-system fsx-csi-controller-sa \ - eks.amazonaws.com/role-arn={arn-aws}iam::111122223333:role/AmazonEKSFSxLustreCSIDriverFullAccess --overwrite=true ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -serviceaccount/fsx-csi-controller-sa annotated ----- - -This procedure uses the https://github.com/kubernetes-sigs/aws-fsx-csi-driver[FSx for Lustre Container Storage Interface (CSI) driver][.noloc]`GitHub` repository to consume a dynamically-provisioned FSx for Lustre volume. - -. Note the security group for your cluster. You can see it in the {aws-management-console} under the *Networking* section or by using the following {aws} CLI command. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks describe-cluster --name $cluster_name --query cluster.resourcesVpcConfig.clusterSecurityGroupId ----- -. Create a security group for your Amazon FSx file system according to the criteria shown in link:fsx/latest/LustreGuide/limit-access-security-groups.html#fsx-vpc-security-groups[Amazon VPC Security Groups,type="documentation"] in the Amazon FSx for Lustre User Guide. For the *VPC*, select the VPC of your cluster as shown under the *Networking* section. For "the security groups associated with your Lustre clients", use your cluster security group. You can leave the outbound rules alone to allow *All traffic*. -. Download the storage class manifest with the following command. -+ -[source,bash,subs="verbatim,attributes"] ----- -curl -O https://raw.githubusercontent.com/kubernetes-sigs/aws-fsx-csi-driver/master/examples/kubernetes/dynamic_provisioning/specs/storageclass.yaml ----- -. Edit the parameters section of the `storageclass.yaml` file. Replace every [.replaceable]`example value` with your own values. -+ -[source,yaml,subs="verbatim,attributes"] ----- -parameters: - subnetId: subnet-0eabfaa81fb22bcaf - securityGroupIds: sg-068000ccf82dfba88 - deploymentType: PERSISTENT_1 - automaticBackupRetentionDays: "1" - dailyAutomaticBackupStartTime: "00:00" - copyTagsToBackups: "true" - perUnitStorageThroughput: "200" - dataCompressionType: "NONE" - weeklyMaintenanceStartTime: "7:09:00" - fileSystemTypeVersion: "2.12" ----- -+ -** *`subnetId`* – The subnet ID that the Amazon FSx for Lustre file system should be created in. Amazon FSx for Lustre isn't supported in all Availability Zones. Open the Amazon FSx for Lustre console at https://console.aws.amazon.com/fsx/ to confirm that the subnet that you want to use is in a supported Availability Zone. The subnet can include your nodes, or can be a different subnet or VPC: -+ -*** You can check for the node subnets in the {aws-management-console} by selecting the node group under the *Compute* section. -*** If the subnet that you specify isn't the same subnet that you have nodes in, then your VPCs must be link:whitepapers/latest/aws-vpc-connectivity-options/amazon-vpc-to-amazon-vpc-connectivity-options.html[connected,type="documentation"], and you must ensure that you have the necessary ports open in your security groups. -** *`securityGroupIds`* – The ID of the security group you created for the file system. -** *`deploymentType` (optional)* – The file system deployment type. Valid values are `SCRATCH_1`, `SCRATCH_2`, `PERSISTENT_1`, and `PERSISTENT_2`. For more information about deployment types, see link:fsx/latest/LustreGuide/getting-started-step1.html[Create your Amazon FSx for Lustre file system,type="documentation"]. -** *other parameters (optional)* – For information about the other parameters, see https://github.com/kubernetes-sigs/aws-fsx-csi-driver/tree/master/examples/kubernetes/dynamic_provisioning#edit-storageclass[Edit StorageClass] on [.noloc]`GitHub`. -. Create the storage class manifest. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl apply -f storageclass.yaml ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -storageclass.storage.k8s.io/fsx-sc created ----- -. Download the persistent volume claim manifest. -+ -[source,bash,subs="verbatim,attributes"] ----- -curl -O https://raw.githubusercontent.com/kubernetes-sigs/aws-fsx-csi-driver/master/examples/kubernetes/dynamic_provisioning/specs/claim.yaml ----- -. (Optional) Edit the `claim.yaml` file. Change [.replaceable]`1200Gi` to one of the following increment values, based on your storage requirements and the `deploymentType` that you selected in a previous step. -+ -[source,yaml,subs="verbatim,attributes"] ----- -storage: 1200Gi ----- -+ -** `SCRATCH_2` and `PERSISTENT` – `1.2 TiB`, `2.4 TiB`, or increments of 2.4 TiB over 2.4 TiB. -** `SCRATCH_1` – `1.2 TiB`, `2.4 TiB`, `3.6 TiB`, or increments of 3.6 TiB over 3.6 TiB. -. Create the persistent volume claim. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl apply -f claim.yaml ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -persistentvolumeclaim/fsx-claim created ----- -. Confirm that the file system is provisioned. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl describe pvc ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -Name: fsx-claim -Namespace: default -StorageClass: fsx-sc -Status: Bound -[...] ----- -+ -NOTE: The `Status` may show as `Pending` for 5-10 minutes, before changing to `Bound`. Don't continue with the next step until the `Status` is `Bound`. If the `Status` shows `Pending` for more than 10 minutes, use warning messages in the `Events` as reference for addressing any problems. -. Deploy the sample application. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/aws-fsx-csi-driver/master/examples/kubernetes/dynamic_provisioning/specs/pod.yaml ----- -. Verify that the sample application is running. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl get pods ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -NAME READY STATUS RESTARTS AGE -fsx-app 1/1 Running 0 8s ----- -. Verify that the file system is mounted correctly by the application. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl exec -ti fsx-app -- df -h ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -Filesystem Size Used Avail Use% Mounted on -overlay 80G 4.0G 77G 5% / -tmpfs 64M 0 64M 0% /dev -tmpfs 3.8G 0 3.8G 0% /sys/fs/cgroup -192.0.2.0@tcp:/abcdef01 1.1T 7.8M 1.1T 1% /data -/dev/nvme0n1p1 80G 4.0G 77G 5% /etc/hosts -shm 64M 0 64M 0% /dev/shm -tmpfs 6.9G 12K 6.9G 1% /run/secrets/kubernetes.io/serviceaccount -tmpfs 3.8G 0 3.8G 0% /proc/acpi -tmpfs 3.8G 0 3.8G 0% /sys/firmware ----- -. Verify that data was written to the FSx for Lustre file system by the sample app. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl exec -it fsx-app -- ls /data ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -out.txt ----- -+ -This example output shows that the sample app successfully wrote the `out.txt` file to the file system. - - -[NOTE] -==== - -Before deleting the cluster, make sure to delete the FSx for Lustre file system. For more information, see link:fsx/latest/LustreGuide/getting-started-step4.html[Clean up resources,type="documentation"] in the _FSx for Lustre User Guide_. - -==== +include::fsx-csi-tuning-non-efa.adoc[leveloffset=+1] diff --git a/latest/ug/storage/fsx-ontap.adoc b/latest/ug/storage/fsx-ontap.adoc index eacbd3bac..cb4cad5f8 100644 --- a/latest/ug/storage/fsx-ontap.adoc +++ b/latest/ug/storage/fsx-ontap.adoc @@ -1,22 +1,35 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[fsx-ontap,fsx-ontap.title]] -= Store high-performance apps with FSx for NetApp ONTAP -:info_doctype: section -:info_title: Store high-performance apps with FSx for NetApp ONTAP +[#fsx-ontap] += Use high-performance app storage with FSx for NetApp ONTAP :info_titleabbrev: Amazon FSx for NetApp ONTAP -:keywords: Amazon FSx for NetApp ONTAP CSI driver, storage -:info_abstract: NetApp's \ - NetApp Trident allows Amazon EKS clusters to manage the lifecycle of \ - persistent volumes (PVs) backed by Amazon FSx for NetApp ONTAP file systems. [abstract] -- -The [.noloc]`NetApp Trident` allows Amazon EKS clusters to manage the lifecycle of persistent volumes (PVs) backed by Amazon FSx for NetApp ONTAP file systems. +The NetApp Trident allows Amazon EKS clusters to manage the lifecycle of persistent volumes (PVs) backed by Amazon FSx for NetApp ONTAP file systems. -- -The [.noloc]`NetApp Trident` provides dynamic storage orchestration using a Container Storage Interface (CSI) compliant driver. This allows Amazon EKS clusters to manage the lifecycle of persistent volumes (PVs) backed by Amazon FSx for NetApp ONTAP file systems. Note that the Amazon FSx for NetApp ONTAP CSI driver is not compatible with Amazon EKS Hybrid Nodes. To get started, see https://docs.netapp.com/us-en/trident/trident-use/trident-fsx.html[Use Trident with Amazon FSx for NetApp ONTAP] in the [.noloc]`NetApp Trident` documentation. +The NetApp Trident provides dynamic storage orchestration using a Container Storage Interface (CSI) compliant driver. This allows Amazon EKS clusters to manage the lifecycle of persistent volumes (PVs) backed by Amazon FSx for NetApp ONTAP file systems. Note that the Amazon FSx for NetApp ONTAP CSI driver is not compatible with Amazon EKS Hybrid Nodes. To get started, see https://docs.netapp.com/us-en/trident/trident-use/trident-fsx.html[Use Trident with Amazon FSx for NetApp ONTAP] in the NetApp Trident documentation. -Amazon FSx for NetApp ONTAP is a storage service that allows you to launch and run fully managed [.noloc]`ONTAP` file systems in the cloud. [.noloc]`ONTAP` is [.noloc]`NetApp's` file system technology that provides a widely adopted set of data access and data management capabilities. FSx for ONTAP provides the features, performance, and APIs of on-premises [.noloc]`NetApp` file systems with the agility, scalability, and simplicity of a fully managed {aws} service. For more information, see the link:fsx/latest/ONTAPGuide/what-is-fsx-ontap.html[FSx for ONTAP User Guide,type="documentation"]. +Amazon FSx for NetApp ONTAP is a storage service that allows you to launch and run fully managed ONTAP file systems in the cloud. ONTAP is NetApp's file system technology that provides a widely adopted set of data access and data management capabilities. FSx for ONTAP provides the features, performance, and APIs of on-premises NetApp file systems with the agility, scalability, and simplicity of a fully managed {aws} service. For more information, see the link:fsx/latest/ONTAPGuide/what-is-fsx-ontap.html[FSx for ONTAP User Guide,type="documentation"]. + + +[IMPORTANT] +==== +If you are using Amazon FSx for NetApp ONTAP alongside the Amazon EBS CSI driver to provision EBS volumes, you must specify to not use EBS devices in the `multipath.conf` file. For supported methods, see https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/dm_multipath/config_file_blacklist#config_file_blacklist[Configuration File Blacklist]. Here is an example. + +[source,json,subs="verbatim,attributes"] +---- + defaults { + user_friendly_names yes + find_multipaths no + } + blacklist { + device { + vendor "NVME" + product "Amazon Elastic Block Store" + } + } +---- +==== diff --git a/latest/ug/storage/fsx-openzfs-csi.adoc b/latest/ug/storage/fsx-openzfs-csi.adoc index 53963c5ef..0435ded7a 100644 --- a/latest/ug/storage/fsx-openzfs-csi.adoc +++ b/latest/ug/storage/fsx-openzfs-csi.adoc @@ -1,16 +1,9 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[fsx-openzfs-csi,fsx-openzfs-csi.title]] -= Store data using Amazon FSx for OpenZFS -:info_doctype: section -:info_title: Store data using Amazon FSx for OpenZFS +[#fsx-openzfs-csi] += Use data storage with Amazon FSx for OpenZFS :info_titleabbrev: Amazon FSx for OpenZFS -:keywords: Amazon FSx for OpenZFS CSI driver, storage -:info_abstract: The Amazon FSx for OpenZFS Container Storage Interface (CSI) driver provides a CSI \ - interface that allows Amazon EKS clusters to manage the life cycle of Amazon FSx for OpenZFS \ - volumes. [abstract] -- diff --git a/latest/ug/storage/images b/latest/ug/storage/images new file mode 120000 index 000000000..5e6757319 --- /dev/null +++ b/latest/ug/storage/images @@ -0,0 +1 @@ +../images \ No newline at end of file diff --git a/latest/ug/storage/removing-s3-csi-eks-add-on.adoc b/latest/ug/storage/removing-s3-csi-eks-add-on.adoc new file mode 100644 index 000000000..77d6adb50 --- /dev/null +++ b/latest/ug/storage/removing-s3-csi-eks-add-on.adoc @@ -0,0 +1,59 @@ +include::../attributes.txt[] + +[.topic] +[#removing-s3-csi-eks-add-on] += Remove the Mountpoint for Amazon S3 Amazon EKS add-on +:info_titleabbrev: Remove the driver + +[abstract] +-- +This procedure will show you how to remove the Mountpoint for Amazon S3 CSI driver. +-- + +You have two options for removing the <>. + +* *Preserve add-on software on your cluster* – This option removes Amazon EKS management of any settings. It also removes the ability for Amazon EKS to notify you of updates and automatically update the Amazon EKS add-on after you initiate an update. However, it preserves the add-on software on your cluster. This option makes the add-on a self-managed installation, rather than an Amazon EKS add-on. With this option, there's no downtime for the add-on. The commands in this procedure use this option. +* *Remove add-on software entirely from your cluster* – We recommend that you remove the Amazon EKS add-on from your cluster only if there are no resources on your cluster that are dependent on it. To do this option, delete `--preserve` from the command you use in this procedure. + +If the add-on has an IAM account associated with it, the IAM account isn't removed. + +You can use the following tools to remove the Amazon S3 CSI add-on: + +* <> +* <> +* <> + +== eksctl [[eksctl_s3_remove_store_app_data]] + +*To remove the Amazon S3 CSI add-on using `eksctl`* + +Replace [.replaceable]`my-cluster` with the name of your cluster, and then run the following command. + +[source,bash,subs="verbatim,attributes"] +---- +eksctl delete addon --cluster my-cluster --name aws-mountpoint-s3-csi-driver --preserve +---- + +== {aws-management-console} [[console_s3_remove_store_app_data]] +. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. +. In the left navigation pane, choose *Clusters*. +. Choose the name of the cluster that you want to remove the Amazon EBS CSI add-on for. +. Choose the *Add-ons* tab. +. Choose *Mountpoint for Amazon S3 CSI Driver*. +. Choose *Remove*. +. In the *Remove: aws-mountpoint-s3-csi-driver* confirmation dialog box, do the following: ++ +.. If you want Amazon EKS to stop managing settings for the add-on, select *Preserve on cluster*. Do this if you want to retain the add-on software on your cluster. This is so that you can manage all of the settings of the add-on on your own. +.. Enter `aws-mountpoint-s3-csi-driver`. +.. Select *Remove*. + +== {aws} CLI [[awscli_s3_remove_store_app_data]] + +*To remove the Amazon S3 CSI add-on using the {aws} CLI* + +Replace [.replaceable]`my-cluster` with the name of your cluster, and then run the following command. + +[source,bash,subs="verbatim,attributes"] +---- +aws eks delete-addon --cluster-name my-cluster --addon-name aws-mountpoint-s3-csi-driver --preserve +---- \ No newline at end of file diff --git a/latest/ug/storage/s3-csi-create.adoc b/latest/ug/storage/s3-csi-create.adoc new file mode 100644 index 000000000..5e7f97dab --- /dev/null +++ b/latest/ug/storage/s3-csi-create.adoc @@ -0,0 +1,299 @@ +include::../attributes.txt[] + +[.topic] +[#s3-csi-create] += Deploy the Mountpoint for Amazon S3 driver +:info_titleabbrev: Deploy the driver + +[abstract] +-- +This procedure will show you how to deploy the Mountpoint for Amazon S3 CSI Amazon EKS driver. +-- + +With the https://github.com/awslabs/mountpoint-s3-csi-driver[Mountpoint for Amazon S3 Container Storage Interface (CSI) driver], your Kubernetes applications can access Amazon S3 objects through a file system interface, achieving high aggregate throughput without changing any application code. + +This procedure will show you how to deploy the <>. Before proceeding, please review the <>. + +[#s3-csi-prereqs] +== Prerequisites + +* An existing {aws} Identity and Access Management (IAM) OpenID Connect (OIDC) provider for your cluster. To determine whether you already have one, or to create one, see <>. +* Version 2.12.3 or later of the {aws} CLI installed and configured on your device or {aws} CloudShell. +* The `kubectl` command line tool is installed on your device or {aws} CloudShell. The version can be the same as or up to one minor version earlier or later than the Kubernetes version of your cluster. For example, if your cluster version is `1.29`, you can use `kubectl` version `1.28`, `1.29`, or `1.30` with it. To install or upgrade `kubectl`, see <>. + +[#s3-create-iam-policy] +== Step 1: Create an IAM policy + +The Mountpoint for Amazon S3 CSI driver requires Amazon S3 permissions to interact with your file system. This section shows how to create an IAM policy that grants the necessary permissions. + +The following example policy follows the IAM permission recommendations for Mountpoint. Alternatively, you can use the {aws} managed policy link:iam/home?#/policies/arn:aws:iam::aws:policy/AmazonS3FullAccess$jsonEditor[AmazonS3FullAccess,type="console"], but this managed policy grants more permissions than are needed for Mountpoint. + +For more information about the recommended permissions for Mountpoint, see https://github.com/awslabs/mountpoint-s3/blob/main/doc/CONFIGURATION.md#iam-permissions[Mountpoint IAM permissions] on GitHub. + +. Open the IAM console at https://console.aws.amazon.com/iam/. +. In the left navigation pane, choose *Policies*. +. On the *Policies* page, choose *Create policy*. +. For *Policy editor*, choose *JSON*. +. Under *Policy editor*, copy and paste the following: ++ +IMPORTANT: Replace `amzn-s3-demo-bucket1` with your own Amazon S3 bucket name. ++ +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:cd120344-570d-4a1a-92d8-6e970962c3d3[] +---- ++ +Directory buckets, introduced with the Amazon S3 Express One Zone storage class, use a different authentication mechanism from general purpose buckets. Instead of using `s3:*` actions, you should use the `s3express:CreateSession` action. For information about directory buckets, see link:AmazonS3/latest/userguide/directory-buckets-overview.html[Directory buckets,type="documentation"] in the _Amazon S3 User Guide_. ++ +Below is an example of least-privilege policy that you would use for a directory bucket. ++ +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:8e86ca18-dd51-464f-99f7-7095e0b72819[] +---- +. Choose *Next*. +. On the *Review and create* page, name your policy. This example walkthrough uses the name `AmazonS3CSIDriverPolicy`. +. Choose *Create policy*. + + +[#s3-create-iam-role] +== Step 2: Create an IAM role + +The Mountpoint for Amazon S3 CSI driver requires Amazon S3 permissions to interact with your file system. This section shows how to create an IAM role to delegate these permissions. To create this role, you can use one of these tools: + +* <> +* <> +* <> + +[NOTE] +==== + +The IAM policy `AmazonS3CSIDriverPolicy` was created in the previous section. + +==== + +=== eksctl [[eksctl_s3_store_app_data]] + +*To create your Mountpoint for Amazon S3 CSI driver IAM role with `eksctl`* + +To create the IAM role and the Kubernetes service account, run the following commands. These commands also attach the `AmazonS3CSIDriverPolicy` IAM policy to the role, annotate the Kubernetes service account (`s3-csi-controller-sa`) with the IAM role's Amazon Resource Name (ARN), and add the Kubernetes service account name to the trust policy for the IAM role. + +[source,bash,subs="verbatim,attributes"] +---- +CLUSTER_NAME=my-cluster +REGION=region-code +ROLE_NAME=AmazonEKS_S3_CSI_DriverRole +POLICY_ARN=AmazonEKS_S3_CSI_DriverRole_ARN +eksctl create iamserviceaccount \ + --name s3-csi-driver-sa \ + --namespace kube-system \ + --cluster $CLUSTER_NAME \ + --attach-policy-arn $POLICY_ARN \ + --approve \ + --role-name $ROLE_NAME \ + --region $REGION \ + --role-only +---- + + +=== {aws-management-console} [[console_s3_store_app_data]] +. Open the IAM console at https://console.aws.amazon.com/iam/. +. In the left navigation pane, choose *Roles*. +. On the *Roles* page, choose *Create role*. +. On the *Select trusted entity* page, do the following: ++ +.. In the *Trusted entity type* section, choose *Web identity*. +.. For *Identity provider*, choose the *OpenID Connect provider URL* for your cluster (as shown under *Overview* in Amazon EKS). ++ +If no URLs are shown, review the <>. +.. For *Audience*, choose `sts.amazonaws.com`. +.. Choose *Next*. +. On the *Add permissions* page, do the following: ++ +.. In the *Filter policies* box, enter AmazonS3CSIDriverPolicy. ++ +NOTE: This policy was created in the previous section. +.. Select the check box to the left of the `AmazonS3CSIDriverPolicy` result that was returned in the search. +.. Choose *Next*. +. On the *Name, review, and create* page, do the following: ++ +.. For *Role name*, enter a unique name for your role, such as AmazonEKS_S3_CSI_DriverRole. +.. Under *Add tags (Optional)*, add metadata to the role by attaching tags as key-value pairs. For more information about using tags in IAM, see link:IAM/latest/UserGuide/id_tags.html[Tagging IAM resources,type="documentation"] in the _IAM User Guide_. +.. Choose *Create role*. +. After the role is created, choose the role in the console to open it for editing. +. Choose the *Trust relationships* tab, and then choose *Edit trust policy*. +. Find the line that looks similar to the following: ++ +[source,json,subs="verbatim,attributes"] +---- +"oidc.eks.region-code.amazonaws.com/id/EXAMPLED539D4633E53DE1B71EXAMPLE:aud": "sts.amazonaws.com" +---- ++ +Add a comma to the end of the previous line, and then add the following line after it. Replace [.replaceable]`region-code` with the {aws} Region that your cluster is in. Replace [.replaceable]`EXAMPLED539D4633E53DE1B71EXAMPLE` with your cluster's OIDC provider ID. ++ +[source,json,subs="verbatim,attributes"] +---- +"oidc.eks.region-code.amazonaws.com/id/EXAMPLED539D4633E53DE1B71EXAMPLE:sub": "system:serviceaccount:kube-system:s3-csi-driver-sa" +---- +. Ensure that the `Condition` operator is set to `"StringEquals"`. +. Choose *Update policy* to finish. + +=== {aws} CLI [[awscli_s3_store_app_data]] +. View the OIDC provider URL for your cluster. Replace [.replaceable]`my-cluster` with the name of your cluster. If the output from the command is `None`, review the <>. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks describe-cluster --name my-cluster --query "cluster.identity.oidc.issuer" --output text +---- ++ +An example output is as follows. ++ +[source,bash,subs="verbatim,attributes"] +---- +https://oidc.eks.region-code.amazonaws.com/id/EXAMPLED539D4633E53DE1B71EXAMPLE +---- +. Create the IAM role, granting the Kubernetes service account the `AssumeRoleWithWebIdentity` action. ++ +.. Copy the following contents to a file named `aws-s3-csi-driver-trust-policy.json`. Replace [.replaceable]`111122223333` with your account ID. Replace [.replaceable]`EXAMPLED539D4633E53DE1B71EXAMPLE` and [.replaceable]`region-code` with the values returned in the previous step. ++ +[source,json,subs="verbatim,attributes"] +---- +include::iam_snippet:0589c046-fab8-43eb-a517-9c187ca15f54[] +---- +.. Create the role. You can change [.replaceable]`AmazonEKS_S3_CSI_DriverRole` to a different name, but if you do, make sure to change it in later steps too. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam create-role \ + --role-name AmazonEKS_S3_CSI_DriverRole \ + --assume-role-policy-document file://"aws-s3-csi-driver-trust-policy.json" +---- +. Attach the previously created IAM policy to the role with the following command. ++ +[source,bash,subs="verbatim,attributes"] +---- +aws iam attach-role-policy \ + --policy-arn {arn-aws}iam::aws:policy/AmazonS3CSIDriverPolicy \ + --role-name AmazonEKS_S3_CSI_DriverRole +---- ++ +NOTE: The IAM policy `AmazonS3CSIDriverPolicy` was created in the previous section. +. Skip this step if you're installing the driver as an Amazon EKS add-on. For self-managed installations of the driver, create Kubernetes service accounts that are annotated with the ARN of the IAM role that you created. ++ +.. Save the following contents to a file named `mountpoint-s3-service-account.yaml`. Replace [.replaceable]`111122223333` with your account ID. ++ +[source,yaml,subs="verbatim,attributes"] +---- +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/name: aws-mountpoint-s3-csi-driver + name: mountpoint-s3-csi-controller-sa + namespace: kube-system + annotations: + eks.amazonaws.com/role-arn: {arn-aws}iam::111122223333:role/AmazonEKS_S3_CSI_DriverRole +---- +.. Create the Kubernetes service account on your cluster. The Kubernetes service account (`mountpoint-s3-csi-controller-sa`) is annotated with the IAM role that you created named [.replaceable]`AmazonEKS_S3_CSI_DriverRole`. ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl apply -f mountpoint-s3-service-account.yaml +---- ++ +NOTE: When you deploy the plugin in this procedure, it creates and is configured to use a service account named `s3-csi-driver-sa`. + + +[#s3-install-driver] +== Step 3: Install the Mountpoint for Amazon S3 CSI driver + +You may install the Mountpoint for Amazon S3 CSI driver through the Amazon EKS add-on. You can use the following tools to add the add-on to your cluster: + +* <> +* <> +* <> + +Alternatively, you may install Mountpoint for Amazon S3 CSI driver as a self-managed installation. For instructions on doing a self-managed installation, see https://github.com/awslabs/mountpoint-s3-csi-driver/blob/main/docs/install.md#deploy-driver[Installation] on GitHub. + +Starting from `v1.8.0`, you can configure taints to tolerate for the CSI driver's Pods. To do this, either specify a custom set of taints to tolerate with `node.tolerations` or tolorate all taints with `node.tolerateAllTaints`. For more information, see https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/[Taints and Tolerations] in the Kubernetes documentation. + + +=== eksctl [[eksctl_s3_add_store_app_data]] + +*To add the Amazon S3 CSI add-on using `eksctl`* + +Run the following command. Replace [.replaceable]`my-cluster` with the name of your cluster, [.replaceable]`111122223333` with your account ID, and [.replaceable]`AmazonEKS_S3_CSI_DriverRole` with the name of the <>. + +[source,bash,subs="verbatim,attributes"] +---- +eksctl create addon --name aws-mountpoint-s3-csi-driver --cluster my-cluster \ + --service-account-role-arn {arn-aws}iam::111122223333:role/AmazonEKS_S3_CSI_DriverRole --force +---- + +If you remove the [.replaceable]`--force` option and any of the Amazon EKS add-on settings conflict with your existing settings, then updating the Amazon EKS add-on fails, and you receive an error message to help you resolve the conflict. Before specifying this option, make sure that the Amazon EKS add-on doesn't manage settings that you need to manage, because those settings are overwritten with this option. For more information about other options for this setting, see https://eksctl.io/usage/addons/[Addons] in the `eksctl` documentation. For more information about Amazon EKS Kubernetes field management, see <>. + +You can customize `eksctl` through configuration files. For more information, see https://eksctl.io/usage/addons/#working-with-configuration-values[Working with configuration values] in the `eksctl` documentation. The following example shows how to tolerate all taints. + +[source,yaml,subs="verbatim,attributes"] +---- +# config.yaml +... + +addons: +- name: aws-mountpoint-s3-csi-driver + serviceAccountRoleARN: {arn-aws}iam::111122223333:role/AmazonEKS_S3_CSI_DriverRole + configurationValues: |- + node: + tolerateAllTaints: true +---- + +=== {aws-management-console} [[console_s3_add_store_app_data]] +. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. +. In the left navigation pane, choose *Clusters*. +. Choose the name of the cluster that you want to configure the Mountpoint for Amazon S3 CSI add-on for. +. Choose the *Add-ons* tab. +. Choose *Get more add-ons*. +. On the *Select add-ons* page, do the following: ++ +.. In the *Amazon EKS-addons* section, select the *Mountpoint for Amazon S3 CSI Driver* check box. +.. Choose *Next*. +. On the *Configure selected add-ons settings* page, do the following: ++ +.. Select the *Version* you'd like to use. +.. For *Select IAM role*, select the name of an IAM role that you attached the Mountpoint for Amazon S3 CSI driver IAM policy to. +.. (Optional) Update the *Conflict resolution method* after expanding the *Optional configuration settings*. If you select *Override*, one or more of the settings for the existing add-on can be overwritten with the Amazon EKS add-on settings. If you don't enable this option and there's a conflict with your existing settings, the operation fails. You can use the resulting error message to troubleshoot the conflict. Before selecting this option, make sure that the Amazon EKS add-on doesn't manage settings that you need to self-manage. +.. (Optional) Configure tolerations in the *Configuration values* field after expanding the *Optional configuration settings*. +.. Choose *Next*. +. On the *Review and add* page, choose *Create*. After the add-on installation is complete, you see your installed add-on. + +=== {aws} CLI [[awscli_s3_add_store_app_data]] + +*To add the Mountpoint for Amazon S3 CSI add-on using the {aws} CLI* + +Run the following command. Replace [.replaceable]`my-cluster` with the name of your cluster, [.replaceable]`111122223333` with your account ID, and [.replaceable]`AmazonEKS_S3_CSI_DriverRole` with the name of the role that was created earlier. + +[source,bash,subs="verbatim,attributes"] +---- +aws eks create-addon --cluster-name my-cluster --addon-name aws-mountpoint-s3-csi-driver \ + --service-account-role-arn {arn-aws}iam::111122223333:role/AmazonEKS_S3_CSI_DriverRole +---- + +You can customize the command with the `--configuration-values` flag. The following alternative example shows how to tolerate all taints. + +[source,bash,subs="verbatim,attributes"] +---- +aws eks create-addon --cluster-name my-cluster --addon-name aws-mountpoint-s3-csi-driver \ + --service-account-role-arn {arn-aws}iam::111122223333:role/AmazonEKS_S3_CSI_DriverRole \ + --configuration-values '{"node":{"tolerateAllTaints":true}}' +---- + +[#s3-configure-mountpoint] +== Step 4: Configure Mountpoint for Amazon S3 + +In most cases, you can configure Mountpoint for Amazon S3 with only a bucket name. For instructions on configuring Mountpoint for Amazon S3, see https://github.com/awslabs/mountpoint-s3/blob/main/doc/CONFIGURATION.md[Configuring Mountpoint for Amazon S3] on GitHub. + +[#s3-sample-app] +== Step 5: Deploy a sample application + +You can deploy static provisioning to the driver on an existing Amazon S3 bucket. For more information, see https://github.com/awslabs/mountpoint-s3-csi-driver/blob/main/examples/kubernetes/static_provisioning/README.md[Static provisioning] on GitHub. diff --git a/latest/ug/storage/s3-csi.adoc b/latest/ug/storage/s3-csi.adoc index f7b080538..a5f04bd3f 100644 --- a/latest/ug/storage/s3-csi.adoc +++ b/latest/ug/storage/s3-csi.adoc @@ -1,421 +1,31 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[s3-csi,s3-csi.title]] +[#s3-csi] = Access Amazon S3 objects with Mountpoint for Amazon S3 CSI driver -:info_doctype: section -:info_title: Access Amazon S3 objects with Mountpoint for Amazon S3 CSI driver :info_titleabbrev: Mountpoint for Amazon S3 -:keywords: Mountpoint for Amazon S3 CSI driver, storage -:info_abstract: Learn about the Amazon S3 Container Storage Interface (CSI) driver, which provides a \ - CSI interface for managing Amazon S3 files and buckets. [abstract] -- Learn about the Amazon S3 Container Storage Interface (CSI) driver, which provides a CSI interface for managing Amazon S3 files and buckets. -- -With the https://github.com/awslabs/mountpoint-s3-csi-driver[Mountpoint for Amazon S3 Container Storage Interface (CSI) driver], your [.noloc]`Kubernetes` applications can access Amazon S3 objects through a file system interface, achieving high aggregate throughput without changing any application code. Built on https://github.com/awslabs/mountpoint-s3[Mountpoint for Amazon S3], the CSI driver presents an Amazon S3 bucket as a volume that can be accessed by containers in Amazon EKS and self-managed [.noloc]`Kubernetes` clusters. This topic shows you how to deploy the [.noloc]`Mountpoint` for Amazon S3 CSI driver to your Amazon EKS cluster. +With the https://github.com/awslabs/mountpoint-s3-csi-driver[Mountpoint for Amazon S3 Container Storage Interface (CSI) driver], your Kubernetes applications can access Amazon S3 objects through a file system interface, achieving high aggregate throughput without changing any application code. Built on https://github.com/awslabs/mountpoint-s3[Mountpoint for Amazon S3], the CSI driver presents an Amazon S3 bucket as a volume that can be accessed by containers in Amazon EKS and self-managed Kubernetes clusters. -[[s3-csi-considerations,s3-csi-considerations.title]] +[#s3-csi-considerations] == Considerations -* The [.noloc]`Mountpoint` for Amazon S3 CSI driver isn't presently compatible with Windows-based container images. +* The Mountpoint for Amazon S3 CSI driver isn't presently compatible with Windows-based container images. * The Mountpoint for Amazon S3 CSI driver isn't presently compatible with Amazon EKS Hybrid Nodes. -* The [.noloc]`Mountpoint` for Amazon S3 CSI driver doesn't support {aws} Fargate. However, containers that are running in Amazon EC2 (either with Amazon EKS or a custom [.noloc]`Kubernetes` installation) are supported. -* The [.noloc]`Mountpoint` for Amazon S3 CSI driver supports only static provisioning. Dynamic provisioning, or creation of new buckets, isn't supported. +* The Mountpoint for Amazon S3 CSI driver doesn't support {aws} Fargate. However, containers that are running in Amazon EC2 (either with Amazon EKS or a custom Kubernetes installation) are supported. +* The Mountpoint for Amazon S3 CSI driver supports only static provisioning. Dynamic provisioning, or creation of new buckets, isn't supported. + -NOTE: Static provisioning refers to using an existing Amazon S3 bucket that is specified as the `bucketName` in the `volumeAttributes` in the `PersistentVolume` object. For more information, see https://github.com/awslabs/mountpoint-s3-csi-driver/blob/main/examples/kubernetes/static_provisioning/README.md[Static Provisioning] on [.noloc]`GitHub`. -* Volumes mounted with the [.noloc]`Mountpoint` for Amazon S3 CSI driver don't support all POSIX file-system features. For details about file-system behavior, see https://github.com/awslabs/mountpoint-s3/blob/main/doc/SEMANTICS.md[Mountpoint for Amazon S3 file system behavior] on [.noloc]`GitHub`. +NOTE: Static provisioning refers to using an existing Amazon S3 bucket that is specified as the `bucketName` in the `volumeAttributes` in the `PersistentVolume` object. For more information, see https://github.com/awslabs/mountpoint-s3-csi-driver/blob/main/examples/kubernetes/static_provisioning/README.md[Static Provisioning] on GitHub. +* Volumes mounted with the Mountpoint for Amazon S3 CSI driver don't support all POSIX file-system features. For details about file-system behavior, see https://github.com/awslabs/mountpoint-s3/blob/main/doc/SEMANTICS.md[Mountpoint for Amazon S3 file system behavior] on GitHub. +For details on deploying the driver, see <>. For details on removing the driver, see <>. -[[s3-csi-prereqs,s3-csi-prereqs.title]] -== Prerequisites +include::s3-csi-create.adoc[leveloffset=+1] -* An existing {aws} Identity and Access Management (IAM) [.noloc]`OpenID Connect` ([.noloc]`OIDC`) provider for your cluster. To determine whether you already have one, or to create one, see <>. -* Version 2.12.3 or later of the {aws} CLI installed and configured on your device or {aws} CloudShell. -* The `kubectl` command line tool is installed on your device or {aws} CloudShell. The version can be the same as or up to one minor version earlier or later than the [.noloc]`Kubernetes` version of your cluster. For example, if your cluster version is `1.29`, you can use `kubectl` version `1.28`, `1.29`, or `1.30` with it. To install or upgrade `kubectl`, see <>. +include::removing-s3-csi-eks-add-on.adoc[leveloffset=+1] - -[[s3-create-iam-policy,s3-create-iam-policy.title]] -== Create an IAM policy - -The [.noloc]`Mountpoint` for Amazon S3 CSI driver requires Amazon S3 permissions to interact with your file system. This section shows how to create an IAM policy that grants the necessary permissions. - -The following example policy follows the IAM permission recommendations for [.noloc]`Mountpoint`. Alternatively, you can use the {aws} managed policy link:iam/home?#/policies/arn:aws:iam::aws:policy/AmazonS3FullAccess$jsonEditor[AmazonS3FullAccess,type="console"], but this managed policy grants more permissions than are needed for [.noloc]`Mountpoint`. - -For more information about the recommended permissions for [.noloc]`Mountpoint`, see https://github.com/awslabs/mountpoint-s3/blob/main/doc/CONFIGURATION.md#iam-permissions[Mountpoint IAM permissions] on [.noloc]`GitHub`. - -. Open the IAM console at https://console.aws.amazon.com/iam/. -. In the left navigation pane, choose *Policies*. -. On the *Policies* page, choose *Create policy*. -. For *Policy editor*, choose *JSON*. -. Under *Policy editor*, copy and paste the following: -+ -IMPORTANT: Replace `amzn-s3-demo-bucket1` with your own Amazon S3 bucket name. -+ -[source,json,subs="verbatim,attributes"] ----- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "MountpointFullBucketAccess", - "Effect": "Allow", - "Action": [ - "s3:ListBucket" - ], - "Resource": [ - "{arn-aws}s3:::amzn-s3-demo-bucket1" - ] - }, - { - "Sid": "MountpointFullObjectAccess", - "Effect": "Allow", - "Action": [ - "s3:GetObject", - "s3:PutObject", - "s3:AbortMultipartUpload", - "s3:DeleteObject" - ], - "Resource": [ - "{arn-aws}s3:::amzn-s3-demo-bucket1/*" - ] - } - ] -} ----- -+ -Directory buckets, introduced with the Amazon S3 Express One Zone storage class, use a different authentication mechanism from general purpose buckets. Instead of using `s3:*` actions, you should use the `s3express:CreateSession` action. For information about directory buckets, see link:AmazonS3/latest/userguide/directory-buckets-overview.html[Directory buckets,type="documentation"] in the _Amazon S3 User Guide_. -+ -Below is an example of least-privilege policy that you would use for a directory bucket. -+ -[source,json,subs="verbatim,attributes"] ----- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": "s3express:CreateSession", - "Resource": "{arn-aws}s3express:us-west-2:111122223333:bucket/amzn-s3-demo-bucket1--usw2-az1--x-s3" - } - ] -} ----- -. Choose *Next*. -. On the *Review and create* page, name your policy. This example walkthrough uses the name `AmazonS3CSIDriverPolicy`. -. Choose *Create policy*. - - -[[s3-create-iam-role,s3-create-iam-role.title]] -== Create an IAM role - -The [.noloc]`Mountpoint` for Amazon S3 CSI driver requires Amazon S3 permissions to interact with your file system. This section shows how to create an IAM role to delegate these permissions. To create this role, you can use one of these tools: - -* <> -* <> -* <> - -[NOTE] -==== - -The IAM policy `AmazonS3CSIDriverPolicy` was created in the previous section. - -==== - -=== [.noloc]`eksctl` [[eksctl_s3_store_app_data]] - -*To create your [.noloc]`Mountpoint` for Amazon S3 CSI driver IAM role with `eksctl`* - -To create the IAM role and the [.noloc]`Kubernetes` service account, run the following commands. These commands also attach the `AmazonS3CSIDriverPolicy` IAM policy to the role, annotate the [.noloc]`Kubernetes` service account (`s3-csi-controller-sa`) with the IAM role's Amazon Resource Name (ARN), and add the [.noloc]`Kubernetes` service account name to the trust policy for the IAM role. - -[source,bash,subs="verbatim,attributes"] ----- -CLUSTER_NAME=my-cluster -REGION=region-code -ROLE_NAME=AmazonEKS_S3_CSI_DriverRole -POLICY_ARN=AmazonEKS_S3_CSI_DriverRole_ARN -eksctl create iamserviceaccount \ - --name s3-csi-driver-sa \ - --namespace kube-system \ - --cluster $CLUSTER_NAME \ - --attach-policy-arn $POLICY_ARN \ - --approve \ - --role-name $ROLE_NAME \ - --region $REGION \ - --role-only ----- - - -=== {aws-management-console} [[console_s3_store_app_data]] -. Open the IAM console at https://console.aws.amazon.com/iam/. -. In the left navigation pane, choose *Roles*. -. On the *Roles* page, choose *Create role*. -. On the *Select trusted entity* page, do the following: -+ -.. In the *Trusted entity type* section, choose *Web identity*. -.. For *Identity provider*, choose the *[.noloc]`OpenID Connect` provider URL* for your cluster (as shown under *Overview* in Amazon EKS). -+ -If no URLs are shown, review the <>. -.. For *Audience*, choose `sts.amazonaws.com`. -.. Choose *Next*. -. On the *Add permissions* page, do the following: -+ -.. In the *Filter policies* box, enter [.noloc]`AmazonS3CSIDriverPolicy`. -+ -NOTE: This policy was created in the previous section. -.. Select the check box to the left of the `AmazonS3CSIDriverPolicy` result that was returned in the search. -.. Choose *Next*. -. On the *Name, review, and create* page, do the following: -+ -.. For *Role name*, enter a unique name for your role, such as [.noloc]`AmazonEKS_S3_CSI_DriverRole`. -.. Under *Add tags (Optional)*, add metadata to the role by attaching tags as key-value pairs. For more information about using tags in IAM, see link:IAM/latest/UserGuide/id_tags.html[Tagging IAM resources,type="documentation"] in the _IAM User Guide_. -.. Choose *Create role*. -. After the role is created, choose the role in the console to open it for editing. -. Choose the *Trust relationships* tab, and then choose *Edit trust policy*. -. Find the line that looks similar to the following: -+ -[source,json,subs="verbatim,attributes"] ----- -"oidc.eks.region-code.amazonaws.com/id/EXAMPLED539D4633E53DE1B71EXAMPLE:aud": "sts.amazonaws.com" ----- -+ -Add a comma to the end of the previous line, and then add the following line after it. Replace [.replaceable]`region-code` with the {aws} Region that your cluster is in. Replace [.replaceable]`EXAMPLED539D4633E53DE1B71EXAMPLE` with your cluster's OIDC provider ID. -+ -[source,json,subs="verbatim,attributes"] ----- -"oidc.eks.region-code.amazonaws.com/id/EXAMPLED539D4633E53DE1B71EXAMPLE:sub": "system:serviceaccount:kube-system:s3-csi-driver-sa" ----- -. Ensure that the `Condition` operator is set to `"StringEquals"`. -. Choose *Update policy* to finish. - -=== {aws} CLI [[awscli_s3_store_app_data]] -. View the OIDC provider URL for your cluster. Replace [.replaceable]`my-cluster` with the name of your cluster. If the output from the command is `None`, review the <>. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws eks describe-cluster --name my-cluster --query "cluster.identity.oidc.issuer" --output text ----- -+ -An example output is as follows. -+ -[source,bash,subs="verbatim,attributes"] ----- -https://oidc.eks.region-code.amazonaws.com/id/EXAMPLED539D4633E53DE1B71EXAMPLE ----- -. Create the IAM role, granting the [.noloc]`Kubernetes` service account the `AssumeRoleWithWebIdentity` action. -+ -.. Copy the following contents to a file named `aws-s3-csi-driver-trust-policy.json`. Replace [.replaceable]`111122223333` with your account ID. Replace [.replaceable]`EXAMPLED539D4633E53DE1B71EXAMPLE` and [.replaceable]`region-code` with the values returned in the previous step. -+ -[source,json,subs="verbatim,attributes"] ----- -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "{arn-aws}iam::111122223333:oidc-provider/oidc.eks.region-code.amazonaws.com/id/EXAMPLED539D4633E53DE1B71EXAMPLE" - }, - "Action": "sts:AssumeRoleWithWebIdentity", - "Condition": { - "StringEquals": { - "oidc.eks.region-code.amazonaws.com/id/EXAMPLED539D4633E53DE1B71EXAMPLE:sub": "system:serviceaccount:kube-system:s3-csi-driver-sa", - "oidc.eks.region-code.amazonaws.com/id/EXAMPLED539D4633E53DE1B71EXAMPLE:aud": "sts.amazonaws.com" - } - } - } - ] -} ----- -.. Create the role. You can change [.replaceable]`AmazonEKS_S3_CSI_DriverRole` to a different name, but if you do, make sure to change it in later steps too. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws iam create-role \ - --role-name AmazonEKS_S3_CSI_DriverRole \ - --assume-role-policy-document file://"aws-s3-csi-driver-trust-policy.json" ----- -. Attach the previously created IAM policy to the role with the following command. -+ -[source,bash,subs="verbatim,attributes"] ----- -aws iam attach-role-policy \ - --policy-arn {arn-aws}iam::aws:policy/AmazonS3CSIDriverPolicy \ - --role-name AmazonEKS_S3_CSI_DriverRole ----- -+ -NOTE: The IAM policy `AmazonS3CSIDriverPolicy` was created in the previous section. -. Skip this step if you're installing the driver as an Amazon EKS add-on. For self-managed installations of the driver, create [.noloc]`Kubernetes` service accounts that are annotated with the ARN of the IAM role that you created. -+ -.. Save the following contents to a file named `mountpoint-s3-service-account.yaml`. Replace [.replaceable]`111122223333` with your account ID. -+ -[source,yaml,subs="verbatim,attributes"] ----- ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - app.kubernetes.io/name: aws-mountpoint-s3-csi-driver - name: mountpoint-s3-csi-controller-sa - namespace: kube-system - annotations: - eks.amazonaws.com/role-arn: {arn-aws}iam::111122223333:role/AmazonEKS_S3_CSI_DriverRole ----- -.. Create the [.noloc]`Kubernetes` service account on your cluster. The [.noloc]`Kubernetes` service account (`mountpoint-s3-csi-controller-sa`) is annotated with the IAM role that you created named [.replaceable]`AmazonEKS_S3_CSI_DriverRole`. -+ -[source,bash,subs="verbatim,attributes"] ----- -kubectl apply -f mountpoint-s3-service-account.yaml ----- -+ -NOTE: When you deploy the plugin in this procedure, it creates and is configured to use a service account named `s3-csi-driver-sa`. - - -[[s3-install-driver,s3-install-driver.title]] -== Install the [.noloc]`Mountpoint` for Amazon S3 CSI driver - -You may install the [.noloc]`Mountpoint` for Amazon S3 CSI driver through the Amazon EKS add-on. You can use the following tools to add the add-on to your cluster: - -* <> -* <> -* <> - -Alternatively, you may install [.noloc]`Mountpoint` for Amazon S3 CSI driver as a self-managed installation. For instructions on doing a self-managed installation, see https://github.com/awslabs/mountpoint-s3-csi-driver/blob/main/docs/install.md#deploy-driver[Installation] on [.noloc]`GitHub`. - -Starting from `v1.8.0`, you can configure taints to tolerate for the CSI driver's [.noloc]`Pods`. To do this, either specify a custom set of taints to tolerate with `node.tolerations` or tolorate all taints with `node.tolerateAllTaints`. For more information, see https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/[Taints and Tolerations] in the [.noloc]`Kubernetes` documentation. - - -=== [.noloc]`eksctl` [[eksctl_s3_add_store_app_data]] - -*To add the Amazon S3 CSI add-on using `eksctl`* - -Run the following command. Replace [.replaceable]`my-cluster` with the name of your cluster, [.replaceable]`111122223333` with your account ID, and [.replaceable]`AmazonEKS_S3_CSI_DriverRole` with the name of the <>. - -[source,bash,subs="verbatim,attributes"] ----- -eksctl create addon --name aws-mountpoint-s3-csi-driver --cluster my-cluster \ - --service-account-role-arn {arn-aws}iam::111122223333:role/AmazonEKS_S3_CSI_DriverRole --force ----- - -If you remove the [.replaceable]`--force` option and any of the Amazon EKS add-on settings conflict with your existing settings, then updating the Amazon EKS add-on fails, and you receive an error message to help you resolve the conflict. Before specifying this option, make sure that the Amazon EKS add-on doesn't manage settings that you need to manage, because those settings are overwritten with this option. For more information about other options for this setting, see https://eksctl.io/usage/addons/[Addons] in the `eksctl` documentation. For more information about Amazon EKS [.noloc]`Kubernetes` field management, see <>. - -You can customize `eksctl` through configuration files. For more information, see https://eksctl.io/usage/addons/#working-with-configuration-values[Working with configuration values] in the `eksctl` documentation. The following example shows how to tolerate all taints. - -[source,yaml,subs="verbatim,attributes"] ----- -# config.yaml -... - -addons: -- name: aws-mountpoint-s3-csi-driver - serviceAccountRoleARN: {arn-aws}iam::111122223333:role/AmazonEKS_S3_CSI_DriverRole - configurationValues: |- - node: - tolerateAllTaints: true ----- - -=== {aws-management-console} [[console_s3_add_store_app_data]] -. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. -. In the left navigation pane, choose *Clusters*. -. Choose the name of the cluster that you want to configure the [.noloc]`Mountpoint` for Amazon S3 CSI add-on for. -. Choose the *Add-ons* tab. -. Choose *Get more add-ons*. -. On the *Select add-ons* page, do the following: -+ -.. In the *Amazon EKS-addons* section, select the *[.noloc]`Mountpoint` for Amazon S3 CSI Driver* check box. -.. Choose *Next*. -. On the *Configure selected add-ons settings* page, do the following: -+ -.. Select the *Version* you'd like to use. -.. For *Select IAM role*, select the name of an IAM role that you attached the [.noloc]`Mountpoint` for Amazon S3 CSI driver IAM policy to. -.. (Optional) Update the *Conflict resolution method* after expanding the *Optional configuration settings*. If you select *Override*, one or more of the settings for the existing add-on can be overwritten with the Amazon EKS add-on settings. If you don't enable this option and there's a conflict with your existing settings, the operation fails. You can use the resulting error message to troubleshoot the conflict. Before selecting this option, make sure that the Amazon EKS add-on doesn't manage settings that you need to self-manage. -.. (Optional) Configure tolerations in the *Configuration values* field after expanding the *Optional configuration settings*. -.. Choose *Next*. -. On the *Review and add* page, choose *Create*. After the add-on installation is complete, you see your installed add-on. - -=== {aws} CLI [[awscli_s3_add_store_app_data]] - -*To add the [.noloc]`Mountpoint` for Amazon S3 CSI add-on using the {aws} CLI* - -Run the following command. Replace [.replaceable]`my-cluster` with the name of your cluster, [.replaceable]`111122223333` with your account ID, and [.replaceable]`AmazonEKS_S3_CSI_DriverRole` with the name of the role that was created earlier. - -[source,bash,subs="verbatim,attributes"] ----- -aws eks create-addon --cluster-name my-cluster --addon-name aws-mountpoint-s3-csi-driver \ - --service-account-role-arn {arn-aws}iam::111122223333:role/AmazonEKS_S3_CSI_DriverRole ----- - -You can customize the command with the `--configuration-values` flag. The following alternative example shows how to tolerate all taints. - -[source,bash,subs="verbatim,attributes"] ----- -aws eks create-addon --cluster-name my-cluster --addon-name aws-mountpoint-s3-csi-driver \ - --service-account-role-arn {arn-aws}iam::111122223333:role/AmazonEKS_S3_CSI_DriverRole \ - --configuration-values '{"node":{"tolerateAllTaints":true}}' ----- - - -[[s3-configure-mountpoint,s3-configure-mountpoint.title]] -== Configure [.noloc]`Mountpoint` for Amazon S3 - -In most cases, you can configure [.noloc]`Mountpoint` for Amazon S3 with only a bucket name. For instructions on configuring [.noloc]`Mountpoint` for Amazon S3, see https://github.com/awslabs/mountpoint-s3/blob/main/doc/CONFIGURATION.md[Configuring Mountpoint for Amazon S3] on [.noloc]`GitHub`. - -[[s3-sample-app,s3-sample-app.title]] -== Deploy a sample application - -You can deploy static provisioning to the driver on an existing Amazon S3 bucket. For more information, see https://github.com/awslabs/mountpoint-s3-csi-driver/blob/main/examples/kubernetes/static_provisioning/README.md[Static provisioning] on [.noloc]`GitHub`. - -[[removing-s3-csi-eks-add-on,removing-s3-csi-eks-add-on.title]] -== Remove [.noloc]`Mountpoint` for Amazon S3 CSI Driver - -You have two options for removing an Amazon EKS add-on. - -* *Preserve add-on software on your cluster* – This option removes Amazon EKS management of any settings. It also removes the ability for Amazon EKS to notify you of updates and automatically update the Amazon EKS add-on after you initiate an update. However, it preserves the add-on software on your cluster. This option makes the add-on a self-managed installation, rather than an Amazon EKS add-on. With this option, there's no downtime for the add-on. The commands in this procedure use this option. -* *Remove add-on software entirely from your cluster* – We recommend that you remove the Amazon EKS add-on from your cluster only if there are no resources on your cluster that are dependent on it. To do this option, delete `--preserve` from the command you use in this procedure. - -If the add-on has an IAM account associated with it, the IAM account isn't removed. - -You can use the following tools to remove the Amazon S3 CSI add-on: - -* <> -* <> -* <> - -=== [.noloc]`eksctl` [[eksctl_s3_remove_store_app_data]] - -*To remove the Amazon S3 CSI add-on using `eksctl`* - -Replace [.replaceable]`my-cluster` with the name of your cluster, and then run the following command. - -[source,bash,subs="verbatim,attributes"] ----- -eksctl delete addon --cluster my-cluster --name aws-mountpoint-s3-csi-driver --preserve ----- - -=== {aws-management-console} [[console_s3_remove_store_app_data]] -. Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. -. In the left navigation pane, choose *Clusters*. -. Choose the name of the cluster that you want to remove the Amazon EBS CSI add-on for. -. Choose the *Add-ons* tab. -. Choose *[.noloc]`Mountpoint` for Amazon S3 CSI Driver*. -. Choose *Remove*. -. In the *Remove: aws-mountpoint-s3-csi-driver* confirmation dialog box, do the following: -+ -.. If you want Amazon EKS to stop managing settings for the add-on, select *Preserve on cluster*. Do this if you want to retain the add-on software on your cluster. This is so that you can manage all of the settings of the add-on on your own. -.. Enter `aws-mountpoint-s3-csi-driver`. -.. Select *Remove*. - -=== {aws} CLI [[awscli_s3_remove_store_app_data]] - -*To remove the Amazon S3 CSI add-on using the {aws} CLI* - -Replace [.replaceable]`my-cluster` with the name of your cluster, and then run the following command. - -[source,bash,subs="verbatim,attributes"] ----- -aws eks delete-addon --cluster-name my-cluster --addon-name aws-mountpoint-s3-csi-driver --preserve ----- diff --git a/latest/ug/storage/storage.adoc b/latest/ug/storage/storage.adoc index 0c4b037c0..301929e62 100644 --- a/latest/ug/storage/storage.adoc +++ b/latest/ug/storage/storage.adoc @@ -1,26 +1,16 @@ -//!!NODE_ROOT include::../attributes.txt[] -[[storage,storage.title]] -= Store application data for your cluster -:doctype: book -:sectnums: -:toc: left -:icons: font -:experimental: -:idprefix: -:idseparator: - -:sourcedir: . -:info_doctype: chapter -:info_title: Store application data for your cluster -:info_titleabbrev: Store app data -:keywords: persistent, data, app, storage -:info_abstract: This chapter covers storage options for Amazon EKS clusters. + +[#storage] += Use application data storage for your cluster +:info_titleabbrev: App data storage [abstract] -- This chapter covers storage options for Amazon EKS clusters. -- +You can use a range of {aws} storage services with Amazon EKS for the storage needs of your applications. Through an {aws}-supported breadth of Container Storage Interface (CSI) drivers, you can easily use Amazon EBS, Amazon S3, Amazon EFS, Amazon FSX, and Amazon File Cache for the storage needs of your applications running on Amazon EKS. + This chapter covers storage options for Amazon EKS clusters. [.topiclist] @@ -28,26 +18,16 @@ This chapter covers storage options for Amazon EKS clusters. include::ebs-csi.adoc[leveloffset=+1] - -include::ebs-csi-migration-faq.adoc[leveloffset=+1] - - include::efs-csi.adoc[leveloffset=+1] - include::fsx-csi.adoc[leveloffset=+1] - include::fsx-ontap.adoc[leveloffset=+1] - include::fsx-openzfs-csi.adoc[leveloffset=+1] - include::file-cache-csi.adoc[leveloffset=+1] - include::s3-csi.adoc[leveloffset=+1] - include::csi-snapshot-controller.adoc[leveloffset=+1] diff --git a/latest/ug/troubleshooting/troubleshooting.adoc b/latest/ug/troubleshooting/troubleshooting.adoc index e2a1ff955..e07454ac3 100644 --- a/latest/ug/troubleshooting/troubleshooting.adoc +++ b/latest/ug/troubleshooting/troubleshooting.adoc @@ -1,33 +1,19 @@ -//!!NODE_ROOT -[[troubleshooting,troubleshooting.title]] +include::../attributes.txt[] + +[#troubleshooting] = Troubleshoot problems with Amazon EKS clusters and nodes -:doctype: book -:sectnums: -:toc: left -:icons: font -:experimental: -:idprefix: -:idseparator: - -:sourcedir: . -:info_doctype: chapter -:info_title: Troubleshoot problems with Amazon EKS clusters and nodes :info_titleabbrev: Troubleshooting -:keywords: troubleshooting, help, FAQ -:info_abstract: This chapter covers some common errors that you may see while using Amazon EKS and how \ - to work around them. - -include::../attributes.txt[] [abstract] -- This chapter covers some common errors that you may see while using Amazon EKS and how to work around them. -- -This chapter covers some common errors that you may see while using Amazon EKS and how to work around them. If you need to troubleshoot specific Amazon EKS areas, see the separate <>, <>, and https://aws-otel.github.io/docs/getting-started/adot-eks-add-on/troubleshooting[Troubleshooting for ADOT using EKS Add-Ons] topics. +This chapter covers some common errors that you may see while using Amazon EKS and how to work around them. If you need to troubleshoot specific Amazon EKS areas, see the separate <>, <>, and https://aws-otel.github.io/docs/getting-started/adot-eks-add-on/troubleshooting[Troubleshooting for ADOT using EKS Add-Ons] topics. For other troubleshooting information, see https://repost.aws/tags/knowledge-center/TA4IvCeWI1TE66q4jEj4Z9zg/amazon-elastic-kubernetes-service[Knowledge Center content about Amazon Elastic Kubernetes Service] on _{aws} re:Post_. -[[ice,ice.title]] +[#ice] == Insufficient capacity If you receive the following error while attempting to create an Amazon EKS cluster, then one of the Availability Zones you specified doesn't have sufficient capacity to support a cluster. @@ -38,20 +24,20 @@ Retry creating your cluster with subnets in your cluster VPC that are hosted in There are Availability Zones that a cluster can't reside in. Compare the Availability Zones that your subnets are in with the list of Availability Zones in the <>. -[[worker-node-fail,worker-node-fail.title]] +[#worker-node-fail] == Nodes fail to join cluster There are a few common reasons that prevent nodes from joining the cluster: -* If the nodes are managed nodes, Amazon EKS adds entries to the `aws-auth` `ConfigMap` when you create the node group. If the entry was removed or modified, then you need to re-add it. For more information, enter `eksctl create iamidentitymapping --help` in your terminal. You can view your current `aws-auth` `ConfigMap` entries by replacing [.replaceable]`my-cluster` in the following command with the name of your cluster and then running the modified command: `eksctl get iamidentitymapping --cluster [.replaceable]``my-cluster```. The ARN of the role that you specify can't include a link:IAM/latest/UserGuide/reference_identifiers.html#identifiers-friendly-names[path,type="documentation"] other than `/`. For example, if the name of your role is `development/apps/my-role`, you'd need to change it to `my-role` when specifying the ARN for the role. Make sure that you specify the node IAM role ARN (not the instance profile ARN). +* If the nodes are managed nodes, Amazon EKS adds entries to the `aws-auth` `ConfigMap` when you create the node group. If the entry was removed or modified, then you need to re-add it. For more information, enter `eksctl create iamidentitymapping --help` in your terminal. You can view your current `aws-auth` `ConfigMap` entries by replacing [.replaceable]`my-cluster` in the following command with the name of your cluster and then running the modified command: `eksctl get iamidentitymapping --cluster [.replaceable]``my-cluster```. The ARN of the role that you specify can't include a link:IAM/latest/UserGuide/reference_identifiers.html#identifiers-friendly-names[path,type="documentation"] other than `/`. For example, if the name of your role is `development/apps/my-role`, you'd need to change it to `my-role` when specifying the ARN for the role. Make sure that you specify the node IAM role ARN (not the instance profile ARN). + If the nodes are self-managed, and you haven't created <> for the ARN of the node's IAM role, then run the same commands listed for managed nodes. If you have created an access entry for the ARN for your node IAM role, then it might not be configured properly in the access entry. Make sure that the node IAM role ARN (not the instance profile ARN) is specified as the principal ARN in your `aws-auth` `ConfigMap` entry or access entry. For more information about access entries, see <>. * The *ClusterName* in your node {aws} CloudFormation template doesn't exactly match the name of the cluster you want your nodes to join. Passing an incorrect value to this field results in an incorrect configuration of the node's `/var/lib/kubelet/kubeconfig` file, and the nodes will not join the cluster. * The node is not tagged as being _owned_ by the cluster. Your nodes must have the following tag applied to them, where [.replaceable]`my-cluster` is replaced with the name of your cluster. + -[cols="1,1", options="header"] +[%header,cols="2"] |=== |Key |Value @@ -60,17 +46,17 @@ If the nodes are self-managed, and you haven't created <.compute.internal` and `domain-name-servers:AmazonProvidedDNS`. For more information, see link:vpc/latest/userguide/VPC_DHCP_Options.html#AmazonDNS[DHCP options sets,type="documentation"] in the _Amazon VPC User Guide_. -* If the nodes in the managed node group do not connect to the cluster within 15 minutes, a health issue of "`NodeCreationFailure`" will be emitted and the console status will be set to `Create failed`. For [.noloc]`Windows` AMIs that have slow launch times, this issue can be resolved using link:AWSEC2/latest/WindowsGuide/win-ami-config-fast-launch.html[fast launch,type="documentation"]. +* The nodes may not be able to access the cluster using a public IP address. Ensure that nodes deployed in public subnets are assigned a public IP address. If not, you can associate an Elastic IP address to a node after it's launched. For more information, see link:AWSEC2/latest/UserGuide/elastic-ip-addresses-eip.html#using-instance-addressing-eips-associating[Associating an Elastic IP address with a running instance or network interface,type="documentation"]. If the public subnet is not set to automatically assign public IP addresses to instances deployed to it, then we recommend enabling that setting. For more information, see link:vpc/latest/userguide/vpc-ip-addressing.html#subnet-public-ip[Modifying the public IPv4 addressing attribute for your subnet,type="documentation"]. If the node is deployed to a private subnet, then the subnet must have a route to a NAT gateway that has a public IP address assigned to it. +* The {aws} STS endpoint for the {aws} Region that you're deploying the nodes to is not enabled for your account. To enable the region, see link:IAM/latest/UserGuide/id_credentials_temp_enable-regions.html#sts-regions-activate-deactivate[Activating and deactivating {aws} STS in an {aws} Region,type="documentation"]. +* The node doesn't have a private DNS entry, resulting in the `kubelet` log containing a `node "" not found` error. Ensure that the VPC where the node is created has values set for `domain-name` and `domain-name-servers` as `Options` in a `DHCP options set`. The default values are `domain-name:.compute.internal` and `domain-name-servers:AmazonProvidedDNS`. For more information, see link:vpc/latest/userguide/VPC_DHCP_Options.html#AmazonDNS[DHCP options sets,type="documentation"] in the _Amazon VPC User Guide_. +* If the nodes in the managed node group do not connect to the cluster within 15 minutes, a health issue of "`NodeCreationFailure`" will be emitted and the console status will be set to `Create failed`. For Windows AMIs that have slow launch times, this issue can be resolved using link:AWSEC2/latest/WindowsGuide/win-ami-config-fast-launch.html[fast launch,type="documentation"]. To identify and troubleshoot common causes that prevent worker nodes from joining a cluster, you can use the `AWSSupport-TroubleshootEKSWorkerNode` runbook. For more information, see `link:systems-manager-automation-runbooks/latest/userguide/automation-awssupport-troubleshooteksworkernode.html[AWSSupport-TroubleshootEKSWorkerNode,type="documentation"]` in the _{aws} Systems Manager Automation runbook reference_. -[[unauthorized,unauthorized.title]] +[#unauthorized] == Unauthorized or access denied (`kubectl`) -If you receive one of the following errors while running `kubectl` commands, then you don't have `kubectl` configured properly for Amazon EKS or the credentials for the IAM principal (role or user) that you're using don't map to a [.noloc]`Kubernetes` username that has sufficient permissions to [.noloc]`Kubernetes` objects on your Amazon EKS cluster. +If you receive one of the following errors while running `kubectl` commands, then you don't have `kubectl` configured properly for Amazon EKS or the credentials for the IAM principal (role or user) that you're using don't map to a Kubernetes username that has sufficient permissions to Kubernetes objects on your Amazon EKS cluster. @@ -83,20 +69,20 @@ This could be due to one of the following reasons: * The cluster was created with credentials for one IAM principal and `kubectl` is configured to use credentials for a different IAM principal. To resolve this, update your `kube config` file to use the credentials that created the cluster. For more information, see <>. -* If your cluster meets the minimum platform requirements in the prerequisites section of <>, an access entry doesn't exist with your IAM principal. If it exists, it doesn't have the necessary [.noloc]`Kubernetes` group names defined for it, or doesn't have the proper access policy associated to it. For more information, see <>. -* If your cluster doesn't meet the minimum platform requirements in <>, an entry with your IAM principal doesn't exist in the `aws-auth` `ConfigMap`. If it exists, it's not mapped to [.noloc]`Kubernetes` group names that are bound to a [.noloc]`Kubernetes` `Role` or `ClusterRole` with the necessary permissions. For more information about [.noloc]`Kubernetes` role-based authorization (RBAC) objects, see https://kubernetes.io/docs/reference/access-authn-authz/rbac/[Using RBAC authorization] in the [.noloc]`Kubernetes` documentation. You can view your current `aws-auth` `ConfigMap` entries by replacing [.replaceable]`my-cluster` in the following command with the name of your cluster and then running the modified command: `eksctl get iamidentitymapping --cluster [.replaceable]``my-cluster```. If an entry for with the ARN of your IAM principal isn't in the `ConfigMap`, enter `eksctl create iamidentitymapping --help` in your terminal to learn how to create one. +* If your cluster meets the minimum platform requirements in the prerequisites section of <>, an access entry doesn't exist with your IAM principal. If it exists, it doesn't have the necessary Kubernetes group names defined for it, or doesn't have the proper access policy associated to it. For more information, see <>. +* If your cluster doesn't meet the minimum platform requirements in <>, an entry with your IAM principal doesn't exist in the `aws-auth` `ConfigMap`. If it exists, it's not mapped to Kubernetes group names that are bound to a Kubernetes `Role` or `ClusterRole` with the necessary permissions. For more information about Kubernetes role-based authorization (RBAC) objects, see https://kubernetes.io/docs/reference/access-authn-authz/rbac/[Using RBAC authorization] in the Kubernetes documentation. You can view your current `aws-auth` `ConfigMap` entries by replacing [.replaceable]`my-cluster` in the following command with the name of your cluster and then running the modified command: `eksctl get iamidentitymapping --cluster [.replaceable]``my-cluster```. If an entry for with the ARN of your IAM principal isn't in the `ConfigMap`, enter `eksctl create iamidentitymapping --help` in your terminal to learn how to create one. -If you install and configure the {aws} CLI, you can configure the IAM credentials that you use. For more information, see link:cli/latest/userguide/cli-chap-getting-started.html[Configuring the {aws} CLI,type="documentation"] in the _{aws} Command Line Interface User Guide_. You can also configure `kubectl` to use an IAM role, if you assume an IAM role to access [.noloc]`Kubernetes` objects on your cluster. For more information, see <>. +If you install and configure the {aws} CLI, you can configure the IAM credentials that you use. For more information, see link:cli/latest/userguide/cli-chap-getting-started.html[Configuring the {aws} CLI,type="documentation"] in the _{aws} Command Line Interface User Guide_. You can also configure `kubectl` to use an IAM role, if you assume an IAM role to access Kubernetes objects on your cluster. For more information, see <>. -[[python-version,python-version.title]] +[#python-version] == `hostname doesn't match` Your system's Python version must be `2.7.9` or later. Otherwise, you receive `hostname doesn't match` errors with {aws} CLI calls to Amazon EKS. For more information, see https://requests.readthedocs.io/en/latest/community/faq.html#what-are-hostname-doesn-t-match-errors[What are "hostname doesn't match" errors?] in the _Python Requests Frequently Asked Questions_. -[[troubleshoot-docker-cidr,troubleshoot-docker-cidr.title]] +[#troubleshoot-docker-cidr] == `getsockopt: no route to host` -[.noloc]`Docker` runs in the `172.17.0.0/16` CIDR range in Amazon EKS clusters. We recommend that your cluster's VPC subnets do not overlap this range. Otherwise, you will receive the following error: +Docker runs in the `172.17.0.0/16` CIDR range in Amazon EKS clusters. We recommend that your cluster's VPC subnets do not overlap this range. Otherwise, you will receive the following error: [source,bash,subs="verbatim,attributes"] ---- @@ -104,12 +90,12 @@ Error: : error upgrading connection: error dialing backend: dial tcp 172.17. ---- -[[instances-failed-to-join,instances-failed-to-join.title]] +[#instances-failed-to-join] == `Instances failed to join the Kubernetes cluster` If you receive the error `Instances failed to join the Kubernetes cluster` in the {aws-management-console}, ensure that either the cluster's private endpoint access is enabled, or that you have correctly configured CIDR blocks for public endpoint access. For more information, see <>. -[[troubleshoot-managed-node-groups,troubleshoot-managed-node-groups.title]] +[#troubleshoot-managed-node-groups] == Managed node group error codes If your managed node group encounters a hardware health issue, Amazon EKS returns an error code to help you to diagnose the issue. These health checks don't detect software issues because they are based on link:AWSEC2/latest/UserGuide/monitoring-system-instance-status-check.html[Amazon EC2 health checks,type="documentation"]. The following list describes the error codes. @@ -117,7 +103,7 @@ If your managed node group encounters a hardware health issue, Amazon EKS return *AccessDenied*:: -Amazon EKS or one or more of your managed nodes is failing to authenticate or authorize with your [.noloc]`Kubernetes` cluster API server. For more information about resolving a common cause, see <>. Private [.noloc]`Windows` AMIs can also cause this error code alongside the `Not authorized for images` error message. For more information, see <>. +Amazon EKS or one or more of your managed nodes is failing to authenticate or authorize with your Kubernetes cluster API server. For more information about resolving a common cause, see <>. Private Windows AMIs can also cause this error code alongside the `Not authorized for images` error message. For more information, see <>. *AmiIdNotFound*:: @@ -129,7 +115,7 @@ We couldn't find the Auto Scaling group associated with the managed node group. *ClusterUnreachable*:: -Amazon EKS or one or more of your managed nodes is unable to communicate with your [.noloc]`Kubernetes` cluster API server. This can happen if there are network disruptions or if API servers are timing out processing requests. +Amazon EKS or one or more of your managed nodes is unable to communicate with your Kubernetes cluster API server. This can happen if there are network disruptions or if API servers are timing out processing requests. *Ec2SecurityGroupNotFound*:: @@ -180,7 +166,7 @@ These errors are usually caused by an Amazon EKS server-side issue. -[[access-denied-managed-node-groups,access-denied-managed-node-groups.title]] +[#access-denied-managed-node-groups] .Fixing a common cause of `AccessDenied` errors for managed node groups [%collapsible] ==== @@ -316,78 +302,71 @@ kubectl apply -f eks-node-manager-role.yaml Retry the node group operation to see if that resolved your issue. ==== -[[not-authorized-for-images,not-authorized-for-images.title]] +[#not-authorized-for-images] == `Not authorized for images` -One potential cause of a `Not authorized for images` error message is using a private Amazon EKS [.noloc]`Windows` AMI to launch [.noloc]`Windows` managed node groups. After releasing new [.noloc]`Windows` AMIs, {aws} makes AMIs that are older than 4 months private, which makes them no longer accessible. If your managed node group is using a private [.noloc]`Windows` AMI, consider <>. While we can't guarantee that we can provide access to AMIs that have been made private, you can request access by filing a ticket with {aws} Support. For more information, see link:AWSEC2/latest/WindowsGuide/aws-windows-ami.html#ami-patches-security-ID[Patches, security updates, and AMI IDs,type="documentation"] in the _Amazon EC2 User Guide_. +One potential cause of a `Not authorized for images` error message is using a private Amazon EKS Windows AMI to launch Windows managed node groups. After releasing new Windows AMIs, {aws} makes AMIs that are older than 4 months private, which makes them no longer accessible. If your managed node group is using a private Windows AMI, consider <>. While we can't guarantee that we can provide access to AMIs that have been made private, you can request access by filing a ticket with {aws} Support. For more information, see link:AWSEC2/latest/WindowsGuide/aws-windows-ami.html#ami-patches-security-ID[Patches, security updates, and AMI IDs,type="documentation"] in the _Amazon EC2 User Guide_. -[[not-ready,not-ready.title]] +[#not-ready] == Node is in `NotReady` state -If your node enters a `NotReady` status, this likely indicates that the node is unhealthy and unavailable to schedule new [.noloc]`Pods`. This can occur for various reasons, such as the node lacking sufficient resources for CPU, memory, or available disk space. +If your node enters a `NotReady` status, this likely indicates that the node is unhealthy and unavailable to schedule new Pods. This can occur for various reasons, such as the node lacking sufficient resources for CPU, memory, or available disk space. -For Amazon EKS optimized [.noloc]`Windows` AMIs, there's no reservation for compute resources specified by default in the `kubelet` configuration. To help prevent resource issues, you can reserve compute resources for system processes by providing the `kubelet` with configuration values for https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#kube-reserved[kube-reserved] and/or https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#system-reserved[system-reserved]. You do this using the `-KubeletExtraArgs` command-line parameter in the bootstrap script. For more information, see https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/[Reserve Compute Resources for System Daemons] in the [.noloc]`Kubernetes` documentation and <> in this user guide. +For Amazon EKS optimized Windows AMIs, there's no reservation for compute resources specified by default in the `kubelet` configuration. To help prevent resource issues, you can reserve compute resources for system processes by providing the `kubelet` with configuration values for https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#kube-reserved[kube-reserved] and/or https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#system-reserved[system-reserved]. You do this using the `-KubeletExtraArgs` command-line parameter in the bootstrap script. For more information, see https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/[Reserve Compute Resources for System Daemons] in the Kubernetes documentation and <> in this user guide. -[[troubleshoot-cni,troubleshoot-cni.title]] -== CNI log collection tool +[#log-collector] +== EKS Log Collector -The [.noloc]`Amazon VPC CNI plugin for Kubernetes` has its own troubleshooting script that is available on nodes at `/opt/cni/bin/aws-cni-support.sh`. You can use the script to collect diagnostic logs for support cases and general troubleshooting. +To troubleshoot issue with Amazon EKS nodes, there is a pre-built script available on nodes located at `/etc/eks/log-collector-script/eks-log-collector.sh`. You can use the script to collect diagnostic logs for support cases and general troubleshooting. Use the following command to run the script on your node: [source,bash,subs="verbatim,attributes"] ---- -sudo bash /opt/cni/bin/aws-cni-support.sh +sudo bash /etc/eks/log-collector-script/eks-log-collector.sh ---- [NOTE] ==== -If the script is not present at that location, then the CNI container failed to run. You can manually download and run the script with the following command: +If the script is not present at that location. You can manually download and run the script with the following command: [source,bash,subs="verbatim,attributes"] ---- -curl -O https://raw.githubusercontent.com/awslabs/amazon-eks-ami/master/log-collector-script/linux/eks-log-collector.sh +curl -O https://amazon-eks.s3.amazonaws.com/support/log-collector-script/linux/eks-log-collector.sh sudo bash eks-log-collector.sh ---- ==== -The script collects the following diagnostic information. The CNI version that you have deployed can be earlier than the script version. +The script collects the following diagnostic information. [source,bash,subs="verbatim,attributes"] ---- - This is version 0.6.1. New versions can be found at https://github.com/awslabs/amazon-eks-ami +$ sudo bash /etc/eks/log-collector-script/eks-log-collector.sh + + This is version 0.7.8. New versions can be found at https://github.com/awslabs/amazon-eks-ami/blob/main/log-collector-script/ -Trying to collect common operating system logs... -Trying to collect kernel logs... -Trying to collect mount points and volume information... -Trying to collect SELinux status... -Trying to collect iptables information... -Trying to collect installed packages... -Trying to collect active system services... -Trying to collect Docker daemon information... -Trying to collect kubelet information... -Trying to collect L-IPAMD information... -Trying to collect sysctls information... -Trying to collect networking information... -Trying to collect CNI configuration information... -Trying to collect running Docker containers and gather container data... -Trying to collect Docker daemon logs... -Trying to archive gathered information... +Trying to collect common operating system logs... +Trying to collect kernel logs... +Trying to collect mount points and volume information... +... +... - Done... your bundled logs are located in /var/log/eks_i-0717c9d54b6cfaa19_2020-03-24_0103-UTC_0.6.1.tar.gz + Done... your bundled logs are located in /var/log/eks_i-EXAMPLE_2025-03-25_0000-UTC_0.7.8.tar.gz ---- The diagnostic information is collected and stored at: [source,none,subs="verbatim,attributes"] ---- -/var/log/eks_i-0717c9d54b6cfaa19_2020-03-24_0103-UTC_0.6.1.tar.gz +/var/log/eks_i-EXAMPLE_2025-03-25_0000-UTC_0.7.8.tar.gz ---- +To retrieve log bundle for Bottlerocket nodes, please refer to https://github.com/bottlerocket-os/bottlerocket?tab=readme-ov-file#logs[Bottlerocket Log] for more details. + -[[troubleshoot-container-runtime-network,troubleshoot-container-runtime-network.title]] +[#troubleshoot-container-runtime-network] == Container runtime network not ready You may receive a `Container runtime network not ready` error and authorization errors similar to the following: @@ -413,7 +392,7 @@ eksctl create iamidentitymapping --cluster my-cluster \ --username system:node:{{EC2PrivateDNSName}} ---- + -The ARN of the role that you specify can't include a link:IAM/latest/UserGuide/reference_identifiers.html#identifiers-friendly-names[path,type="documentation"] other than `/`. For example, if the name of your role is `development/apps/my-role`, you'd need to change it to `my-role` when specifying the ARN of the role. Make sure that you specify the node IAM role ARN (not the instance profile ARN). +The ARN of the role that you specify can't include a link:IAM/latest/UserGuide/reference_identifiers.html#identifiers-friendly-names[path,type="documentation"] other than `/`. For example, if the name of your role is `development/apps/my-role`, you'd need to change it to `my-role` when specifying the ARN of the role. Make sure that you specify the node IAM role ARN (not the instance profile ARN). . Your self-managed nodes are in a cluster with a platform version at the minimum version listed in the prerequisites in the <> topic, but an entry isn't listed in the `aws-auth` `ConfigMap` (see previous item) for the node's IAM role or an access entry doesn't exist for the role. To resolve the issue, view your existing access entries by replacing [.replaceable]`my-cluster` in the following command with the name of your cluster and then running the modified command: `aws eks list-access-entries --cluster-name [.replaceable]``my-cluster```. The following command adds an access entry for the node's IAM role. Replace [.replaceable]`111122223333` with the {aws} account ID for the IAM role and [.replaceable]`myAmazonEKSNodeRole` with the name of your node's role. If you have a Windows node, replace [.replaceable]`EC2_LINUX` with `EC2_Windows`. Make sure that you specify the node IAM role ARN (not the instance profile ARN). + [source,bash,subs="verbatim,attributes"] @@ -422,7 +401,7 @@ aws eks create-access-entry --cluster-name my-cluster --principal-arn {arn-aws}i ---- -[[troubleshoot-tls-handshake-timeout,troubleshoot-tls-handshake-timeout.title]] +[#troubleshoot-tls-handshake-timeout] == TLS handshake timeout When a node is unable to establish a connection to the public API server endpoint, you may see an error similar to the following error. @@ -436,17 +415,17 @@ The `kubelet` process will continually respawn and test the API server endpoint. To resolve the issue, check the route table and security groups to ensure that traffic from the nodes can reach the public endpoint. -[[default-region-env-variable,default-region-env-variable.title]] +[#default-region-env-variable] == InvalidClientTokenId -If you're using IAM roles for service accounts for a [.noloc]`Pod` or [.noloc]`DaemonSet` deployed to a cluster in a China {aws} Region, and haven't set the `AWS_DEFAULT_REGION` environment variable in the spec, the [.noloc]`Pod` or [.noloc]`DaemonSet` may receive the following error: +If you're using IAM roles for service accounts for a Pod or DaemonSet deployed to a cluster in a China {aws} Region, and haven't set the `AWS_DEFAULT_REGION` environment variable in the spec, the Pod or DaemonSet may receive the following error: [source,bash,subs="verbatim,attributes"] ---- An error occurred (InvalidClientTokenId) when calling the GetCallerIdentity operation: The security token included in the request is invalid ---- -To resolve the issue, you need to add the `AWS_DEFAULT_REGION` environment variable to your [.noloc]`Pod` or [.noloc]`DaemonSet` spec, as shown in the following example [.noloc]`Pod` spec. +To resolve the issue, you need to add the `AWS_DEFAULT_REGION` environment variable to your Pod or DaemonSet spec, as shown in the following example Pod spec. [source,yaml,subs="verbatim,attributes"] ---- @@ -466,24 +445,24 @@ spec: ---- -[[troubleshoot-node-grups-must-match-kubernetes-version,troubleshoot-node-grups-must-match-kubernetes-version.title]] -== Node groups must match [.noloc]`Kubernetes` version before upgrading control plane +[#troubleshoot-node-grups-must-match-kubernetes-version] +== Node groups must match Kubernetes version before upgrading control plane -Before you upgrade a control plane to a new [.noloc]`Kubernetes` version, the minor version of the managed and Fargate nodes in your cluster must be the same as the version of your control plane's current version. The Amazon EKS `update-cluster-version` API rejects requests until you upgrade all Amazon EKS managed nodes to the current cluster version. Amazon EKS provides APIs to upgrade managed nodes. For information on upgrading a managed node group's [.noloc]`Kubernetes` version, see <>. To upgrade the version of a Fargate node, delete the [.noloc]`pod` that's represented by the node and redeploy the [.noloc]`pod` after you upgrade your control plane. For more information, see <>. +Before you upgrade a control plane to a new Kubernetes version, the minor version of the managed and Fargate nodes in your cluster must be the same as the version of your control plane's current version. The Amazon EKS `update-cluster-version` API rejects requests until you upgrade all Amazon EKS managed nodes to the current cluster version. Amazon EKS provides APIs to upgrade managed nodes. For information on upgrading a managed node group's Kubernetes version, see <>. To upgrade the version of a Fargate node, delete the pod that's represented by the node and redeploy the pod after you upgrade your control plane. For more information, see <>. -[[too-many-requests,too-many-requests.title]] +[#too-many-requests] == When launching many nodes, there are `Too Many Requests` errors -If you launch many nodes simultaneously, you may see an error message in the link:AWSEC2/latest/UserGuide/user-data.html#user-data-shell-scripts[Amazon EC2 user data,type="documentation"] execution logs that says `Too Many Requests`. This can occur because the control plane is being overloaded with `describeCluster` calls. The overloading results in throttling, nodes failing to run the bootstrap script, and nodes failing to join the cluster altogether. +If you launch many nodes simultaneously, you may see an error message in the link:AWSEC2/latest/UserGuide/user-data.html#user-data-shell-scripts[Amazon EC2 user data,type="documentation"] execution logs that says `Too Many Requests`. This can occur because the control plane is being overloaded with `describeCluster` calls. The overloading results in throttling, nodes failing to run the bootstrap script, and nodes failing to join the cluster altogether. Make sure that `--apiserver-endpoint`, `--b64-cluster-ca`, and `--dns-cluster-ip` arguments are being passed to the node's bootstrap script. When including these arguments, there's no need for the bootstrap script to make a `describeCluster` call, which helps prevent the control plane from being overloaded. For more information, see <>. -[[troubleshooting-boundservicetoken,troubleshooting-boundservicetoken.title]] -== HTTP 401 unauthorized error response on [.noloc]`Kubernetes` API server requests +[#troubleshooting-boundservicetoken] +== HTTP 401 unauthorized error response on Kubernetes API server requests -You see these errors if a [.noloc]`Pod's` service account token has expired on a cluster. +You see these errors if a Pod's service account token has expired on a cluster. -Your Amazon EKS cluster's [.noloc]`Kubernetes` API server rejects requests with tokens older than 90 days. In previous [.noloc]`Kubernetes` versions, tokens did not have an expiration. This means that clients that rely on these tokens must refresh them within an hour. To prevent the [.noloc]`Kubernetes` API server from rejecting your request due to an invalid token, the https://kubernetes.io/docs/reference/using-api/client-libraries/[Kubernetes client SDK] version used by your workload must be the same, or later than the following versions: +Your Amazon EKS cluster's Kubernetes API server rejects requests with tokens older than 90 days. In previous Kubernetes versions, tokens did not have an expiration. This means that clients that rely on these tokens must refresh them within an hour. To prevent the Kubernetes API server from rejecting your request due to an invalid token, the https://kubernetes.io/docs/reference/using-api/client-libraries/[Kubernetes client SDK] version used by your workload must be the same, or later than the following versions: @@ -493,14 +472,14 @@ Your Amazon EKS cluster's [.noloc]`Kubernetes` API server rejects requests with * JavaScript version `0.10.3` and later * Ruby `master` branch * Haskell version `0.3.0.0` -* [.noloc]`C#` version `7.0.5` and later +* C# version `7.0.5` and later -You can identify all existing [.noloc]`Pods` in your cluster that are using stale tokens. For more information, see <>. +You can identify all existing Pods in your cluster that are using stale tokens. For more information, see <>. -[[troubleshooting-platform-version,troubleshooting-platform-version.title]] +[#troubleshooting-platform-version] == Amazon EKS platform version is more than two versions behind the current platform version -This can happen when Amazon EKS isn't able to automatically update your cluster's <>. Though there are many causes for this, some of the common causes follow. If any of these problems apply to your cluster, it may still function, its platform version just won't be updated by Amazon EKS. +This can happen when Amazon EKS isn't able to automatically update your cluster's link:eks/latest/userguide/platform-versions.html[platform-version,type="documentation"]. Though there are many causes for this, some of the common causes follow. If any of these problems apply to your cluster, it may still function, its platform version just won't be updated by Amazon EKS. .Problem The <> was deleted – This role was specified when the cluster was created. You can see which role was specified with the following command. Replace [.replaceable]`my-cluster` with the name of your cluster. @@ -559,7 +538,7 @@ An example output is as follows. If the subnet IDs returned in the output don't match the subnet IDs that were specified when the cluster was created, then if you want Amazon EKS to update the cluster, you need to change the subnets used by the cluster. This is because if you specified more than two subnets when you created your cluster, Amazon EKS randomly selects subnets that you specified to create new elastic network interfaces in. These network interfaces enable the control plane to communicate with your nodes. Amazon EKS won't update the cluster if the subnet it selects doesn't exist. You have no control over which of the subnets that you specified at cluster creation that Amazon EKS chooses to create a new network interface in. -When you initiate a [.noloc]`Kubernetes` version update for your cluster, the update can fail for the same reason. +When you initiate a Kubernetes version update for your cluster, the update can fail for the same reason. .Problem A security group specified during cluster creation was deleted – If you specified security groups during cluster creation, you can see their IDs with the following command. Replace [.replaceable]`my-cluster` with the name of your cluster. @@ -600,7 +579,7 @@ An example output is as follows. If the security group IDs returned in the output don't match the security group IDs that were specified when the cluster was created, then if you want Amazon EKS to update the cluster, you need to change the security groups used by the cluster. Amazon EKS won't update a cluster if the security group IDs specified at cluster creation don't exist. -When you initiate a [.noloc]`Kubernetes` version update for your cluster, the update can fail for the same reason. +When you initiate a Kubernetes version update for your cluster, the update can fail for the same reason. @@ -608,19 +587,19 @@ When you initiate a [.noloc]`Kubernetes` version update for your cluster, the up * You enabled <> when you created your cluster and the {aws} KMS key that you specified has been deleted. If you want Amazon EKS to update the cluster, you need to create a new cluster -[[cluster-health-status,cluster-health-status.title]] +[#cluster-health-status] == Cluster health FAQs and error codes with resolution paths -Amazon EKS detects issues with your EKS clusters and the cluster infrastructure and stores it in the _cluster health_. You can detect, troubleshoot, and address cluster issues more rapidly with the aid of cluster health information. This enables you to create application environments that are more secure and up-to-date. Additionally, it may be impossible for you to upgrade to newer versions of [.noloc]`Kubernetes` or for Amazon EKS to install security updates on a degraded cluster as a result of issues with the necessary infrastructure or cluster configuration. Amazon EKS can take 3 hours to detect issues or detect that an issue is resolved. +Amazon EKS detects issues with your EKS clusters and the cluster infrastructure and stores it in the _health_ object of your EKS cluster resource. You can detect, troubleshoot, and address cluster issues more rapidly with the aid of cluster health information. This enables you to create application environments that are more secure and up-to-date. Additionally, it may be impossible for you to upgrade to newer versions of Kubernetes or for Amazon EKS to install security updates on a degraded cluster as a result of issues with the necessary infrastructure or cluster configuration. Amazon EKS can take 3 hours to detect issues or detect that an issue is resolved. The health of an Amazon EKS cluster is a shared responsibility between Amazon EKS and its users. You are responsible for the prerequisite infrastructure of IAM roles and Amazon VPC subnets, as well as other necessary infrastructure, that must be provided in advance. Amazon EKS detects changes in the configuration of this infrastructure and the cluster. -To access your health of your cluster in the Amazon EKS console, look for a section called *Health Issues* in the *Overview* tab of the Amazon EKS cluster detail page. This data will be also be available by calling the `DescribeCluster` action in the EKS API, for example from within the {aws} Command Line Interface. +To access the health of your cluster in the Amazon EKS console, look for a table called *Health Issues* in the *Cluster health issues* tab of the observability dashboard accessed from the Amazon EKS cluster detail page. This data will be also be available by calling the `DescribeCluster` action in the EKS API, for example from within the {aws} Command Line Interface. *Why should I use this feature?*:: -You will get increased visibility into the health of your Amazon EKS cluster, quickly diagnose and fix any issues, without needing to spend time debugging or opening {aws} support cases. For example: you accidentally deleted a subnet for the Amazon EKS cluster, Amazon EKS won't be able to create cross account network interfaces and [.noloc]`Kubernetes` {aws} CLI commands such as `kubectl` exec or `kubectl` logs. These will fail with the error: `Error from server: error dialing backend: remote error: tls: internal error.` Now you will see an Amazon EKS health issue that says: `subnet-da60e280 was deleted: could not create network interface`. +You will get increased visibility into the health of your Amazon EKS cluster, quickly diagnose and fix any issues, without needing to spend time debugging or opening {aws} support cases. For example: you accidentally deleted a subnet for the Amazon EKS cluster, Amazon EKS won't be able to create cross account network interfaces and Kubernetes {aws} CLI commands such as `kubectl` exec or `kubectl` logs. These will fail with the error: `Error from server: error dialing backend: remote error: tls: internal error.` Now you will see an Amazon EKS health issue that says: `subnet-da60e280 was deleted: could not create network interface`. *How does this feature relate or work with other {aws} services?*:: @@ -637,15 +616,15 @@ Yes, cluster issues are detected for EKS clusters in the {aws} Cloud including _ *Can I get notified when new issues are detected?*:: -Yes. {aws} sends an email and Personal Health Dashboard notification when new Cluster Health issues are detected. +Yes. {aws} sends an email and Personal Health Dashboard notification when new cluster health issues are detected. *Does the console give me warnings for health issues?*:: Yes, any cluster with health issues will include a banner at the top of the console. -The first two columns are what are needed for API response values. The third field of the link:eks/latest/APIReference/API_ClusterIssue.html[Health ClusterIssue,type="documentation"] object is [.noloc]`resourceIds`, the return of which is dependent on the issue type. +The first two columns are what are needed for API response values. The third field of the link:eks/latest/APIReference/API_ClusterIssue.html[Health ClusterIssue,type="documentation"] object is resourceIds, the return of which is dependent on the issue type. -[cols="1,1,1,1", options="header"] +[%header,cols="4"] |=== |Code |Message @@ -653,68 +632,68 @@ The first two columns are what are needed for API response values. The third fie |Cluster Recoverable? -|[.noloc]`SUBNET_NOT_FOUND` -|We couldn't find one or more subnets currently associated with your cluster. Call Amazon EKS [.noloc]`update-cluster-config` API to update subnets. +|SUBNET_NOT_FOUND +|We couldn't find one or more subnets currently associated with your cluster. Call Amazon EKS update-cluster-config API to update subnets. |Subnet Ids |Yes -|[.noloc]`SECURITY_GROUP_NOT_FOUND` +|SECURITY_GROUP_NOT_FOUND |We couldn't find one or more security groups currently associated with your cluster. Call Amazon EKS update-cluster-config API to update security groups |Security group Ids |Yes -|[.noloc]`IP_NOT_AVAILABLE` +|IP_NOT_AVAILABLE |One or more of the subnets associated with your cluster does not have enough available IP addresses for Amazon EKS to perform cluster management operations. Free up addresses in the subnet(s), or associate different subnets to your cluster using the Amazon EKS update-cluster-config API. |Subnet Ids |Yes -|[.noloc]`VPC_NOT_FOUND` +|VPC_NOT_FOUND |We couldn't find the VPC associated with your cluster. You must delete and recreate your cluster. |VPC id |No -|[.noloc]`ASSUME_ROLE_ACCESS_DENIED` +|ASSUME_ROLE_ACCESS_DENIED |Your cluster is not using the Amazon EKS service-linked-role. We couldn't assume the role associated with your cluster to perform required Amazon EKS management operations. Check the role exists and has the required trust policy. |The cluster IAM role |Yes -|[.noloc]`PERMISSION_ACCESS_DENIED` +|PERMISSION_ACCESS_DENIED |Your cluster is not using the Amazon EKS service-linked-role. The role associated with your cluster does not grant sufficient permissions for Amazon EKS to perform required management operations. Check the policies attached to the cluster role and if any separate deny policies are applied. |The cluster IAM role |Yes -|[.noloc]`ASSUME_ROLE_ACCESS_DENIED_USING_SLR` +|ASSUME_ROLE_ACCESS_DENIED_USING_SLR |We couldn't assume the Amazon EKS cluster management service-linked-role. Check the role exists and has the required trust policy. |The Amazon EKS service-linked-role |Yes -|[.noloc]`PERMISSION_ACCESS_DENIED_USING_SLR` +|PERMISSION_ACCESS_DENIED_USING_SLR |The Amazon EKS cluster management service-linked-role does not grant sufficient permissions for Amazon EKS to perform required management operations. Check the policies attached to the cluster role and if any separate deny policies are applied. |The Amazon EKS service-linked-role |Yes -|[.noloc]`OPT_IN_REQUIRED` +|OPT_IN_REQUIRED |Your account doesn't have an Amazon EC2 service subscription. Update your account subscriptions in your account settings page. |N/A |Yes -|[.noloc]`STS_REGIONAL_ENDPOINT_DISABLED` -|The [.noloc]`STS` regional endpoint is disabled. Enable the endpoint for Amazon EKS to perform required cluster management operations. +|STS_REGIONAL_ENDPOINT_DISABLED +|The STS regional endpoint is disabled. Enable the endpoint for Amazon EKS to perform required cluster management operations. |N/A |Yes -|[.noloc]`KMS_KEY_DISABLED` +|KMS_KEY_DISABLED |The {aws} KMS Key associated with your cluster is disabled. Re-enable the key to recover your cluster. -|The [.noloc]`KMS Key Arn` +|The KMS Key Arn |Yes -|[.noloc]`KMS_KEY_NOT_FOUND` +|KMS_KEY_NOT_FOUND |We couldn't find the {aws} KMS key associated with your cluster. You must delete and recreate the cluster. -|The [.noloc]`KMS Key ARN` +|The KMS Key ARN |No -|[.noloc]`KMS_GRANT_REVOKED` +|KMS_GRANT_REVOKED |Grants for the {aws} KMS Key associated with your cluster are revoked. You must delete and recreate the cluster. -|The [.noloc]`KMS Key Arn` +|The KMS Key Arn |No |=== diff --git a/latest/ug/versioning/disable-extended-support.adoc b/latest/ug/versioning/disable-extended-support.adoc new file mode 100644 index 000000000..4db5cc1c1 --- /dev/null +++ b/latest/ug/versioning/disable-extended-support.adoc @@ -0,0 +1,39 @@ +include::../attributes.txt[] + +[.topic] +[#disable-extended-support] += Prevent increased cluster costs by disabling EKS extended support +:info_titleabbrev: Disable extended support + +This topic describes how to set the _upgrade policy_ of an EKS cluster to disable extended support. The upgrade policy of an EKS cluster determines what happens when a cluster reaches the end of the standard _support period_. If a cluster upgrade policy has extended support disabled, it will be automatically upgraded to the next Kubernetes version. + +For more information about upgrade policies, see link:eks/latest/userguide/view-upgrade-policy.html[Cluster upgrade policy,type="documentation"]. + +[IMPORTANT] +==== + +You cannot disable extended support once your cluster has entered it. You can only disable extended support for clusters on standard support. + +{aws} recommends upgrading your cluster to a version in the standard support period. + +==== + +[#disable-support-policy-console] +== Disable EKS extended support ({aws} Console) +. Navigate to your EKS cluster in the {aws} Console. Select the *Overview* tab on the *Cluster Info* page. +. In the *Kubernetes version setting* section, select *Manage*. +. Select *Standard support* and then *Save changes*. + + +[#disable-support-policy-cli] +== Disable EKS extended support ({aws} CLI) +. Verify the {aws} CLI is installed and you are logged in. link:cli/latest/userguide/getting-started-install.html[Learn how to update and install the {aws} CLI.,type="documentation"] +. Determine the name of your EKS cluster. +. Run the following command: ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks update-cluster-config \ +--name \ +--upgrade-policy supportType=STANDARD +---- \ No newline at end of file diff --git a/latest/ug/versioning/enable-extended-support.adoc b/latest/ug/versioning/enable-extended-support.adoc new file mode 100644 index 000000000..12d8e85c0 --- /dev/null +++ b/latest/ug/versioning/enable-extended-support.adoc @@ -0,0 +1,43 @@ +include::../attributes.txt[] + +[.topic] +[#enable-extended-support] += Add flexibility to plan Kubernetes version upgrades by enabling EKS extended support +:info_titleabbrev: Enable extended support + +This topic describes how to set the _upgrade policy_ of an EKS cluster to enable extended support. The upgrade policy of an EKS cluster determines what happens when a cluster reaches the end of the standard _support period_. If a cluster upgrade policy has extended support enabled, it will enter the extended support period at the end of the standard support period. The cluster will not be automatically upgraded at the end of the standard support period. + +Clusters actually in the _extended support period_ incur higher costs. If a cluster merely has the upgrade policy set to enable extended support, and is otherwise in the _standard support period_, it incurs standard costs. + +If you create a cluster in the {aws} console, it will have the upgrade policy set to disable extended support. If you create a cluster in another way, it will have the upgrade policy set to enable extended support. For example, clusters created with the {aws} API have extended support enabled. + +For more information about upgrade policies, see link:eks/latest/userguide/view-upgrade-policy.html[Cluster upgrade policy,type="documentation"]. + +[IMPORTANT] +==== + +If you want your cluster to stay on its current Kubernetes version to take advantage of the extended support period, you must enable the extended support upgrade policy before the end of standard support period. + +If you do not enable extended support, your cluster will be automatically upgraded. + +==== + +[#enable-support-policy-console] +== Enable EKS extended support ({aws} Console) +. Navigate to your EKS cluster in the {aws} Console. Select the *Overview* tab on the *Cluster Info* page. +. In the *Kubernetes version settings* section, select *Manage*. +. Select *Extended support* and then *Save changes*. + + +[#enable-support-policy-cli] +== Enable EKS extended support ({aws} CLI) +. Verify the {aws} CLI is installed and you are logged in. link:cli/latest/userguide/getting-started-install.html[Learn how to update and install the {aws} CLI.,type="documentation"] +. Determine the name of your EKS cluster. +. Run the following command: ++ +[source,bash,subs="verbatim,attributes"] +---- +aws eks update-cluster-config \ +--name \ +--upgrade-policy supportType=EXTENDED +---- \ No newline at end of file diff --git a/latest/ug/versioning/kubernetes-versions-extended.adoc b/latest/ug/versioning/kubernetes-versions-extended.adoc new file mode 100644 index 000000000..c8620934a --- /dev/null +++ b/latest/ug/versioning/kubernetes-versions-extended.adoc @@ -0,0 +1,75 @@ +include::../attributes.txt[] + +[.topic] +[#kubernetes-versions-extended] += Review release notes for Kubernetes versions on extended support +:info_titleabbrev: Extended support versions + +[abstract] +-- +This topic gives important changes to be aware of for each Kubernetes version in extended support. +-- + +Amazon EKS supports Kubernetes versions longer than they are supported upstream, with standard support for Kubernetes minor versions for 14 months from the time they are released in Amazon EKS, and extended support for Kubernetes minor versions for an additional 12 months of support (26 total months per version). + +This topic gives important changes to be aware of for each [.noloc]`Kubernetes` version in extended support. When upgrading, carefully review the changes that have occurred between the old and new versions for your cluster. + +[#kubernetes-1-30] +== Kubernetes 1.30 + +Kubernetes `1.30` is now available in Amazon EKS. For more information about Kubernetes `1.30`, see the https://kubernetes.io/blog/2024/04/17/kubernetes-v1-30-release/[official release announcement]. + +[IMPORTANT] +==== + + +* Starting with Amazon EKS version `1.30` or newer, any newly created managed node groups will automatically default to using Amazon Linux 2023 (AL2023) as the node operating system. Previously, new node groups would default to Amazon Linux 2 (AL2). You can continue to use AL2 by choosing it as the AMI type when creating a new node group. +** For information about migrating from AL2 to AL2023, see <>. +** For more information about Amazon Linux, see link:linux/al2023/ug/compare-with-al2.html[Comparing AL2 and AL2023,type="documentation"] in the Amazon Linux User Guide. +** For more information about specifiying the operating system for a managed node group, see <>. + +==== + +* With Amazon EKS `1.30`, the `topology.k8s.aws/zone-id` label is added to worker nodes. You can use Availability Zone IDs (AZ IDs) to determine the location of resources in one account relative to the resources in another account. For more information, see link:ram/latest/userguide/working-with-az-ids.html[Availability Zone IDs for your {aws} resources,type="documentation"] in the _{aws} RAM User Guide_. +* Starting with `1.30`, Amazon EKS no longer includes the `default` annotation on the `gp2 StorageClass` resource applied to newly created clusters. This has no impact if you are referencing this storage class by name. You must take action if you were relying on having a default `StorageClass` in the cluster. You should reference the `StorageClass` by the name `gp2`. Alternatively, you can deploy the Amazon EBS recommended default storage class by setting the `defaultStorageClass.enabled` parameter to true when installing version `1.31.0` or later of the `aws-ebs-csi-driver add-on`. +* The minimum required IAM policy for the Amazon EKS cluster IAM role has changed. The action `ec2:DescribeAvailabilityZones` is required. For more information, see <>. + +For the complete Kubernetes `1.30` changelog, see https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.30.md. + + + +[#kubernetes-1-29] +== Kubernetes 1.29 + +Kubernetes `1.29` is now available in Amazon EKS. For more information about Kubernetes `1.29`, see the https://kubernetes.io/blog/2023/12/13/kubernetes-v1-29-release/[official release announcement]. + +[IMPORTANT] +==== + + +* The deprecated `flowcontrol.apiserver.k8s.io/v1beta2` API version of `FlowSchema` and `PriorityLevelConfiguration` are no longer served in Kubernetes version `1.29`. If you have manifests or client software that uses the deprecated beta API group, you should change these before you upgrade to version `1.29`. + +==== + +* The `.status.kubeProxyVersion` field for node objects is now deprecated, and the Kubernetes project is proposing to remove that field in a future release. The deprecated field is not accurate and has historically been managed by `kubelet` - which does not actually know the `kube-proxy` version, or even whether `kube-proxy` is running. If you've been using this field in client software, stop - the information isn't reliable and the field is now deprecated. +* In Kubernetes `1.29` to reduce potential attack surface, the `LegacyServiceAccountTokenCleanUp` feature labels legacy auto-generated secret-based tokens as invalid if they have not been used for a long time (1 year by default), and automatically removes them if use is not attempted for a long time after being marked as invalid (1 additional year by default). To identify such tokens, a you can run: ++ +[source,bash,subs="verbatim,attributes"] +---- +kubectl get cm kube-apiserver-legacy-service-account-token-tracking -n kube-system +---- + +For the complete Kubernetes `1.29` changelog, see https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.29.md#changelog-since-v1280. + +[#kubernetes-1-28] +== Kubernetes 1.28 + +Kubernetes `1.28` is now available in Amazon EKS. For more information about Kubernetes `1.28`, see the https://kubernetes.io/blog/2023/08/15/kubernetes-v1-28-release/[official release announcement]. + + + +* Kubernetes `v1.28` expanded the supported skew between core node and control plane components by one minor version, from `n-2` to `n-3`, so that node components (``kubelet`` and `kube-proxy`) for the oldest supported minor version can work with control plane components (``kube-apiserver``, `kube-scheduler`, `kube-controller-manager`, `cloud-controller-manager`) for the newest supported minor version. +* Metrics `force_delete_pods_total` and `force_delete_pod_errors_total` in the `Pod GC Controller` are enhanced to account for all forceful pods deletion. A reason is added to the metric to indicate whether the pod is forcefully deleted because it's terminated, orphaned, terminating with the out-of-service taint, or terminating and unscheduled. +* The `PersistentVolume (PV)` controller has been modified to automatically assign a default `StorageClass` to any unbound `PersistentVolumeClaim` with the `storageClassName` not set. Additionally, the `PersistentVolumeClaim` admission validation mechanism within the API server has been adjusted to allow changing values from an unset state to an actual `StorageClass` name. + +For the complete Kubernetes `1.28` changelog, see https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.28.md#changelog-since-v1270. diff --git a/latest/ug/versioning/kubernetes-versions-standard.adoc b/latest/ug/versioning/kubernetes-versions-standard.adoc new file mode 100644 index 000000000..06af6276b --- /dev/null +++ b/latest/ug/versioning/kubernetes-versions-standard.adoc @@ -0,0 +1,145 @@ +include::../attributes.txt[] + +[.topic] +[#kubernetes-versions-standard] += Review release notes for Kubernetes versions on standard support +:info_titleabbrev: Standard support versions + +[abstract] +-- +This topic gives important changes to be aware of for each Kubernetes version in standard support. +-- + +This topic gives important changes to be aware of for each Kubernetes version in standard support. When upgrading, carefully review the changes that have occurred between the old and new versions for your cluster. + +[#kubernetes-1-34] +== Kubernetes 1.34 + +Kubernetes `1.34` is now available in Amazon EKS. For more information about Kubernetes `1.34`, see the https://kubernetes.io/blog/2025/08/27/kubernetes-v1-34-release/[official release announcement]. + +[IMPORTANT] +==== +* Containerd updated to 2.1 in Version 1.34 for launch. +** If you experience any issues after upgrade, check the link:https://github.com/containerd/containerd/releases/tag/v2.1.0[containerd 2.1 release notes]. +* {aws} is not releasing an EKS-optimized Amazon Linux 2 AMI for Kubernetes 1.34. +** {aws} encourages you to migrate to Amazon Linux 2023. Learn how to <>. +** For more information, see <>. +* AppArmor is deprecated in Kubernetes 1.34. +** We recommend migrating to alternative container security solutions like link:https://kubernetes.io/docs/tutorials/security/seccomp/[seccomp] or link:https://kubernetes.io/docs/concepts/security/pod-security-standards/[Pod Security Standards]. +* VolumeAttributesClass (VAC) graduates to GA in Kubernetes 1.34, migrating from the beta API (`storage.k8s.io/v1beta1`) to the stable API (`storage.k8s.io/v1`). +** If you use the EBS CSI driver with {aws}-managed sidecar containers (from link:https://gallery.ecr.aws/csi-components[CSI Components] on the ECR Gallery), volume modification will continue to work seamlessly on EKS 1.31-1.33 clusters. {aws} will patch the sidecars to support beta VAC APIs until the end of EKS 1.33 standard support (July 29, 2026). +** If you self-manage your CSI sidecar containers, you may need to pin to older sidecar versions on pre-1.34 clusters to maintain VAC functionality. +** To use GA VolumeAttributesClass features (such as modification rollback), upgrade to EKS 1.34 or later. + +==== + +* *Dynamic Resource Allocation (DRA) Core APIs (GA):* Dynamic Resource Allocation has graduated to stable, enabling efficient management of specialized hardware like GPUs through standardized allocation interfaces - simplifying resource management for hardware accelerators and improving utilization of specialized resources. +* *Projected ServiceAccount Tokens for Kubelet (Beta):* This enhancement improves security by using short-lived credentials for container image pulls instead of long-lived secrets - reducing the risk of credential exposure and strengthening the overall security posture of your clusters. +* *Pod-level Resource Requests and Limits (Beta):* This feature simplifies resource management by allowing shared resource pools for multi-container pods - enabling more efficient resource allocation and utilization for complex applications with multiple containers. +* *Mutable CSI Node Allocatable Count (Beta):* The `MutableCSINodeAllocatableCount` feature gate is enabled by default in EKS 1.34, making the CSINode max attachable volume count attribute mutable and introducing a mechanism to update it dynamically based on user configuration at the CSI driver level. These updates can be triggered either by periodic intervals or by failure detection, enhancing the reliability of stateful pod scheduling by addressing mismatches between reported and actual attachment capacity on nodes. +** For more information, see link:https://kubernetes.io/blog/2025/09/11/kubernetes-v1-34-mutable-csi-node-allocatable-count/[Kubernetes v1.34: Mutable CSI Node Allocatable Count] on the _Kubernetes Blog_. +* *Deprecation Notice - cgroup driver configuration:* Manual cgroup driver configuration is being deprecated in favor of automatic detection. +** *Customer impact:* If you currently set the `--cgroup-driver` flag manually in your kubelet configuration, you should prepare to remove this configuration. +** *Required action:* Plan to update node bootstrap scripts and custom AMI configurations to remove manual cgroup driver settings before the feature is removed in a future Kubernetes release. +** For more information, see the link:https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/configure-cgroup-driver/[cgroup driver documentation]. + +For the complete Kubernetes `1.34` changelog, see https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.34.md + + +[#kubernetes-1-33] +== Kubernetes 1.33 + +Kubernetes `1.33` is now available in Amazon EKS. For more information about Kubernetes `1.33`, see the https://kubernetes.io/blog/2025/04/23/kubernetes-v1-33-release/[official release announcement]. + +[IMPORTANT] +==== +* The Dynamic Resource Allocation _beta_ Kubernetes API is enabled. +** This beta API improves the experience of scheduling and monitoring workloads that require resources such as GPUs. +** The beta API is defined by the Kubernetes community, and might change in future versions of Kubernetes. +** Carefully review https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/#feature-stages[Feature stages] in the Kubernetes documentation to understand the implications of using beta APIs. +* {aws} is not releasing an EKS-optimized Amazon Linux 2 AMI for Kubernetes 1.33. +** {aws} encourages you to migrate to Amazon Linux 2023. Learn how to <>. +** For more information, see <>. +==== + +* *In-Place Pod Resource Resize (Beta):* In-place resource resize has been promoted to beta, allowing dynamic updates to CPU and memory resources for existing Pods without restarts - enabling vertical scaling of stateful workloads with zero downtime and seamless resource adjustments based on traffic patterns. +* *Sidecar Containers Now Stable:* Sidecar containers have graduated to stable, implementing sidecars as special init containers with `restartPolicy: Always` that start before application containers, run throughout the pod lifecycle, and support probes for operational state signaling. +** For more information, see https://kubernetes.io/docs/concepts/workloads/pods/sidecar-containers/[Sidecar Containers] in the _Kubernetes Documentation_. +* *Endpoints API Deprecation:* The Endpoints API is now officially deprecated and will return warnings when accessed - migrate workloads and scripts to use the EndpointSlices API instead, which supports modern features like dual-stack networking and handles multiple EndpointSlices per Service. +** For more information, see https://kubernetes.io/blog/2025/04/24/endpoints-deprecation/[Kubernetes v1.33: Continuing the transition from Endpoints to EndpointSlice] on the _Kubernetes Blog_. +* *Elastic Fabric Adapter Support:* The default security group for Amazon EKS clusters now supports Elastic Fabric Adapter (EFA) traffic. The default security group has a new outbound rule that allows EFA traffic with the destination of the same security group. This allows EFA traffic within the cluster. +** For more information, see link:AWSEC2/latest/UserGuide/efa.html["Elastic Fabric Adapter for AI/ML and HPC workloads on Amazon EC2",type="documentation"] in the Amazon Elastic Compute Cloud User Guide. + +For the complete Kubernetes `1.33` changelog, see https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.33.md + +[#kubernetes-1-32] +== Kubernetes 1.32 + +Kubernetes `1.32` is now available in Amazon EKS. For more information about Kubernetes `1.32`, see the https://kubernetes.io/blog/2024/12/11/kubernetes-v1-32-release/[official release announcement]. + +[IMPORTANT] +==== +* The `flowcontrol.apiserver.k8s.io/v1beta3` API version of FlowSchema and PriorityLevelConfiguration has been removed in version `1.32`. If you are using these APIs, you must update your configurations to use the latest supported version before upgrading. + +* ServiceAccount `metadata.annotations[kubernetes.io/enforce-mountable-secrets]` has been deprecated in version `1.32` and will be removed in a future Kubernetes minor version release. It is recommended to use separate namespaces to isolate access to mounted secrets. + +* Kubernetes version `1.32` is the last version for which Amazon EKS will release Amazon Linux 2 (AL2) AMIs. From version `1.33` onwards, Amazon EKS will continue to release Amazon Linux 2023 (AL2023) and Bottlerocket based AMIs. + +==== + +* The Memory Manager feature has graduated to Generally Available (GA) status in Kubernetes version `1.32`. This enhancement provides more efficient and predictable memory allocation for containerized applications, particularly beneficial for workloads with specific memory requirements. + +* PersistentVolumeClaims (PVCs) created by StatefulSets now include automatic cleanup functionality. When PVCs are no longer needed, they will be automatically deleted while maintaining data persistence during StatefulSet updates and node maintenance operations. This feature simplifies storage management and helps prevent orphaned PVCs in your cluster. + +* Custom Resource Field Selector functionality has been introduced, allowing developers to add field selectors to custom resources. This feature provides the same filtering capabilities available for built-in Kubernetes objects to custom resources, enabling more precise and efficient resource filtering and promoting better API design practices. + +For the complete Kubernetes `1.32` changelog, see https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.32.md + +=== Anonymous authentication changes + +Starting with Amazon EKS `1.32`, anonymous authentication is restricted to the following API server health check endpoints: + +* `/healthz` +* `/livez` +* `/readyz` + +Requests to any other endpoint using the `system:unauthenticated` user will receive a `401 Unauthorized` HTTP response. This security enhancement helps prevent unintended cluster access that could occur due to misconfigured RBAC policies. + +[NOTE] +==== +The `public-info-viewer` RBAC role continues to apply for the health check endpoints listed above. +==== + +[#al2-ami-deprecation] +=== Amazon Linux 2 AMI deprecation + +For Kubernetes versions 1.33 and later, EKS will not provide pre-built optimized Amazon Linux 2 (AL2) Amazon Machine Images (AMIs). + +{aws} suggests adopting EKS Auto Mode, or migrating to a more recent operating system, such as Amazon Linux 2023 (AL2023) or Bottlerocket. + +- <> +- <> +- <> + +NOTE: This update applies to EKS-optimized AL2 AMIs. For more information about the operating system itself, see link:amazon-linux-2/faqs/[Amazon Linux 2 FAQs,type="marketing"]. + + +[#kubernetes-1-31] +== Kubernetes 1.31 + +Kubernetes `1.31` is now available in Amazon EKS. For more information about Kubernetes `1.31`, see the https://kubernetes.io/blog/2024/08/13/kubernetes-v1-31-release/[official release announcement]. + +[IMPORTANT] +==== + + +* The kubelet flag `--keep-terminated-pod-volumes` deprecated since 2017 has been removed as part of the version `1.31` release. This change impacts how terminated pod volumes are handled by the kubelet. If you are using this flag in your node configurations, you must update your bootstrap scripts and launch templates to remove it before upgrading. + +==== + +* The beta `VolumeAttributesClass` feature gate and API resource is enabled in Amazon EKS version `1.31`. This feature allows cluster operators to modify mutable properties of Persistent Volumes (PVs) managed by compatible CSI Drivers, including the Amazon EBS CSI Driver. To leverage this feature, ensure that your CSI Driver supports the `VolumeAttributesClass` feature (for the Amazon EBS CSI Driver, upgrade to version `1.35.0` or later to automatically enable the feature). You will be able to create `VolumeAttributesClass` objects to define the desired volume attributes, such as volume type and throughput, and associate them with your Persistent Volume Claims (PVCs). See the https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/[official Kubernetes documentation] as well as the documentation of your CSI driver for more information. +** For more information about the Amazon EBS CSI Driver, see <>. +* Kubernetes support for https://apparmor.net/[AppArmor] has graduated to stable and is now generally available for public use. This feature allows you to protect your containers with AppArmor by setting the `appArmorProfile.type` field in the container's `securityContext`. Prior to Kubernetes version `1.30`, AppArmor was controlled by annotations. Starting with version `1.30`, it is controlled using fields. To leverage this feature, we recommend migrating away from annotations and using the `appArmorProfile.type` field to ensure that your workloads are compatible. +* The PersistentVolume last phase transition time feature has graduated to stable and is now generally available for public use in Kubernetes version `1.31`. This feature introduces a new field, `.status.lastTransitionTime`, in the PersistentVolumeStatus, which provides a timestamp of when a PersistentVolume last transitioned to a different phase. This enhancement allows for better tracking and management of PersistentVolumes, particularly in scenarios where understanding the lifecycle of volumes is important. + +For the complete Kubernetes `1.31` changelog, see https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.31.md diff --git a/latest/ug/versioning/kubernetes-versions.adoc b/latest/ug/versioning/kubernetes-versions.adoc new file mode 100644 index 000000000..8ec407b4f --- /dev/null +++ b/latest/ug/versioning/kubernetes-versions.adoc @@ -0,0 +1,261 @@ +include::../attributes.txt[] + +[.topic] +[#kubernetes-versions] += Understand the Kubernetes version lifecycle on EKS +:info_titleabbrev: Kubernetes versions + + +[abstract] +-- +Learn how Amazon EKS supports Kubernetes versions with standard and extended support periods, allowing you to proactively update clusters with the latest versions, features, and security patches. +-- + +Kubernetes rapidly evolves with new features, design updates, and bug fixes. The community releases new Kubernetes minor versions (such as `{k8s-n}`) on average once every four months. Amazon EKS follows the upstream release and deprecation cycle for minor versions. As new Kubernetes versions become available in Amazon EKS, we recommend that you proactively update your clusters to use the latest available version. + +A minor version is under standard support in Amazon EKS for the first 14 months after it's released. Once a version is past the end of standard support date, it enters extended support for the next 12 months. Extended support allows you to stay at a specific Kubernetes version for longer at an additional cost per cluster hour. If you haven't updated your cluster before the extended support period ends, your cluster is auto-upgraded to the oldest currently supported extended version. + +Extended support is enabled by default. To disable, see link:eks/latest/userguide/disable-extended-support.html[Disable EKS extended support,type="documentation"]. + +We recommend that you create your cluster with the latest available Kubernetes version supported by Amazon EKS. If your application requires a specific version of Kubernetes, you can select older versions. You can create new Amazon EKS clusters on any version offered in standard or extended support. + + + +video::_dJdAZ_J_jw[youtube,align = center,height = 405,fileref = https://www.youtube.com/embed/_dJdAZ_J_jw,width = 720] + + +[#available-versions] +== Available versions on standard support + +The following Kubernetes versions are currently available in Amazon EKS standard support: + +* `1.34` +* `1.33` +* `1.32` +* `1.31` + + +For important changes to be aware of for each version in standard support, see link:eks/latest/userguide/kubernetes-versions-standard.html[Kubernetes versions standard support,type="documentation"]. + +[#available-versions-extended] +== Available versions on extended support + +The following Kubernetes versions are currently available in Amazon EKS extended support: + +* `1.30` +* `1.29` +* `1.28` + +For important changes to be aware of for each version in extended support, see link:eks/latest/userguide/kubernetes-versions-extended.html[Kubernetes versions extended support,type="documentation"]. + +[#kubernetes-release-calendar] +== Amazon EKS Kubernetes release calendar + +The following table shows important release and support dates to consider for each Kubernetes version. Billing for extended support starts at the beginning of the day that the version reaches end of standard support, in the UTC+0 timezone. The dates in the following table use the UTC+0 timezone. + +[NOTE] +==== +Dates with only a month and a year are approximate and are updated with an exact date when it's known. +==== + +To receive notifications of all source file changes to this specific documentation page, you can subscribe to the following URL with an RSS reader: + +[source,none] +---- +https://github.com/awsdocs/amazon-eks-user-guide/commits/mainline/latest/ug/clusters/kubernetes-versions.adoc.atom +---- + +[%header,cols="5"] +|=== +|Kubernetes version +|Upstream release +|Amazon EKS release +|End of standard support +|End of extended support + +|`1.34` +|August 27, 2025 +|October 2, 2025 +|December 2, 2026 +|December 2, 2027 + +|`1.33` +|April 23, 2025 +|May 29, 2025 +|July 29, 2026 +|July 29, 2027 + +|`1.32` +|December 11, 2024 +|January 23, 2025 +|March 23, 2026 +|March 23, 2027 + +|`1.31` +|August 13, 2024 +|September 26, 2024 +|November 26, 2025 +|November 26, 2026 + +|`1.30` +|April 17, 2024 +|May 23, 2024 +|July 23, 2025 +|July 23, 2026 + +|`1.29` +|December 13, 2023 +|January 23, 2024 +|March 23, 2025 +|March 23, 2026 + +|`1.28` +|August 15, 2023 +|September 26, 2023 +|November 26, 2024 +|November 26, 2025 + +|=== + +[#version-cli] +== Get version information with {aws} CLI + +You can use the {aws} CLI to get information about Kubernetes versions available on EKS, such as the end date of Standard Support. + +=== To retrieve information about available Kubernetes versions on EKS using the {aws} CLI + +. Open your terminal. +. Ensure you have the {aws} CLI installed and configured. For more information, see link:cli/latest/userguide/getting-started-install.html["Installing or updating to the latest version of the CLI",type="documentation"]. +. Run the following command: ++ +``` +aws eks describe-cluster-versions +``` +. The command will return a JSON output with details about the available cluster versions. Here's an example of the output: ++ +```json +{ + "clusterVersions": [ + { + "clusterVersion": "1.31", + "clusterType": "eks", + "defaultPlatformVersion": "eks.21", + "defaultVersion": true, + "releaseDate": "2024-09-25T17:00:00-07:00", + "endOfStandardSupportDate": "2025-11-25T16:00:00-08:00", + "endOfExtendedSupportDate": "2026-11-25T16:00:00-08:00", + "status": "STANDARD_SUPPORT", + "kubernetesPatchVersion": "1.31.3" + } + ] +} +``` + +*The output provides the following information for each cluster version:* + +* `clusterVersion`: The Kubernetes version of the EKS cluster +* `clusterType`: The type of cluster (e.g., "eks") +* `defaultPlatformVersion`: The default EKS platform version +* `defaultVersion`: Whether this is the default version +* `releaseDate`: The date when this version was released +* `endOfStandardSupportDate`: The date when standard support ends +* `endOfExtendedSupportDate`: The date when extended support ends +* `status`: The current support status of the version, such as `STANDARD_SUPPORT` or `EXTENDED_SUPPORT` +* `kubernetesPatchVersion`: The specific Kubernetes patch version + +[#version-faqs] +== Amazon EKS version FAQs + +*How many Kubernetes versions are available in standard support?*:: +In line with the Kubernetes community support for Kubernetes versions, Amazon EKS is committed to offering support for at least three Kubernetes versions at any given time. We will announce the end of standard support date of a given Kubernetes minor version at least 60 days in advance. Because of the Amazon EKS qualification and release process for new Kubernetes versions, the end of standard support date of a Kubernetes version on Amazon EKS will be after the date that the Kubernetes project stops supporting the version upstream. + +*How long does a Kubernetes receive standard support by Amazon EKS?*:: +A Kubernetes version received standard support for 14 months after first being available on Amazon EKS. This is true even if upstream Kubernetes no longer support a version that's available on Amazon EKS. We backport security patches that are applicable to the Kubernetes versions that are supported on Amazon EKS. + + +*Am I notified when standard support is ending for a Kubernetes version on Amazon EKS?*:: +Yes. If any clusters in your account are running the version nearing the end of support, Amazon EKS sends out a notice through the {aws} Health Dashboard approximately 12 months after the Kubernetes version was released on Amazon EKS. The notice includes the end of support date. This is at least 60 days from the date of the notice. + + +*Which Kubernetes features are supported by Amazon EKS?*:: +Amazon EKS supports all generally available (GA) features of the Kubernetes API. New beta APIs aren't enabled in clusters by default. However, previously existing beta APIs and new versions of existing beta APIs continue to be enabled by default. Alpha features aren't supported. + + +*Are Amazon EKS managed node groups automatically updated along with the cluster control plane version?*:: +No. A managed node group creates Amazon EC2 instances in your account. These instances aren't automatically upgraded when you or Amazon EKS update your control plane. For more information, see <>. We recommend maintaining the same Kubernetes version on your control plane and nodes. + + +*Are self-managed node groups automatically updated along with the cluster control plane version?*:: +No. A self-managed node group includes Amazon EC2 instances in your account. These instances aren't automatically upgraded when you or Amazon EKS update the control plane version on your behalf. A self-managed node group doesn't have any indication in the console that it needs updating. You can view the `kubelet` version installed on a node by selecting the node in the *Nodes* list on the *Overview* tab of your cluster to determine which nodes need updating. You must manually update the nodes. For more information, see <>. ++ +The Kubernetes project tests compatibility between the control plane and nodes for up to three minor versions. For example, `{k8s-n-3}` nodes continue to operate when orchestrated by a `{k8s-n}` control plane. However, running a cluster with nodes that are persistently three minor versions behind the control plane isn't recommended. For more information, see https://kubernetes.io/docs/setup/version-skew-policy/[Kubernetes version and version skew support policy] in the Kubernetes documentation. We recommend maintaining the same Kubernetes version on your control plane and nodes. + + +*Are Pods running on Fargate automatically upgraded with an automatic cluster control plane version upgrade?*:: +No. We strongly recommend running Fargate Pods as part of a replication controller, such as a Kubernetes deployment. Then do a rolling restart of all Fargate Pods. The new version of the Fargate Pod is deployed with a `kubelet` version that's the same version as your updated cluster control plane version. For more information, see https://kubernetes.io/docs/concepts/workloads/controllers/deployment[Deployments] in the Kubernetes documentation. ++ +IMPORTANT: If you update the control plane, you must still update the Fargate nodes yourself. To update Fargate nodes, delete the Fargate Pod represented by the node and redeploy the Pod. The new Pod is deployed with a `kubelet` version that's the same version as your cluster. + + +*What Kubernetes versions are supported for hybrid nodes?*:: +Amazon EKS Hybrid Nodes supports the same Kubernetes versions as Amazon EKS clusters with other node compute types, including standard and extended Kubernetes version support. Hybrid nodes are not automatically upgraded when you upgrade your control plane version and you are responsible for upgrading your hybrid nodes. For more information, see <>. + + +[#extended-support-faqs] +== Amazon EKS extended support FAQs + +*The standard support and extended support terminology is new to me. What do those terms mean?*:: +Standard support for a Kubernetes version in Amazon EKS begins when a Kubernetes version is released on Amazon EKS, and will end 14 months after the release date. Extended support for a Kubernetes version will begin immediately after the end of standard support, and will end after the next 12 months. For example, standard support for version `1.23` in Amazon EKS ended on October 11, 2023. Extended support for version `1.23` began on October 12, 2023 and ended on October 11, 2024. + + +*What do I need to do to get extended support for Amazon EKS clusters?*:: +You will need to enable extended support (see link:eks/latest/userguide/enable-extended-support.html[EKS extended support,type="documentation"]) for your cluster by changing the cluster upgrade policy to EXTENDED. By default, for all new and existing clusters, the upgrade policy is set to EXTENDED, unless specified otherwise. See link:eks/latest/userguide/view-upgrade-policy.html[Cluster upgrade policy,type="documentation"] to view the upgrade policy for your cluster. Standard support will begin when a Kubernetes version is released on Amazon EKS, and will end 14 months after the release date. Extended support for a Kubernetes version will begin immediately after the end of standard support, and will end after the next 12 months. + + +*For which Kubernetes versions can I get extended support?*:: +You can run clusters on any version for up to 12 months after the end of standard support for that version. This means that each version will be supported for 26 months in Amazon EKS (14 months of standard support plus 12 months of extended support). + + +*What if I don't want to use extended support?*:: +If you don't want to be automatically enrolled in extended support, you can upgrade your cluster to a Kubernetes version that's in standard Amazon EKS support. To disable extended support, see link:eks/latest/userguide/disable-extended-support.html[Disable EKS extended support,type="documentation"]. Note: If you disable extended support, your cluster will be auto upgraded at the end of standard support. + + +*What will happen at the end of 12 months of extended support?*:: +Clusters running on a Kubernetes version that has completed its 26-month lifecycle (14 months of standard support plus 12 months of extended support) will be auto-upgraded to the next version. The auto-upgrade includes only the Kubernetes control plane. If you have EKS Auto Mode nodes, they may automatically update. Self managed nodes and EKS Managed Node Groups will remain on the previous version. ++ +On the end of extended support date, you can no longer create new Amazon EKS clusters with the unsupported version. Existing control planes are automatically updated by Amazon EKS to the earliest supported version through a gradual deployment process after the end of support date. After the automatic control plane update, make sure to manually update cluster add-ons and Amazon EC2 nodes. For more information, see <>. + + +*When exactly is my control plane automatically updated after the end of extended support date?*:: +Amazon EKS can't provide specific time frames. Automatic updates can happen at any time after the end of extended support date. You won't receive any notification before the update. We recommend that you proactively update your control plane without relying on the Amazon EKS automatic update process. For more information, see <>. + + +*Can I leave my control plane on a Kubernetes version indefinitely?*:: +No. Cloud security at {aws} is the highest priority. Past a certain point (usually one year), the Kubernetes community stops releasing common vulnerabilities and exposures (CVE) patches and discourages CVE submission for unsupported versions. This means that vulnerabilities specific to an older version of Kubernetes might not even be reported. This leaves clusters exposed with no notice and no remediation options in the event of a vulnerability. Given this, Amazon EKS doesn't allow control planes to stay on a version that reached end of extended support. + + +*Is there additional cost to get extended support?*:: +Yes, there is additional cost for Amazon EKS clusters running in extended support. For pricing details, see link:containers/amazon-eks-extended-support-for-kubernetes-versions-pricing[Amazon EKS extended support for Kubernetes version pricing,type="blog"] on the {aws} blog or our link:eks/pricing/[pricing page,type="marketing"]. + + +*What is included in extended support?*:: +Amazon EKS clusters in Extended Support receive ongoing security patches for the Kubernetes control plane. Additionally, Amazon EKS will release patches for the Amazon VPC CNI, `kube-proxy`, and CoreDNS add-ons for Extended Support versions. Amazon EKS will also release patches for {aws}-published Amazon EKS optimized AMIs for Amazon Linux, Bottlerocket, and Windows, as well as Amazon EKS Fargate nodes for those versions. All clusters in Extended Support will continue to get access to technical support from {aws}. + +*Are there any limitations to patches for non-Kubernetes components in extended support?*:: +While Extended Support covers all of the Kubernetes specific components from {aws}, it will only provide support for {aws}-published Amazon EKS optimized AMIs for Amazon Linux, Bottlerocket, and Windows at all times. This means, you will potentially have newer components (such as OS or kernel) on your Amazon EKS optimized AMI while using Extended Support. For example, once Amazon Linux 2 reaches the link:amazon-linux-2/faqs/[end of its lifecycle in 2025,type="marketing"], the Amazon EKS optimized Amazon Linux AMIs will be built using a newer Amazon Linux OS. Amazon EKS will announce and document important support lifecycle discrepancies such as this for each Kubernetes version. + + +*Can I create new clusters using a version on extended support?*:: +Yes. + +include::kubernetes-versions-standard.adoc[leveloffset=+1] + +include::kubernetes-versions-extended.adoc[leveloffset=+1] + +include::view-support-status.adoc[leveloffset=+1] + +include::view-upgrade-policy.adoc[leveloffset=+1] + +include::enable-extended-support.adoc[leveloffset=+1] + +include::disable-extended-support.adoc[leveloffset=+1] diff --git a/latest/ug/clusters/platform-versions.adoc b/latest/ug/versioning/platform-versions.adoc similarity index 57% rename from latest/ug/clusters/platform-versions.adoc rename to latest/ug/versioning/platform-versions.adoc index a69c27f88..aaf417e9e 100644 --- a/latest/ug/clusters/platform-versions.adoc +++ b/latest/ug/versioning/platform-versions.adoc @@ -1,31 +1,25 @@ -//!!NODE_ROOT
+include::../attributes.txt[] [.topic] -[[platform-versions,platform-versions.title]] -= View Amazon EKS platform versions for each [.noloc]`Kubernetes` version -:info_doctype: section -:info_title: View Amazon EKS platform versions for each Kubernetes version +[#platform-versions] += View Amazon EKS platform versions for each Kubernetes version :info_titleabbrev: Platform versions -include::../attributes.txt[] - -Amazon EKS platform versions represent the capabilities of the Amazon EKS cluster control plane, such as which [.noloc]`Kubernetes` API server flags are enabled, as well as the current [.noloc]`Kubernetes` patch version. Each [.noloc]`Kubernetes` minor version has one or more associated Amazon EKS platform versions. The platform versions for different [.noloc]`Kubernetes` minor versions are independent. You can <> using the {aws} CLI or {aws-management-console}. If you have a local cluster on {aws} Outposts, see <> instead of this topic. +Amazon EKS platform versions represent the capabilities of the Amazon EKS cluster control plane, such as which Kubernetes API server flags are enabled, as well as the current Kubernetes patch version. Each Kubernetes minor version has one or more associated Amazon EKS platform versions. The platform versions for different Kubernetes minor versions are independent. You can <> using the {aws} CLI or {aws-management-console}. If you have a local cluster on {aws} Outposts, see <> instead of this topic. -When a new [.noloc]`Kubernetes` minor version is available in Amazon EKS, such as 1.30, the initial Amazon EKS platform version for that [.noloc]`Kubernetes` minor version starts at `eks.1`. However, Amazon EKS releases new platform versions periodically to enable new [.noloc]`Kubernetes` control plane settings and to provide security fixes. +When a new Kubernetes minor version is available in Amazon EKS, such as {k8s-n}, the initial Amazon EKS platform version for that Kubernetes minor version starts at `eks.1`. However, Amazon EKS releases new platform versions periodically to enable new Kubernetes control plane settings and to provide security fixes. When new Amazon EKS platform versions become available for a minor version: - - * The Amazon EKS platform version number is incremented (`eks.`). -* Amazon EKS automatically upgrades all existing clusters to the latest Amazon EKS platform version for their corresponding [.noloc]`Kubernetes` minor version. Automatic upgrades of existing Amazon EKS platform versions are rolled out incrementally. The roll-out process might take some time. If you need the latest Amazon EKS platform version features immediately, you should create a new Amazon EKS cluster. +* Amazon EKS automatically upgrades all existing clusters to the latest Amazon EKS platform version for their corresponding Kubernetes minor version. Automatic upgrades of existing Amazon EKS platform versions are rolled out incrementally. The roll-out process might take some time. If you need the latest Amazon EKS platform version features immediately, you should create a new Amazon EKS cluster. + If your cluster is more than two platform versions behind the current platform version, then it's possible that Amazon EKS wasn't able to automatically update your cluster. For details of what may cause this, see <>. -* Amazon EKS might publish a new node AMI with a corresponding patch version. However, all patch versions are compatible between the EKS control plane and node AMIs for a given [.noloc]`Kubernetes` minor version. +* Amazon EKS might publish a new node AMI with a corresponding patch version. However, all patch versions are compatible between the EKS control plane and node AMIs for a given Kubernetes minor version. New Amazon EKS platform versions don't introduce breaking changes or cause service interruptions. -Clusters are always created with the latest available Amazon EKS platform version (`eks.`) for the specified [.noloc]`Kubernetes` version. If you update your cluster to a new [.noloc]`Kubernetes` minor version, your cluster receives the current Amazon EKS platform version for the [.noloc]`Kubernetes` minor version that you updated to. +Clusters are always created with the latest available Amazon EKS platform version (`eks.`) for the specified Kubernetes version. If you update your cluster to a new Kubernetes minor version, your cluster receives the current Amazon EKS platform version for the Kubernetes minor version that you updated to. The current and recent Amazon EKS platform versions are described in the following tables. @@ -36,805 +30,743 @@ The current and recent Amazon EKS platform versions are described in the followi ==== +To receive notifications of all source file changes to this specific documentation page, you can subscribe to the following URL with an RSS reader: + +[source,none] +---- +https://github.com/awsdocs/amazon-eks-user-guide/commits/mainline/latest/ug/clusters/platform-versions.adoc.atom +---- + + -[[platform-versions-1.31,platform-versions-1.31.title]] -== [.noloc]`Kubernetes` version `1.31` +[#platform-versions-1-34] +== Kubernetes version `1.34` -The following admission controllers are enabled for all `1.31` platform versions: `NodeRestriction`, `ExtendedResourceToleration`, `NamespaceLifecycle`, `LimitRanger`, `ServiceAccount`, `TaintNodesByCondition`, `PodSecurity`, `Priority`, `DefaultTolerationSeconds`, `DefaultStorageClass`, `StorageObjectInUseProtection`, `PersistentVolumeClaimResize`, `RuntimeClass`, `CertificateApproval`, `CertificateSigning`, `CertificateSubjectRestriction`, `DefaultIngressClass`, `MutatingAdmissionWebhook`, `ValidatingAdmissionWebhook`, `ResourceQuota`. +The following admission controllers are enabled for all `1.34` platform versions: `NodeRestriction`, `ExtendedResourceToleration`, `NamespaceLifecycle`, `LimitRanger`, `ServiceAccount`, `TaintNodesByCondition`, `PodSecurity`, `Priority`, `DefaultTolerationSeconds`, `DefaultStorageClass`, `StorageObjectInUseProtection`, `PersistentVolumeClaimResize`, `RuntimeClass`, `CertificateApproval`, `CertificateSigning`, `CertificateSubjectRestriction`, `DefaultIngressClass`, `MutatingAdmissionWebhook`, `ValidatingAdmissionWebhook`, `ResourceQuota`. -[cols="1,1,1,1", options="header"] +[%header,cols="4"] |=== | Kubernetes version | EKS platform version | Release notes | Release date -| `1.31.2` -| `eks.12` -| New platform version with Amazon EKS Hybrid Nodes support and enhancements to control plane observability. See <> and see link:blogs/containers/amazon-eks-enhances-kubernetes-control-plane-observability/[Amazon EKS enhances performance observability,type="blog"], respectively. -| November 15, 2024 - -| `1.31.1` -| `eks.6` -| New platform version with security fixes and enhancements. -| October 21, 2024 - -| `1.31.0` +| `1.34.1` | `eks.4` -| Initial release of Kubernetes version `1.31` for EKS. For more information, see <>. -| September 26, 2024 +| Initial release of Kubernetes version `1.34` for EKS. For more information, see <>. +| October 2, 2025 |=== -[[platform-versions-1.30,platform-versions-1.30.title]] -== [.noloc]`Kubernetes` version `1.30` -The following admission controllers are enabled for all `1.30` platform versions: `NodeRestriction`, `ExtendedResourceToleration`, `NamespaceLifecycle`, `LimitRanger`, `ServiceAccount`, `TaintNodesByCondition`, `PodSecurity`, `Priority`, `DefaultTolerationSeconds`, `DefaultStorageClass`, `StorageObjectInUseProtection`, `PersistentVolumeClaimResize`, `RuntimeClass`, `CertificateApproval`, `CertificateSigning`, `CertificateSubjectRestriction`, `DefaultIngressClass`, `MutatingAdmissionWebhook`, `ValidatingAdmissionWebhook`, `ResourceQuota`. +[#platform-versions-1-33] +== Kubernetes version `1.33` -[cols="1,1,1,1", options="header"] +The following admission controllers are enabled for all `1.33` platform versions: `NodeRestriction`, `ExtendedResourceToleration`, `NamespaceLifecycle`, `LimitRanger`, `ServiceAccount`, `TaintNodesByCondition`, `PodSecurity`, `Priority`, `DefaultTolerationSeconds`, `DefaultStorageClass`, `StorageObjectInUseProtection`, `PersistentVolumeClaimResize`, `RuntimeClass`, `CertificateApproval`, `CertificateSigning`, `CertificateSubjectRestriction`, `DefaultIngressClass`, `MutatingAdmissionWebhook`, `ValidatingAdmissionWebhook`, `ResourceQuota`. + +[%header,cols="4"] |=== | Kubernetes version | EKS platform version | Release notes | Release date -| `1.30.6` -| `eks.20` -| New platform version with Amazon EKS Hybrid Nodes support and enhancements to control plane observability. See <> and see link:blogs/containers/amazon-eks-enhances-kubernetes-control-plane-observability/[Amazon EKS enhances performance observability,type="blog"], respectively. -| November 15, 2024 - -| `1.30.5` -| `eks.12` -| New platform version with security fixes and enhancements. -| October 21, 2024 - -| `1.30.4` -| `eks.8` -| New platform version with security fixes and enhancements. -| September 3, 2024 - - -| `1.30.3` -| `eks.7` -| New platform version with security fixes and enhancements. -| August 28, 2024 +| `1.33.2` +| `eks.9` +| New platform version with security fixes and enhancements. 1.33 `eks.7` and 1.33 `eks.8` were discarded internally and never released. +| July 30, 2025 -| `1.30.3` +| `1.33.1` | `eks.6` | New platform version with security fixes and enhancements. -| August 9, 2024 +| June 26, 2025 -| `1.30.2` +| `1.33.1` | `eks.5` | New platform version with security fixes and enhancements. -| July 2, 2024 +| June 11, 2025 + +| `1.33.1` +| `eks.4` +| Initial release of Kubernetes version `1.33` for EKS. For more information, see <>. +| May 28, 2025 -| `1.30.0` -| `eks.2` -| Initial release of Kubernetes version `1.30` for EKS. For more information, see <>. -| May 23, 2024 |=== -[[platform-versions-1.29,platform-versions-1.29.title]] -== [.noloc]`Kubernetes` version `1.29` -The following admission controllers are enabled for all `1.29` platform versions: `NodeRestriction`, `ExtendedResourceToleration`, `NamespaceLifecycle`, `LimitRanger`, `ServiceAccount`, `TaintNodesByCondition`, `PodSecurity`, `Priority`, `DefaultTolerationSeconds`, `DefaultStorageClass`, `StorageObjectInUseProtection`, `PersistentVolumeClaimResize`, `RuntimeClass`, `CertificateApproval`, `CertificateSigning`, `CertificateSubjectRestriction`, `DefaultIngressClass`, `MutatingAdmissionWebhook`, `ValidatingAdmissionWebhook`, `ResourceQuota`. +[#platform-versions-1-32] +== Kubernetes version `1.32` + +The following admission controllers are enabled for all `1.32` platform versions: `NodeRestriction`, `ExtendedResourceToleration`, `NamespaceLifecycle`, `LimitRanger`, `ServiceAccount`, `TaintNodesByCondition`, `PodSecurity`, `Priority`, `DefaultTolerationSeconds`, `DefaultStorageClass`, `StorageObjectInUseProtection`, `PersistentVolumeClaimResize`, `RuntimeClass`, `CertificateApproval`, `CertificateSigning`, `CertificateSubjectRestriction`, `DefaultIngressClass`, `MutatingAdmissionWebhook`, `ValidatingAdmissionWebhook`, `ResourceQuota`. -[cols="1,1,1,1", options="header"] +[%header,cols="4"] |=== | Kubernetes version | EKS platform version | Release notes | Release date -| `1.29.10` -| `eks.23` -| New platform version with Amazon EKS Hybrid Nodes support and enhancements to control plane observability. See <> and see link:blogs/containers/amazon-eks-enhances-kubernetes-control-plane-observability/[Amazon EKS enhances performance observability,type="blog"], respectively. -| November 15, 2024 - -| `1.29.9` -| `eks.17` -| New platform version with security fixes and enhancements. -| October 21, 2024 +| `1.32.6` +| `eks.16` +| New platform version with security fixes and enhancements. 1.32 `eks.14` and 1.32 `eks.15` were discarded internally and never released. +| July 30, 2025 -| `1.29.8` +| `1.32.5` | `eks.13` | New platform version with security fixes and enhancements. -| September 3, 2024 - +| June 26, 2025 -| `1.29.7` +| `1.32.5` | `eks.12` | New platform version with security fixes and enhancements. -| August 28, 2024 +| June 11, 2025 -| `1.29.7` +| `1.32.5` | `eks.11` | New platform version with security fixes and enhancements. -| August 9, 2024 +| May 30, 2025 -| `1.29.6` +| `1.32.3` | `eks.10` | New platform version with security fixes and enhancements. -| July 2, 2024 +| May 16, 2025 -| `1.29.4` +| `1.32.3` +| `eks.9` +| New platform version with security fixes and enhancements. +| April 29, 2025 + +| `1.32.3` +| `eks.8` +| New platform version with security fixes and enhancements. +| April 18, 2025 + +| `1.32.3` | `eks.7` -| New platform version with CoreDNS autoscaling, security fixes and enhancements. For more information about CoreDNS autoscaling, see <>. -| May 16, 2024 +| New platform version with security fixes and enhancements. +| April 18, 2025 -| `1.29.3` +| `1.32.3` | `eks.6` | New platform version with security fixes and enhancements. -| April 18, 2024 +| April 2, 2025 -| `1.29.1` +| `1.32.2` | `eks.5` | New platform version with security fixes and enhancements. -| March 29, 2024 +| March 17, 2025 -| `1.29.1` +| `1.32.2` | `eks.4` | New platform version with security fixes and enhancements. -| March 20, 2024 +| March 4, 2025 -| `1.29.1` +| `1.32.1` | `eks.3` | New platform version with security fixes and enhancements. -| March 12, 2024 +| February 24, 2025 + +| `1.32.0` +| `eks.2` +| Initial release of Kubernetes version `1.32` for EKS. For more information, see <>. +| January 2025 -| `1.29.0` -| `eks.1` -| Initial release of Kubernetes version `1.29` for EKS. For more information, see <>. -| January 23, 2024 |=== -[[platform-versions-1.28,platform-versions-1.28.title]] -== [.noloc]`Kubernetes` version `1.28` +[#platform-versions-1-31] +== Kubernetes version `1.31` -The following admission controllers are enabled for all `1.28` platform versions: `NodeRestriction`, `ExtendedResourceToleration`, `NamespaceLifecycle`, `LimitRanger`, `ServiceAccount`, `TaintNodesByCondition`, `PodSecurity`, `Priority`, `DefaultTolerationSeconds`, `DefaultStorageClass`, `StorageObjectInUseProtection`, `PersistentVolumeClaimResize`, `RuntimeClass`, `CertificateApproval`, `CertificateSigning`, `CertificateSubjectRestriction`, `DefaultIngressClass`, `MutatingAdmissionWebhook`, `ValidatingAdmissionWebhook`, `ResourceQuota`. +The following admission controllers are enabled for all `1.31` platform versions: `NodeRestriction`, `ExtendedResourceToleration`, `NamespaceLifecycle`, `LimitRanger`, `ServiceAccount`, `TaintNodesByCondition`, `PodSecurity`, `Priority`, `DefaultTolerationSeconds`, `DefaultStorageClass`, `StorageObjectInUseProtection`, `PersistentVolumeClaimResize`, `RuntimeClass`, `CertificateApproval`, `CertificateSigning`, `CertificateSubjectRestriction`, `DefaultIngressClass`, `MutatingAdmissionWebhook`, `ValidatingAdmissionWebhook`, `ResourceQuota`, `ObjectCount`. -[cols="1,1,1,1", options="header"] +[%header,cols="4"] |=== | Kubernetes version | EKS platform version | Release notes | Release date -| `1.28.15` -| `eks.29` -| New platform version with Amazon EKS Hybrid Nodes support and enhancements to control plane observability. See <> and see link:blogs/containers/amazon-eks-enhances-kubernetes-control-plane-observability/[Amazon EKS enhances performance observability,type="blog"], respectively. -| November 15, 2024 +| `1.31.10` +| `eks.32` +| New platform version with security fixes and enhancements. 1.31 `eks.30` and 1.31 `eks.31` were discarded internally and never released. +| July 30, 2025 -| `1.28.14` -| `eks.23` +| `1.31.9` +| `eks.29` | New platform version with security fixes and enhancements. -| October 21, 2024 +|June 26, 2025 -| `1.28.13` -| `eks.19` +| `1.31.9` +| `eks.28` | New platform version with security fixes and enhancements. -| September 3, 2024 +| June 11, 2025 -| `1.28.12` -| `eks.18` +| `1.31.9` +| `eks.27` | New platform version with security fixes and enhancements. -| August 28, 2024 +| May 30, 2025 -| `1.28.11` -| `eks.17` +| `1.31.7` +| `eks.26` | New platform version with security fixes and enhancements. -| August 9, 2024 +| May 16, 2025 -| `1.28.11` -| `eks.16` +| `1.31.7` +| `eks.25` | New platform version with security fixes and enhancements. -| July 2, 2024 +| April 29, 2025 -| `1.28.9` -| `eks.13` -| New platform version with CoreDNS autoscaling, security fixes and enhancements. For more information about CoreDNS autoscaling, see <>. -| May 16, 2024 +| `1.31.7` +| `eks.24` +| New platform version with security fixes and enhancements. +| April 18, 2025 -| `1.28.8` -| `eks.12` +| `1.31.7` +| `eks.23` | New platform version with security fixes and enhancements. -| April 18, 2024 +| April 18, 2025 -| `1.28.7` -| `eks.11` +| `1.31.7` +| `eks.22` | New platform version with security fixes and enhancements. -| March 29, 2024 +| April 2, 2025 -| `1.28.7` -| `eks.10` +| `1.31.6` +| `eks.21` | New platform version with security fixes and enhancements. -| March 20, 2024 +| March 17, 2025 -| `1.28.6` -| `eks.9` +| `1.31.6` +| `eks.20` | New platform version with security fixes and enhancements. -| March 12, 2024 +| March 4, 2025 -| `1.28.5` -| `eks.7` +| `1.31.5` +| `eks.19` | New platform version with security fixes and enhancements. -| January 17, 2024 +| February 24, 2025 -| `1.28.4` -| `eks.6` -| New platform version with <>, security fixes and enhancements. -| December 14, 2023 +| `1.31.5` +| `eks.18` +| New platform version with security fixes and enhancements. +| February 24, 2025 -| `1.28.4` -| `eks.5` +| `1.31.4` +| `eks.17` | New platform version with security fixes and enhancements. -| December 12, 2023 +| January 17, 2025 -| `1.28.3` -| `eks.4` -| New platform version with <>, security fixes and enhancements. -| November 10, 2023 +| `1.31.2` +| `eks.12` +| New platform version with Amazon EKS Hybrid Nodes support and enhancements to control plane observability. See <> and see link:containers/amazon-eks-enhances-kubernetes-control-plane-observability/[Amazon EKS enhances performance observability,type="blog"], respectively. +| November 15, 2024 -| `1.28.3` -| `eks.3` +| `1.31.1` +| `eks.6` | New platform version with security fixes and enhancements. -| November 3, 2023 +| October 21, 2024 -| `1.28.2` -| `eks.2` -| New platform version with security fixes and enhancements. -| October 16, 2023 +| `1.31.0` +| `eks.4` +| Initial release of Kubernetes version `1.31` for EKS. For more information, see <>. +| September 26, 2024 -| `1.28.1` -| `eks.1` -| Initial release of Kubernetes version `1.28` for EKS. For more information, see <>. -| September 26, 2023 |=== -[[platform-versions-1.27,platform-versions-1.27.title]] -== [.noloc]`Kubernetes` version `1.27` -The following admission controllers are enabled for all `1.27` platform versions: `NodeRestriction`, `ExtendedResourceToleration`, `NamespaceLifecycle`, `LimitRanger`, `ServiceAccount`, `TaintNodesByCondition`, `PodSecurity`, `Priority`, `DefaultTolerationSeconds`, `DefaultStorageClass`, `StorageObjectInUseProtection`, `PersistentVolumeClaimResize`, `RuntimeClass`, `CertificateApproval`, `CertificateSigning`, `CertificateSubjectRestriction`, `DefaultIngressClass`, `MutatingAdmissionWebhook`, `ValidatingAdmissionWebhook`, `ResourceQuota`. +[#platform-versions-1-30] +== Kubernetes version `1.30` -[cols="1,1,1,1", options="header"] +The following admission controllers are enabled for all `1.30` platform versions: `NodeRestriction`, `ExtendedResourceToleration`, `NamespaceLifecycle`, `LimitRanger`, `ServiceAccount`, `TaintNodesByCondition`, `PodSecurity`, `Priority`, `DefaultTolerationSeconds`, `DefaultStorageClass`, `StorageObjectInUseProtection`, `PersistentVolumeClaimResize`, `RuntimeClass`, `CertificateApproval`, `CertificateSigning`, `CertificateSubjectRestriction`, `DefaultIngressClass`, `MutatingAdmissionWebhook`, `ValidatingAdmissionWebhook`, `ResourceQuota`. + +[%header,cols="4"] |=== | Kubernetes version | EKS platform version | Release notes | Release date -| `1.27.16` -| `eks.33` -| New platform version with Amazon EKS Hybrid Nodes support, security fixes and enhancements. For more information about Amazon EKS Hybrid Nodes, see <>. -| November 15, 2024 +| `1.30.14` +| `eks.40` +| New platform version with security fixes and enhancements. 1.30 `eks.38` and 1.30 `eks.39` were discarded internally and never released. +| July 30, 2025 -| `1.27.16` -| `eks.27` +| `1.30.13` +| `eks.37` | New platform version with security fixes and enhancements. -| October 21, 2024 +| June 26, 2025 -| `1.27.16` -| `eks.23` +| `1.30.13` +| `eks.36` | New platform version with security fixes and enhancements. -| September 3, 2024 +| June 11, 2025 -| `1.27.16` -| `eks.22` +| `1.30.13` +| `eks.35` | New platform version with security fixes and enhancements. -| August 28, 2024 +| May 30, 2025 -| `1.27.16` -| `eks.21` +| `1.30.11` +| `eks.34` | New platform version with security fixes and enhancements. -| August 9, 2024 +| May 16, 2025 -| `1.27.15` -| `eks.20` +| `1.30.11` +| `eks.33` | New platform version with security fixes and enhancements. -| July 2, 2024 +| April 29, 2025 -| `1.27.13` -| `eks.17` -| New platform version with CoreDNS autoscaling, security fixes and enhancements. For more information about CoreDNS autoscaling, see <>. -| May 16, 2024 +| `1.30.11` +| `eks.32` +| New platform version with security fixes and enhancements. +| April 18, 2025 -| `1.27.12` -| `eks.16` +| `1.30.11` +| `eks.31` | New platform version with security fixes and enhancements. -| April 18, 2024 +| April 18, 2025 -| `1.27.11` -| `eks.15` +| `1.30.11` +| `eks.30` | New platform version with security fixes and enhancements. -| March 29, 2024 +| April 2, 2025 -| `1.27.11` -| `eks.14` +| `1.30.10` +| `eks.29` | New platform version with security fixes and enhancements. -| March 20, 2024 +| March 17, 2025 -| `1.27.10` -| `eks.13` +| `1.30.10` +| `eks.28` | New platform version with security fixes and enhancements. -| March 12, 2024 +| March 4, 2025 -| `1.27.9` -| `eks.11` +| `1.30.9` +| `eks.27` | New platform version with security fixes and enhancements. -| January 17, 2024 +| February 24, 2025 -| `1.27.8` -| `eks.10` -| New platform version with <>, security fixes and enhancements. -| December 14, 2023 +| `1.30.8` +| `eks.25` +| New platform version with security fixes and enhancements. +| January 17, 2025 -| `1.27.8` -| `eks.9` +| `1.30.8` +| `eks.24` | New platform version with security fixes and enhancements. -| December 12, 2023 +| January 3, 2025 -| `1.27.7` -| `eks.8` -| New platform version with <>, security fixes and enhancements. -| November 10, 2023 +| `1.30.7` +| `eks.23` +| New platform version with security fixes and enhancements. +| December 13, 2024 -| `1.27.7` -| `eks.7` +| `1.30.6` +| `eks.22` | New platform version with security fixes and enhancements. -| November 3, 2023 +| December 13, 2024 -| `1.27.6` -| `eks.6` +| `1.30.6` +| `eks.21` | New platform version with security fixes and enhancements. -| October 16, 2023 +| December 13, 2024 -| `1.27.4` -| `eks.5` +| `1.30.6` +| `eks.20` +| New platform version with Amazon EKS Hybrid Nodes support and enhancements to control plane observability. See <> and see link:containers/amazon-eks-enhances-kubernetes-control-plane-observability/[Amazon EKS enhances performance observability,type="blog"], respectively. +| November 15, 2024 + +| `1.30.5` +| `eks.12` | New platform version with security fixes and enhancements. -| August 30, 2023 +| October 21, 2024 -| `1.27.4` -| `eks.4` +| `1.30.4` +| `eks.8` | New platform version with security fixes and enhancements. -| July 30, 2023 +| September 3, 2024 -| `1.27.3` -| `eks.3` + +| `1.30.3` +| `eks.7` | New platform version with security fixes and enhancements. -| June 30, 2023 +| August 28, 2024 -| `1.27.2` -| `eks.2` +| `1.30.3` +| `eks.6` | New platform version with security fixes and enhancements. -| June 9, 2023 +| August 9, 2024 -| `1.27.1` -| `eks.1` -| Initial release of Kubernetes version `1.27` for EKS. For more information, see <>. -| May 24, 2023 +| `1.30.2` +| `eks.5` +| New platform version with security fixes and enhancements. +| July 2, 2024 + +| `1.30.0` +| `eks.2` +| Initial release of Kubernetes version `1.30` for EKS. For more information, see <>. +| May 23, 2024 |=== -[[platform-versions-1.26,platform-versions-1.26.title]] -== [.noloc]`Kubernetes` version `1.26` +[#platform-versions-1-29] +== Kubernetes version `1.29` -The following admission controllers are enabled for all `1.26` platform versions: `NodeRestriction`, `ExtendedResourceToleration`, `NamespaceLifecycle`, `LimitRanger`, `ServiceAccount`, `TaintNodesByCondition`, `PodSecurity`, `Priority`, `DefaultTolerationSeconds`, `DefaultStorageClass`, `StorageObjectInUseProtection`, `PersistentVolumeClaimResize`, `RuntimeClass`, `CertificateApproval`, `CertificateSigning`, `CertificateSubjectRestriction`, `DefaultIngressClass`, `MutatingAdmissionWebhook`, `ValidatingAdmissionWebhook`, `ResourceQuota`. +The following admission controllers are enabled for all `1.29` platform versions: `NodeRestriction`, `ExtendedResourceToleration`, `NamespaceLifecycle`, `LimitRanger`, `ServiceAccount`, `TaintNodesByCondition`, `PodSecurity`, `Priority`, `DefaultTolerationSeconds`, `DefaultStorageClass`, `StorageObjectInUseProtection`, `PersistentVolumeClaimResize`, `RuntimeClass`, `CertificateApproval`, `CertificateSigning`, `CertificateSubjectRestriction`, `DefaultIngressClass`, `MutatingAdmissionWebhook`, `ValidatingAdmissionWebhook`, `ResourceQuota`. -[cols="1,1,1,1", options="header"] +[%header,cols="4"] |=== | Kubernetes version | EKS platform version | Release notes | Release date -| `1.26.15` -| `eks.35` -| New platform version with Amazon EKS Hybrid Nodes support, security fixes and enhancements. For more information about Amazon EKS Hybrid Nodes, see <>. -| November 15, 2024 - -| `1.26.15` -| `eks.28` -| New platform version with security fixes and enhancements. -| October 21, 2024 +| `1.29.15` +| `eks.43` +| New platform version with security fixes and enhancements. 1.29 `eks.41` and 1.29 `eks.42` were discarded internally and never released. +| July 30, 2025 -| `1.26.15` -| `eks.24` +| `1.29.15` +| `eks.40` | New platform version with security fixes and enhancements. -| September 3, 2024 +| June 26, 2025 -| `1.26.15` -| `eks.23` +| `1.29.15` +| `eks.39` | New platform version with security fixes and enhancements. -| August 28, 2024 +| June 11, 2025 -| `1.26.15` -| `eks.22` +| `1.29.15` +| `eks.38` | New platform version with security fixes and enhancements. -| August 9, 2024 +| May 30, 2025 -| `1.26.15` -| `eks.21` +| `1.29.15` +| `eks.37` | New platform version with security fixes and enhancements. -| July 2, 2024 +| May 16, 2025 -| `1.26.15` -| `eks.18` -| New platform version with CoreDNS autoscaling, security fixes and enhancements. For more information about CoreDNS autoscaling, see <>. -| May 16, 2024 - -| `1.26.15` -| `eks.17` +| `1.29.15` +| `eks.36` | New platform version with security fixes and enhancements. -| April 18, 2024 +| April 29, 2025 -| `1.26.14` -| `eks.16` +| `1.29.15` +| `eks.35` | New platform version with security fixes and enhancements. -| March 29, 2024 +| April 18, 2025 -| `1.26.14` -| `eks.15` +| `1.29.15` +| `eks.34` | New platform version with security fixes and enhancements. -| March 20, 2024 +| April 18.2025 -| `1.26.13` -| `eks.14` +| `1.29.15` +| `eks.33` | New platform version with security fixes and enhancements. -| March 12, 2024 +| April 2, 2025 -| `1.26.12` -| `eks.12` +| `1.29.14` +| `eks.32` | New platform version with security fixes and enhancements. -| January 17, 2024 - -| `1.26.11` -| `eks.11` -| New platform version with <>, security fixes and enhancements. -| December 14, 2023 +| March 17, 2025 -| `1.26.11` -| `eks.10` +| `1.29.14` +| `eks.31` | New platform version with security fixes and enhancements. -| December 12, 2023 - -| `1.26.10` -| `eks.9` -| New platform version with <>, security fixes and enhancements. -| November 10, 2023 +| March 4, 2025 -| `1.26.10` -| `eks.8` +| `1.29.13` +| `eks.30` | New platform version with security fixes and enhancements. -| November 3, 2023 +| February 24, 2025 -| `1.26.9` -| `eks.7` +| `1.29.13` +| `eks.29` | New platform version with security fixes and enhancements. -| October 16, 2023 +| February 24, 2025 -| `1.26.7` -| `eks.6` +| `1.29.12` +| `eks.28` | New platform version with security fixes and enhancements. -| August 30, 2023 +| January 20, 2025 -| `1.26.7` -| `eks.5` +| `1.29.12` +| `eks.27` | New platform version with security fixes and enhancements. -| July 30, 2023 +| January 3, 2025 -| `1.26.6` -| `eks.4` +| `1.29.11` +| `eks.26` | New platform version with security fixes and enhancements. -| June 30, 2023 +| December 13, 2024 -| `1.26.5` -| `eks.3` +| `1.29.10` +| `eks.25` | New platform version with security fixes and enhancements. -| June 9, 2023 +| December 13, 2024 -| `1.26.4` -| `eks.2` +| `1.29.10` +| `eks.24` | New platform version with security fixes and enhancements. -| May 5, 2023 - -| `1.26.2` -| `eks.1` -| Initial release of Kubernetes version `1.26` for EKS. For more information, see <>. -| April 11, 2023 -|=== - -[[platform-versions-1.25,platform-versions-1.25.title]] -== [.noloc]`Kubernetes` version `1.25` - -The following admission controllers are enabled for all `1.25` platform versions: `NodeRestriction`, `ExtendedResourceToleration`, `NamespaceLifecycle`, `LimitRanger`, `ServiceAccount`, `TaintNodesByCondition`, `PodSecurity`, `Priority`, `DefaultTolerationSeconds`, `DefaultStorageClass`, `StorageObjectInUseProtection`, `PersistentVolumeClaimResize`, `RuntimeClass`, `CertificateApproval`, `CertificateSigning`, `CertificateSubjectRestriction`, `DefaultIngressClass`, `MutatingAdmissionWebhook`, `ValidatingAdmissionWebhook`, `ResourceQuota`. - -[cols="1,1,1,1", options="header"] -|=== -| Kubernetes version -| EKS platform version -| Release notes -| Release date +| December 13, 2024 -| `1.25.16` -| `eks.35` -| New platform version with Amazon EKS Hybrid Nodes support, security fixes and enhancements. For more information about Amazon EKS Hybrid Nodes, see <>. +| `1.29.10` +| `eks.23` +| New platform version with Amazon EKS Hybrid Nodes support and enhancements to control plane observability. See <> and see link:containers/amazon-eks-enhances-kubernetes-control-plane-observability/[Amazon EKS enhances performance observability,type="blog"], respectively. | November 15, 2024 -| `1.25.16` -| `eks.29` +| `1.29.9` +| `eks.17` | New platform version with security fixes and enhancements. | October 21, 2024 -| `1.25.16` -| `eks.25` +| `1.29.8` +| `eks.13` | New platform version with security fixes and enhancements. | September 3, 2024 -| `1.25.16` -| `eks.24` +| `1.29.7` +| `eks.12` | New platform version with security fixes and enhancements. | August 28, 2024 -| `1.25.16` -| `eks.23` +| `1.29.7` +| `eks.11` | New platform version with security fixes and enhancements. | August 9, 2024 -| `1.25.16` -| `eks.22` +| `1.29.6` +| `eks.10` | New platform version with security fixes and enhancements. | July 2, 2024 -| `1.25.16` -| `eks.19` +| `1.29.4` +| `eks.7` | New platform version with CoreDNS autoscaling, security fixes and enhancements. For more information about CoreDNS autoscaling, see <>. | May 16, 2024 -| `1.25.16` -| `eks.18` +| `1.29.3` +| `eks.6` | New platform version with security fixes and enhancements. | April 18, 2024 -| `1.25.16` -| `eks.17` +| `1.29.1` +| `eks.5` | New platform version with security fixes and enhancements. | March 29, 2024 -| `1.25.16` -| `eks.16` +| `1.29.1` +| `eks.4` | New platform version with security fixes and enhancements. | March 20, 2024 -| `1.25.16` -| `eks.15` +| `1.29.1` +| `eks.3` | New platform version with security fixes and enhancements. | March 12, 2024 -| `1.25.16` -| `eks.13` -| New platform version with security fixes and enhancements. -| January 17, 2024 +| `1.29.0` +| `eks.1` +| Initial release of Kubernetes version `1.29` for EKS. For more information, see <>. +| January 23, 2024 +|=== -| `1.25.16` -| `eks.12` -| New platform version with <>, security fixes and enhancements. -| December 14, 2023 +[#platform-versions-1-28] +== Kubernetes version `1.28` -| `1.25.16` -| `eks.11` -| New platform version with security fixes and enhancements. -| December 12, 2023 +The following admission controllers are enabled for all `1.28` platform versions: `NodeRestriction`, `ExtendedResourceToleration`, `NamespaceLifecycle`, `LimitRanger`, `ServiceAccount`, `TaintNodesByCondition`, `PodSecurity`, `Priority`, `DefaultTolerationSeconds`, `DefaultStorageClass`, `StorageObjectInUseProtection`, `PersistentVolumeClaimResize`, `RuntimeClass`, `CertificateApproval`, `CertificateSigning`, `CertificateSubjectRestriction`, `DefaultIngressClass`, `MutatingAdmissionWebhook`, `ValidatingAdmissionWebhook`, `ResourceQuota`. -| `1.25.15` -| `eks.10` -| New platform version with <>, security fixes and enhancements. -| November 10, 2023 +[%header,cols="4"] +|=== +| Kubernetes version +| EKS platform version +| Release notes +| Release date -| `1.25.15` -| `eks.9` -| New platform version with security fixes and enhancements. -| November 3, 2023 +| `1.28.15` +| `eks.49` +| New platform version with security fixes and enhancements. 1.28 `eks.47` and 1.28 `eks.48` were discarded internally and never released. +| July 30, 2025 -| `1.25.14` -| `eks.8` +| `1.28.15` +| `eks.46` | New platform version with security fixes and enhancements. -| October 16, 2023 +| June 26, 2025 -| `1.25.12` -| `eks.7` +| `1.28.15` +| `eks.45` | New platform version with security fixes and enhancements. -| August 30, 2023 +| June 11, 2025 -| `1.25.12` -| `eks.6` +| `1.28.15` +| `eks.44` | New platform version with security fixes and enhancements. -| July 30, 2023 +| May 30. 2025 -| `1.25.11` -| `eks.5` +| `1.28.15` +| `eks.43` | New platform version with security fixes and enhancements. -| June 30, 2023 +| May 16, 2025 -| `1.25.10` -| `eks.4` +| `1.28.15` +| `eks.42` | New platform version with security fixes and enhancements. -| June 9, 2023 +| April 29, 2025 -| `1.25.9` -| `eks.3` +| `1.28.15` +| `eks.41` | New platform version with security fixes and enhancements. -| May 5, 2023 +| April 18, 2025 -| `1.25.8` -| `eks.2` +| `1.28.15` +| `eks.40` | New platform version with security fixes and enhancements. -| March 24, 2023 - -| `1.25.6` -| `eks.1` -| Initial release of Kubernetes version `1.25` for EKS. For more information, see <>. -| February 21, 2023 -|=== - -[[platform-versions-1.24,platform-versions-1.24.title]] -== [.noloc]`Kubernetes` version `1.24` - -The following admission controllers are enabled for all `1.24` platform versions: `CertificateApproval`, `CertificateSigning`, `CertificateSubjectRestriction`, `DefaultIngressClass`, `DefaultStorageClass`, `DefaultTolerationSeconds`, `ExtendedResourceToleration`, `LimitRanger`, `MutatingAdmissionWebhook`, `NamespaceLifecycle`, `NodeRestriction`, `PersistentVolumeClaimResize`, `Priority`, `PodSecurityPolicy`, `ResourceQuota`, `RuntimeClass`, `ServiceAccount`, `StorageObjectInUseProtection`, `TaintNodesByCondition`, and `ValidatingAdmissionWebhook`. +| April 18, 2025 -[cols="1,1,1,1", options="header"] -|=== -| Kubernetes version -| EKS platform version -| Release notes -| Release date - -| `1.24.17` +| `1.28.15` | `eks.39` | New platform version with security fixes and enhancements. -| November 15, 2024 +| April 2, 2025 -| `1.24.17` -| `eks.32` +| `1.28.15` +| `eks.38` | New platform version with security fixes and enhancements. -| October 21, 2024 +| March 17, 2025 -| `1.24.17` -| `eks.28` +| `1.28.15` +| `eks.37` | New platform version with security fixes and enhancements. -| September 3, 2024 +| March 4, 2025 - -| `1.24.17` -| `eks.27` +| `1.28.15` +| `eks.36` | New platform version with security fixes and enhancements. -| August 28, 2024 +| February 24, 2025 +| `1.28.15` +| `eks.34` +| New platform version with security fixes and enhancements. +| January 17, 2025 -| `1.24.17` -| `eks.26` +| `1.28.15` +| `eks.33` | New platform version with security fixes and enhancements. -| August 9, 2024 +| January 3, 2025 -| `1.24.17` -| `eks.25` +| `1.28.15` +| `eks.32` | New platform version with security fixes and enhancements. -| July 2, 2024 +| December 13, 2024 -| `1.24.17` -| `eks.22` +| `1.28.15` +| `eks.31` | New platform version with security fixes and enhancements. -| May 16, 2024 +| December 13, 2024 -| `1.24.17` -| `eks.21` +| `1.28.15` +| `eks.30` | New platform version with security fixes and enhancements. -| April 18, 2024 +| December 13, 2024 -| `1.24.17` -| `eks.20` +| `1.28.15` +| `eks.29` +| New platform version with Amazon EKS Hybrid Nodes support and enhancements to control plane observability. See <> and see link:containers/amazon-eks-enhances-kubernetes-control-plane-observability/[Amazon EKS enhances performance observability,type="blog"], respectively. +| November 15, 2024 + +| `1.28.14` +| `eks.23` | New platform version with security fixes and enhancements. -| March 29, 2024 +| October 21, 2024 -| `1.24.17` +| `1.28.13` | `eks.19` | New platform version with security fixes and enhancements. -| March 20, 2024 +| September 3, 2024 -| `1.24.17` +| `1.28.12` | `eks.18` | New platform version with security fixes and enhancements. -| March 12, 2024 +| August 28, 2024 -| `1.24.17` -| `eks.16` +| `1.28.11` +| `eks.17` | New platform version with security fixes and enhancements. -| January 17, 2024 - -| `1.24.17` -| `eks.15` -| New platform version with <>, security fixes and enhancements. -| December 14, 2023 +| August 9, 2024 -| `1.24.17` -| `eks.14` +| `1.28.11` +| `eks.16` | New platform version with security fixes and enhancements. -| December 12, 2023 +| July 2, 2024 -| `1.24.17` +| `1.28.9` | `eks.13` -| New platform version with <>, security fixes and enhancements. -| November 10, 2023 +| New platform version with CoreDNS autoscaling, security fixes and enhancements. For more information about CoreDNS autoscaling, see <>. +| May 16, 2024 -| `1.24.17` +| `1.28.8` | `eks.12` | New platform version with security fixes and enhancements. -| November 3, 2023 +| April 18, 2024 -| `1.24.17` +| `1.28.7` | `eks.11` | New platform version with security fixes and enhancements. -| October 16, 2023 +| March 29, 2024 -| `1.24.16` +| `1.28.7` | `eks.10` | New platform version with security fixes and enhancements. -| August 30, 2023 +| March 20, 2024 -| `1.24.16` +| `1.28.6` | `eks.9` | New platform version with security fixes and enhancements. -| July 30, 2023 - -| `1.24.15` -| `eks.8` -| New platform version with security fixes and enhancements. -| June 30, 2023 +| March 12, 2024 -| `1.24.14` +| `1.28.5` | `eks.7` -| New platform version with security fixes and enhancements. -| June 9, 2023 +| New platform version with security fixes and enhancements. +| January 17, 2024 -| `1.24.13` +| `1.28.4` | `eks.6` -| New platform version with security fixes and enhancements. -| May 5, 2023 +| New platform version with <>, security fixes and enhancements. +| December 14, 2023 -| `1.24.12` +| `1.28.4` | `eks.5` -| New platform version with security fixes and enhancements. -| March 24, 2023 +| New platform version with security fixes and enhancements. +| December 12, 2023 -| `1.24.8` +| `1.28.3` | `eks.4` -| New platform version with security fixes and enhancements. -| January 27, 2023 +| New platform version with <>, security fixes and enhancements. +| November 10, 2023 -| `1.24.7` +| `1.28.3` | `eks.3` -| New platform version with security fixes and enhancements. -| December 5, 2022 +| New platform version with security fixes and enhancements. +| November 3, 2023 -| `1.24.7` +| `1.28.2` | `eks.2` -| New platform version with security fixes and enhancements. -| November 18, 2022 +| New platform version with security fixes and enhancements. +| October 16, 2023 -| `1.24.7` +| `1.28.1` | `eks.1` -| Initial release of Kubernetes version `1.24` for EKS. For more information, see <>. -| November 15, 2022 +| Initial release of Kubernetes version `1.28` for EKS. For more information, see <>. +| September 26, 2023 |=== -[[get-platform-version,get-platform-version.title]] +[#get-platform-version] == Get current platform version . Open the Amazon EKS console. . In the navigation pane, choose *Clusters*. . In the list of clusters, choose the *Cluster Name* to check the platform version of. . Choose the *Overview* tab. -. The *Platform Version* is available under in the *Details* section. +. The *Platform Version* is available under in the *Details* section. . Determine the *Name* of the cluster you want to check the platform version of. . Run the following command: + @@ -851,9 +783,9 @@ An example output is as follows. ---- -[[change-platform-version,change-platform-version.title]] +[#change-platform-version] == Change platform version -You cannot change the platform version of an EKS cluster. When new Amazon EKS platform versions become available for a [.noloc]`Kubernetes` version, EKS automatically upgrades all existing clusters to the latest Amazon EKS platform version for their corresponding [.noloc]`Kubernetes` version. Automatic upgrades of existing Amazon EKS platform versions are rolled out incrementally. You cannot use the {aws} Console or CLI to change the platform version. +You cannot change the platform version of an EKS cluster. When new Amazon EKS platform versions become available for a Kubernetes version, EKS automatically upgrades all existing clusters to the latest Amazon EKS platform version for their corresponding Kubernetes version. Automatic upgrades of existing Amazon EKS platform versions are rolled out incrementally. You cannot use the {aws} Console or CLI to change the platform version. -If you upgrade your [.noloc]`Kubernetes` version, your cluster will move onto the most recent platform version for the [.noloc]`Kubernetes` version. +If you upgrade your Kubernetes version, your cluster will move onto the most recent platform version for the Kubernetes version. diff --git a/latest/ug/versioning/versioning.adoc b/latest/ug/versioning/versioning.adoc new file mode 100644 index 000000000..ac259bb22 --- /dev/null +++ b/latest/ug/versioning/versioning.adoc @@ -0,0 +1,15 @@ +include::../attributes.txt[] + +[.topic] +[#versioning] += Versioning on Amazon EKS +:info_titleabbrev: Versioning + +This section is designed to help you learn how Kubernetes versioning operates within Amazon Elastic Kubernetes Service (EKS). You'll find details on supported versions, our deprecation and support policies, upgrade processes, and key considerations for managing your cluster versions. + +[.topiclist] +[[Topic List]] + +include::kubernetes-versions.adoc[leveloffset=+1] + +include::platform-versions.adoc[leveloffset=+1] diff --git a/latest/ug/versioning/view-support-status.adoc b/latest/ug/versioning/view-support-status.adoc new file mode 100644 index 000000000..da5bd397b --- /dev/null +++ b/latest/ug/versioning/view-support-status.adoc @@ -0,0 +1,21 @@ +include::../attributes.txt[] + +[.topic] +[#view-support-status] += View current cluster support period +:info_titleabbrev: View support period + +The *cluster support period* section of the {aws} console indicates if your cluster is _currently_ on standard or extended support. If your cluster support period is *Extended support*, you are being charged for EKS extended support. + +For more information about standard and extended support, see link:eks/latest/userguide/kubernetes-versions.html[Understand the Kubernetes version lifecycle on EKS,type="documentation"]. + +. Navigate to the *Clusters* page in the EKS section of the {aws} Console. Confirm the console is set to the same {aws} region as the cluster you want to review. +. Review the *Support Period* column. If the value is *Standard support until...*, you are not currently being charged for extended support. You are within the standard support period. If the value is *Extended support...* this cluster is currently being charged for extended support. + + +[NOTE] +==== + +The *Support Period* cannot be retrieved with the {aws} API or CLI. + +==== \ No newline at end of file diff --git a/latest/ug/clusters/view-upgrade-policy.adoc b/latest/ug/versioning/view-upgrade-policy.adoc similarity index 51% rename from latest/ug/clusters/view-upgrade-policy.adoc rename to latest/ug/versioning/view-upgrade-policy.adoc index 5c8d515c2..4a0a0fff2 100644 --- a/latest/ug/clusters/view-upgrade-policy.adoc +++ b/latest/ug/versioning/view-upgrade-policy.adoc @@ -1,28 +1,29 @@ include::../attributes.txt[] + [.topic] -[[view-upgrade-policy,view-upgrade-policy.title]] +[#view-upgrade-policy] = View current cluster upgrade policy :info_titleabbrev: View upgrade policy -The *cluster upgrade policy* determines what happens to your cluster when it leaves the standard support period. If your upgrade policy is `EXTENDED`, the cluster will not be automatically upgraded, and will enter extended support. If your upgrade policy is `STANDARD`, it will be automatically upgraded. +The *cluster upgrade policy* determines what happens to your cluster when it leaves the standard support period. If your upgrade policy is `EXTENDED`, the cluster will not be automatically upgraded, and will enter extended support. If your upgrade policy is `STANDARD`, it will be automatically upgraded. -Amazon EKS controls for [.noloc]`Kubernetes` version policy allows you to choose the end of standard support behavior for your EKS clusters. With these controls you can decide which clusters should enter extended support and which clusters should be automatically upgraded at the end of standard support for a [.noloc]`Kubernetes` version. +Amazon EKS controls for Kubernetes version policy allows you to choose the end of standard support behavior for your EKS clusters. With these controls you can decide which clusters should enter extended support and which clusters should be automatically upgraded at the end of standard support for a Kubernetes version. -A minor version is under standard support in Amazon EKS for the first 14 months after it's released. Once a version is past the end of standard support date, it enters extended support for the next 12 months. Extended support allows you to stay at a specific [.noloc]`Kubernetes` version for longer at an additional cost per cluster hour. You can enable or disable extended support for an EKS Cluster. If you disable extended support, {aws} will automatically upgrade your cluster to the next version at the end of standard support. If you enable extended support, you can stay at the current version for an additional cost for a limited period of time. Plan to regularly upgrade your [.noloc]`Kubernetes` cluster, even if you use extended support. +A minor version is under standard support in Amazon EKS for the first 14 months after it's released. Once a version is past the end of standard support date, it enters extended support for the next 12 months. Extended support allows you to stay at a specific Kubernetes version for longer at an additional cost per cluster hour. You can enable or disable extended support for an EKS Cluster. If you disable extended support, {aws} will automatically upgrade your cluster to the next version at the end of standard support. If you enable extended support, you can stay at the current version for an additional cost for a limited period of time. Plan to regularly upgrade your Kubernetes cluster, even if you use extended support. You can set the version policy for both new and existing clusters, using the `supportType` property. There are two options that can be used to set the version support policy: -* `*STANDARD*` -- Your EKS cluster eligible for automatic upgrade at the end of standard support. You will not incur extended support charges with this setting but you EKS cluster will automatically upgrade to the next supported [.noloc]`Kubernetes` version in standard support. -* `*EXTENDED*` -- Your EKS cluster will enter into extended support once the [.noloc]`Kubernetes` version reaches end of standard support. You will incur extended support charges with this setting. You can upgrade your cluster to a standard supported [.noloc]`Kubernetes` version to stop incurring extended support charges. Clusters running on extended support will be eligible for automatic upgrade at the end of extended support. +* `*STANDARD*` -- Your EKS cluster eligible for automatic upgrade at the end of standard support. You will not incur extended support charges with this setting but you EKS cluster will automatically upgrade to the next supported Kubernetes version in standard support. +* `*EXTENDED*` -- Your EKS cluster will enter into extended support once the Kubernetes version reaches end of standard support. You will incur extended support charges with this setting. You can upgrade your cluster to a standard supported Kubernetes version to stop incurring extended support charges. Clusters running on extended support will be eligible for automatic upgrade at the end of extended support. Extended support is enabled by default for new clusters, and existing clusters. You can view if extended support is enabled for a cluster in the {aws-management-console}, or by using the {aws} CLI. [IMPORTANT] ==== -If you want your cluster to stay on its current [.noloc]`Kubernetes` version to take advantage of the extended support period, you must enable the extended support upgrade policy before the end of standard support period. +If you want your cluster to stay on its current Kubernetes version to take advantage of the extended support period, you must enable the extended support upgrade policy before the end of standard support period. ==== @@ -30,15 +31,15 @@ You can only set the version support policy for your clusters while its running For example, if you have set your version support policy as `standard` then you will not be able to change this setting after the Kubernetes version running on your cluster reaches the end of standard support. If you have set your version support policy as `extended` then you will not be able to change this setting after the Kubernetes version running on your cluster reaches end of standard support. In order to change the version support policy setting, your cluster must be running on a standard supported Kubernetes version. -[[view-period-console,view-period-console.title]] +[#view-period-console] == View cluster upgrade policy ({aws} Console) . Navigate to the *Clusters* page in the EKS section of the {aws} Console. Confirm the console is set to the same {aws} region as the cluster you want to review. -. Review the *Upgrade Policy* column. If the value is *Standard Support*, your cluster will not enter extended support. If the value is *Extended Support*, your cluster will enter extended support. +. Review the *Upgrade Policy* column. If the value is *Standard Support*, your cluster will not enter extended support. If the value is *Extended Support*, your cluster will enter extended support. -[[view-period-cli,view-period-cli.title]] +[#view-period-cli] == View cluster upgrade policy ({aws} CLI) -. Verify the {aws} CLI is installed and you are logged in. link:cli/latest/userguide/getting-started-install.html[Learn how to update and install the {aws} CLI.,type="documentation"] +. Verify the {aws} CLI is installed and you are logged in. link:cli/latest/userguide/getting-started-install.html[Learn how to update and install the {aws} CLI.,type="documentation"] . Determine the name of your EKS cluster. Set the CLI to the same {aws} region as your EKS cluster. . Run the following command: + @@ -48,4 +49,4 @@ aws eks describe-cluster \ --name \ --query "cluster.upgradePolicy.supportType" ---- -. If the value is `STANDARD`, your cluster will not enter extended support. If the value is `EXTENDED`, your cluster will enter extended support. +. If the value is `STANDARD`, your cluster will not enter extended support. If the value is `EXTENDED`, your cluster will enter extended support. \ No newline at end of file diff --git a/latest/ug/what-is/common-use-cases.adoc b/latest/ug/what-is/common-use-cases.adoc index cb382dbf1..62a98729e 100644 --- a/latest/ug/what-is/common-use-cases.adoc +++ b/latest/ug/what-is/common-use-cases.adoc @@ -1,21 +1,16 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[common-use-cases,common-use-cases.title]] +[#common-use-cases] = Common use cases in Amazon EKS -:info_doctype: section -:info_title: Common use cases in Amazon EKS :info_titleabbrev: Common use cases -:keywords: Amazon Elastic Kubernetes Service, Amazon EKS, use cases, summary, description -:info_abstract: Discover how Amazon EKS helps deploy highly available containerized applications, build microservices architectures, automate software release processes, run serverless applications, execute machine learning workloads, deploy consistently on-premises and in the cloud, process big data cost-effectively, and ensure security and compliance. [abstract] -- Discover how Amazon EKS helps deploy highly available containerized applications, build microservices architectures, automate software release processes, run serverless applications, execute machine learning workloads, deploy consistently on-premises and in the cloud, process big data cost-effectively, and ensure security and compliance. -- -Amazon EKS offers robust managed [.noloc]`Kubernetes` services on {aws}, designed to optimize containerized applications. The following are a few of the most common use cases of Amazon EKS, helping you leverage its strengths for your specific needs. +Amazon EKS offers robust managed Kubernetes services on {aws}, designed to optimize containerized applications. The following are a few of the most common use cases of Amazon EKS, helping you leverage its strengths for your specific needs. @@ -24,7 +19,7 @@ Using link:elasticloadbalancing/[Elastic Load Balancing,type="marketing"], you c *Building microservices architectures*:: -Use [.noloc]`Kubernetes` service discovery features with link:cloud-map/[{aws} Cloud Map,type="marketing"] or link:vpc/lattice/[Amazon VPC Lattice,type="marketing"] to build resilient systems. +Use Kubernetes service discovery features with link:cloud-map/[{aws} Cloud Map,type="marketing"] or link:vpc/lattice/[Amazon VPC Lattice,type="marketing"] to build resilient systems. *Automating software release process*:: @@ -48,4 +43,4 @@ Utilize link:ec2/spot/[Spot Instances,type="marketing"] to run your batch proces *Securing application and ensuring compliance*:: -Implement strong security practices and maintain compliance with Amazon EKS, which integrates with {aws} security services such as link:iam/[{aws} Identity and Access Management,type="marketing"] (IAM), link:vpc/[Amazon Virtual Private Cloud,type="marketing"] (Amazon VPC), and link:kms/[{aws} Key Management Service,type="marketing"] ({aws} KMS). This ensures data privacy and protection as per industry standards. +Implement strong security practices and maintain compliance with Amazon EKS, which integrates with {aws} security services such as link:iam/[{aws} Identity and Access Management,type="marketing"] (IAM), link:vpc/[Amazon Virtual Private Cloud,type="marketing"] (Amazon VPC), and link:kms/[{aws} Key Management Service,type="marketing"] ({aws} KMS). This ensures data privacy and protection as per industry standards. \ No newline at end of file diff --git a/latest/ug/what-is/eks-architecture.adoc b/latest/ug/what-is/eks-architecture.adoc index 6fb916dcf..6679097ab 100644 --- a/latest/ug/what-is/eks-architecture.adoc +++ b/latest/ug/what-is/eks-architecture.adoc @@ -1,26 +1,21 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[eks-architecture,eks-architecture.title]] +[#eks-architecture] = Amazon EKS architecture -:info_doctype: section -:info_title: Amazon EKS architecture :info_titleabbrev: Architecture -:keywords: Amazon Elastic Kubernetes Service, Amazon EKS, architecture, control plane, nodes, data plane -:info_abstract: Learn how Amazon EKS aligns with Kubernetes cluster architecture, offering a highly available and resilient control plane, and flexible compute options like {aws} Fargate, Karpenter, managed node groups, and self-managed nodes to meet diverse workload requirements. [abstract] -- -Learn how Amazon EKS aligns with [.noloc]`Kubernetes` cluster architecture, offering a highly available and resilient control plane, and flexible compute options like {aws} Fargate, [.noloc]`Karpenter`, managed node groups, and self-managed nodes to meet diverse workload requirements. +Learn how Amazon EKS aligns with Kubernetes cluster architecture, offering a highly available and resilient control plane, and flexible compute options like {aws} Fargate, Karpenter, managed node groups, and self-managed nodes to meet diverse workload requirements. -- -Amazon EKS aligns with the general cluster architecture of [.noloc]`Kubernetes`. For more information, see https://kubernetes.io/docs/concepts/overview/components/[Kubernetes Components] in the [.noloc]`Kubernetes` documentation. The following sections summarize some extra architecture details for Amazon EKS. +Amazon EKS aligns with the general cluster architecture of Kubernetes. For more information, see https://kubernetes.io/docs/concepts/overview/components/[Kubernetes Components] in the Kubernetes documentation. The following sections summarize some extra architecture details for Amazon EKS. -[[control-plane,control-plane.title]] +[#control-plane] == Control plane -Amazon EKS ensures every cluster has its own unique [.noloc]`Kubernetes` control plane. This design keeps each cluster's infrastructure separate, with no overlaps between clusters or {aws} accounts. The setup includes: +Amazon EKS ensures every cluster has its own unique Kubernetes control plane. This design keeps each cluster's infrastructure separate, with no overlaps between clusters or {aws} accounts. The setup includes: @@ -39,22 +34,22 @@ If a control plane instance falters, Amazon EKS quickly replaces it, using diffe *Consistent uptime*:: By running clusters across multiple Availability Zones, a reliable link:eks/sla[API server endpoint availability Service Level Agreement (SLA),type="marketing"] is achieved. -Amazon EKS uses Amazon Virtual Private Cloud (Amazon VPC) to limit traffic between control plane components within a single cluster. Cluster components can't view or receive communication from other clusters or {aws} accounts, except when authorized by [.noloc]`Kubernetes` role-based access control (RBAC) policies. +Amazon EKS uses Amazon Virtual Private Cloud (Amazon VPC) to limit traffic between control plane components within a single cluster. Cluster components can't view or receive communication from other clusters or {aws} accounts, except when authorized by Kubernetes role-based access control (RBAC) policies. -[[nodes,nodes.title]] +[#nodes] == Compute In addition to the control plane, an Amazon EKS cluster has a set of worker machines called nodes. Selecting the appropriate Amazon EKS cluster node type is crucial for meeting your specific requirements and optimizing resource utilization. Amazon EKS offers the following primary node types: *EKS Auto Mode*:: -xref:automode[EKS Auto Mode] extends {aws} management beyond the control plane to include the data plane, automating cluster infrastructure management. It integrates core Kubernetes capabilities as built-in components, including compute autoscaling, networking, load balancing, DNS, storage, and GPU support. EKS Auto Mode dynamically manages nodes based on workload demands, using immutable AMIs with enhanced security features. It automates updates and upgrades while respecting Pod Disruption Budgets, and includes managed components that would otherwise require add-on management. This option is ideal for users who want to leverage {aws} expertise for day-to-day operations, minimize operational overhead, and focus on application development rather than infrastructure management. +<> extends {aws} management beyond the control plane to include the data plane, automating cluster infrastructure management. It integrates core Kubernetes capabilities as built-in components, including compute autoscaling, networking, load balancing, DNS, storage, and GPU support. EKS Auto Mode dynamically manages nodes based on workload demands, using immutable AMIs with enhanced security features. It automates updates and upgrades while respecting Pod Disruption Budgets, and includes managed components that would otherwise require add-on management. This option is ideal for users who want to leverage {aws} expertise for day-to-day operations, minimize operational overhead, and focus on application development rather than infrastructure management. *{aws} Fargate*:: <> is a serverless compute engine for containers that eliminates the need to manage the underlying instances. With Fargate, you specify your application's resource needs, and {aws} automatically provisions, scales, and maintains the infrastructure. This option is ideal for users who prioritize ease-of-use and want to concentrate on application development and deployment rather than managing infrastructure. -*[.noloc]`Karpenter`*:: -https://karpenter.sh/[Karpenter] is a flexible, high-performance [.noloc]`Kubernetes` cluster autoscaler that helps improve application availability and cluster efficiency. [.noloc]`Karpenter` launches right-sized compute resources in response to changing application load. This option can provision just-in-time compute resources that meet the requirements of your workload. +*Karpenter*:: +https://karpenter.sh/[Karpenter] is a flexible, high-performance Kubernetes cluster autoscaler that helps improve application availability and cluster efficiency. Karpenter launches right-sized compute resources in response to changing application load. This option can provision just-in-time compute resources that meet the requirements of your workload. *Managed node groups*:: @@ -65,4 +60,4 @@ https://karpenter.sh/[Karpenter] is a flexible, high-performance [.noloc]`Kubern <> offer full control over your Amazon EC2 instances within an Amazon EKS cluster. You are in charge of managing, scaling, and maintaining the nodes, giving you total control over the underlying infrastructure. This option is suitable for users who need granular control and customization of their nodes and are ready to invest time in managing and maintaining their infrastructure. *Amazon EKS Hybrid Nodes*:: -With <>, you can use your on-premises and edge infrastructure as nodes in Amazon EKS clusters. Amazon EKS Hybrid Nodes unifies Kubernetes management across environments and offloads Kubernetes control plane management to {aws} for your on-premises and edge applications. +With <>, you can use your on-premises and edge infrastructure as nodes in Amazon EKS clusters. Amazon EKS Hybrid Nodes unifies Kubernetes management across environments and offloads Kubernetes control plane management to {aws} for your on-premises and edge applications. \ No newline at end of file diff --git a/latest/ug/what-is/eks-deployment-options.adoc b/latest/ug/what-is/eks-deployment-options.adoc index ba61dcac4..f1791179c 100644 --- a/latest/ug/what-is/eks-deployment-options.adoc +++ b/latest/ug/what-is/eks-deployment-options.adoc @@ -1,40 +1,37 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[eks-deployment-options,eks-deployment-options.title]] +[#eks-deployment-options] = Deploy Amazon EKS clusters across cloud and on-premises environments -:info_doctype: section -:info_title: Deploy Amazon EKS clusters across cloud and on-premises environments :info_titleabbrev: Deployment options -:keywords: Amazon Elastic Kubernetes Service, Amazon EKS, deployment, options, comparison -:info_abstract: Learn to deploy Kubernetes clusters with Amazon EKS across cloud and on-premises environments to meet your operational needs, while leveraging {aws} services and support. - -include::../attributes.txt[] [abstract] -- -Learn to deploy [.noloc]`Kubernetes` clusters with Amazon EKS across cloud and on-premises environments to meet your operational needs, while leveraging {aws} services and support. +Learn to deploy Kubernetes clusters with Amazon EKS across cloud and on-premises environments to meet your operational needs, while leveraging {aws} services and support. -- -[[understand-deployment-options,understand-deployment-options.title]] +[#understand-deployment-options] == Understand Amazon EKS deployment options Amazon Elastic Kubernetes Service (Amazon EKS) is a fully managed Kubernetes service that enables you to run Kubernetes seamlessly in the cloud and in your on-premises environments. In the cloud, Amazon EKS automates Kubernetes cluster infrastructure management for the Kubernetes control plane and nodes. This is essential for scheduling containers, managing application availability, dynamically scaling resources, optimizing compute, storing cluster data, and performing other critical functions. With Amazon EKS, you get the robust performance, scalability, reliability, and availability of {aws} infrastructure, along with native integrations with {aws} networking, security, storage, and observability services. -To simplify running Kubernetes in your on-premises environments, you can use the same Amazon EKS clusters, features, and tools to <> or link:eks/latest/userguide/hybrid-nodes-overview.html[Amazon EKS Hybrid Nodes,type="documentation"] on your own infrastructure, or you can use https://anywhere.eks.amazonaws.com/[Amazon EKS Anywhere]for self-contained air-gapped environments. +To simplify running Kubernetes in your on-premises environments, you can use the same Amazon EKS clusters, features, and tools to <> or <> on your own infrastructure, or you can use https://anywhere.eks.amazonaws.com/[Amazon EKS Anywhere] for self-contained air-gapped environments. -[[eks-cloud-deployment-options,eks-cloud-deployment-options.title]] +[#eks-cloud-deployment-options] == Amazon EKS in the cloud -You can use Amazon EKS with compute in {aws} Regions, {aws} Local Zones, and {aws} Wavelength Zones. With Amazon EKS in the cloud, the security, scalability, and availability of the Kubernetes control plane is fully managed by {aws} in the {aws} Region. When running applications with compute in {aws} Regions, you get the full breadth of {aws} and Amazon EKS features, including Amazon EKS Auto Mode, which fully automates Kubernetes cluster infrastructure management for compute, storage, and networking on {aws} with a single click. When running applications with compute in {aws} Local Zones and {aws} Wavelength Zones, you can use Amazon EKS self-managed nodes to connect Amazon EC2 instances for your cluster compute and can use the other available {aws} services in {aws} Local Zones and {aws} Wavelength Zones. For more information see https://aws.amazon.com/about-aws/global-infrastructure/localzones/features/[{aws} Local Zones features] and https://aws.amazon.com/wavelength/features/[{aws} Wavelength Zones features]. +You can use Amazon EKS with compute in {aws} Regions, {aws} Local Zones, and {aws} Wavelength Zones. With Amazon EKS in the cloud, the security, scalability, and availability of the Kubernetes control plane is fully managed by {aws} in the {aws} Region. When running applications with compute in {aws} Regions, you get the full breadth of {aws} and Amazon EKS features, including Amazon EKS Auto Mode, which fully automates Kubernetes cluster infrastructure management for compute, storage, and networking on {aws} with a single click. When running applications with compute in {aws} Local Zones and {aws} Wavelength Zones, you can use Amazon EKS self-managed nodes to connect Amazon EC2 instances for your cluster compute and can use the other available {aws} services in {aws} Local Zones and {aws} Wavelength Zones. For more information see link:about-aws/global-infrastructure/localzones/features/[{aws} Local Zones features,type="marketing"] and link:wavelength/features/[{aws} Wavelength Zones features,type="marketing"]. + -[cols="1,1,1", options="header"] +[%header,cols="3"] |=== | |Amazon EKS in {aws} Regions |Amazon EKS in Local/Wavelength Zones + |Kuberenetes control plane management |{aws}-managed |{aws}-managed @@ -54,20 +51,21 @@ a|* Amazon EKS Managed Node Groups (Local Zones only) |Kubernetes data plane location |{aws} Regions |{aws} Local or Wavelength Zones + |=== -[[dc-or-edge-deployment-options,dc-or-edge-deployment-options.title]] +[#dc-or-edge-deployment-options] == Amazon EKS in your data center or edge environments -If you need to run applications in your own data centers or edge environments, you can use <> or link:eks/latest/userguide/hybrid-nodes.html[Amazon EKS Hybrid Nodes,type="documentation"]. You can use self-managed nodes with Amazon EC2 instances on {aws} Outposts for your cluster compute, or you can use Amazon EKS Hybrid Nodes with your own on-premises or edge infrastructure for your cluster compute. {aws} Outposts is {aws}-managed infrastructure that you run in your data centers or co-location facilities, whereas Amazon EKS Hybrid Nodes runs on your physical or virtual machines that you manage in your on-premises or edge environments. Amazon EKS on {aws} Outposts and Amazon EKS Hybrid Nodes require a reliable connection from your on-premises environments to an {aws} Region, and you can use the same Amazon EKS clusters, features, and tools you use to run applications in the cloud. When running on {aws} Outposts, you can alternatively deploy the entire Kubernetes cluster on {aws} Outposts with Amazon EKS local clusters on {aws} Outposts. - +If you need to run applications in your own data centers or edge environments, you can use <> or <>. You can use self-managed nodes with Amazon EC2 instances on {aws} Outposts for your cluster compute, or you can use Amazon EKS Hybrid Nodes with your own on-premises or edge infrastructure for your cluster compute. {aws} Outposts is {aws}-managed infrastructure that you run in your data centers or co-location facilities, whereas Amazon EKS Hybrid Nodes runs on your physical or virtual machines that you manage in your on-premises or edge environments. Amazon EKS on {aws} Outposts and Amazon EKS Hybrid Nodes require a reliable connection from your on-premises environments to an {aws} Region, and you can use the same Amazon EKS clusters, features, and tools you use to run applications in the cloud. When running on {aws} Outposts, you can alternatively deploy the entire Kubernetes cluster on {aws} Outposts with Amazon EKS local clusters on {aws} Outposts. -[cols="1,1,1", options="header"] +[%header,cols="3"] |=== | |Amazon EKS Hybrid Nodes |Amazon EKS on {aws} Outposts + |Kuberenetes control plane management |{aws}-managed |{aws}-managed @@ -83,18 +81,20 @@ If you need to run applications in your own data centers or edge environments, y |Kubernetes data plane location |Customer data center or edge environment |Customer data center or edge environment + |=== -[[air-gapped-deployment-options,air-gapped-deployment-options.title]] +[#air-gapped-deployment-options] == Amazon EKS Anywhere for air-gapped environments -https://aws.amazon.com/eks/eks-anywhere/[Amazon EKS Anywhere] simplifies Kubernetes cluster management through the automation of undifferentiated heavy lifting such as infrastructure setup and Kubernetes cluster lifecycle operations in on-premises and edge environments. Unlike Amazon EKS, Amazon EKS Anywhere is a customer-managed product and customers are responsible for cluster lifecycle operations and maintenance of Amazon EKS Anywhere clusters. Amazon EKS Anywhere is built on the Kubernetes sub-project Cluster API (CAPI) and supports a range of infrastructure including VMware vSphere, bare metal, Nutanix, Apache CloudStack, and {aws} Snow. Amazon EKS Anywhere can be run in air-gapped environments and offers optional integrations with regional {aws} services for observability and identity management. To receive support for Amazon EKS Anywhere and access to {aws}-vended Kubernetes add-ons, you can purchase https://aws.amazon.com/eks/eks-anywhere/pricing/[Amazon EKS Anywhere Enterprise Subscriptions]. +link:eks/eks-anywhere/[Amazon EKS Anywhere,type="marketing"] simplifies Kubernetes cluster management through the automation of undifferentiated heavy lifting such as infrastructure setup and Kubernetes cluster lifecycle operations in on-premises and edge environments. Unlike Amazon EKS, Amazon EKS Anywhere is a customer-managed product and customers are responsible for cluster lifecycle operations and maintenance of Amazon EKS Anywhere clusters. Amazon EKS Anywhere is built on the Kubernetes sub-project Cluster API (CAPI) and supports a range of infrastructure including VMware vSphere, bare metal, Nutanix, Apache CloudStack, and {aws} Snow. Amazon EKS Anywhere can be run in air-gapped environments and offers optional integrations with regional {aws} services for observability and identity management. To receive support for Amazon EKS Anywhere and access to {aws}-vended Kubernetes add-ons, you can purchase link:eks/eks-anywhere/pricing/[Amazon EKS Anywhere Enterprise Subscriptions,type="marketing"]. -[cols="1,1", options="header"] +[%header,cols="2"] |=== | |Amazon EKS Anywhere + |Kuberenetes control plane management |Customer-managed @@ -106,11 +106,12 @@ https://aws.amazon.com/eks/eks-anywhere/[Amazon EKS Anywhere] simplifies Kuberne |Kubernetes data plane location |Customer data center or edge environment + |=== -[[tooling-deployment-options,tooling-deployment-options.title]] +[#tooling-deployment-options] == Amazon EKS tooling You can use the <> to register and connect any conformant Kubernetes cluster to {aws} and view it in the Amazon EKS console. After a cluster is connected, you can see the status, configuration, and workloads for that cluster in the Amazon EKS console. You can use this feature to view connected clusters in Amazon EKS console, but the Amazon EKS Connector does not enable management or mutating operations for your connected clusters through the Amazon EKS console. -https://aws.amazon.com/eks/eks-distro/[Amazon EKS Distro] is the {aws} distribution of the underlying Kubernetes components that power all Amazon EKS offerings. It includes the core components required for a functioning Kubernetes cluster such as Kubernetes control plane components (etcd, kube-apiserver, kube-scheduler, kube-controller-manager) and networking components (CoreDNS, kube-proxy, CNI plugins). Amazon EKS Distro can be used to self-manage Kubernetes clusters with your choice of tooling. Amazon EKS Distro deployments are not covered by {aws} Support Plans. +link:eks/eks-distro/[Amazon EKS Distro,type="marketing"] is the {aws} distribution of the underlying Kubernetes components that power all Amazon EKS offerings. It includes the core components required for a functioning Kubernetes cluster such as Kubernetes control plane components (etcd, kube-apiserver, kube-scheduler, kube-controller-manager) and networking components (CoreDNS, kube-proxy, CNI plugins). Amazon EKS Distro can be used to self-manage Kubernetes clusters with your choice of tooling. Amazon EKS Distro deployments are not covered by {aws} Support Plans. \ No newline at end of file diff --git a/latest/ug/what-is/kubernetes-concepts.adoc b/latest/ug/what-is/kubernetes-concepts.adoc index c583f8d3e..3e9e060b6 100644 --- a/latest/ug/what-is/kubernetes-concepts.adoc +++ b/latest/ug/what-is/kubernetes-concepts.adoc @@ -1,33 +1,28 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[kubernetes-concepts,kubernetes-concepts.title]] -= [.noloc]`Kubernetes` concepts -:info_doctype: section -:info_title: Kubernetes concepts +[#kubernetes-concepts] += Kubernetes concepts :info_titleabbrev: Kubernetes concepts -:keywords: Amazon Elastic Kubernetes Service, Amazon EKS, architecture, control plane, nodes, data plane -:info_abstract: Learn core Kubernetes concepts and how they relate to deploying workloads, managing clusters, and working with control planes, nodes, Pods, containers, and networking on Amazon EKS. [abstract] -- -Learn core [.noloc]`Kubernetes` concepts and how they relate to deploying workloads, managing clusters, and working with control planes, nodes, Pods, containers, and networking on Amazon EKS. +Learn core Kubernetes concepts and how they relate to deploying workloads, managing clusters, and working with control planes, nodes, Pods, containers, and networking on Amazon EKS. -- -Amazon Elastic Kubernetes Service (Amazon EKS) is an {aws} managed service based on the open source https://kubernetes.io/[Kubernetes] project. While there are things you need to know about how the Amazon EKS service integrates with {aws} Cloud (particularly when you first create an Amazon EKS cluster), once it's up and running, you use your Amazon EKS cluster in much that same way as you would any other [.noloc]`Kubernetes` cluster. So to begin managing [.noloc]`Kubernetes` clusters and deploying workloads, you need at least a basic understanding of [.noloc]`Kubernetes` concepts. +Amazon Elastic Kubernetes Service (Amazon EKS) is an {aws} managed service based on the open source https://kubernetes.io/[Kubernetes] project. While there are things you need to know about how the Amazon EKS service integrates with {aws} Cloud (particularly when you first create an Amazon EKS cluster), once it's up and running, you use your Amazon EKS cluster in much that same way as you would any other Kubernetes cluster. So to begin managing Kubernetes clusters and deploying workloads, you need at least a basic understanding of Kubernetes concepts. -This page divides [.noloc]`Kubernetes` concepts into three sections: <>, <>, and <>. The first section describes the value of running a [.noloc]`Kubernetes` service, in particular as a managed service like Amazon EKS. The Workloads section covers how [.noloc]`Kubernetes` applications are built, stored, run, and managed. The Clusters section lays out the different components that make up [.noloc]`Kubernetes` clusters and what your responsibilities are for creating and maintaining [.noloc]`Kubernetes` clusters. +This page divides Kubernetes concepts into three sections: <>, <>, and <>. The first section describes the value of running a Kubernetes service, in particular as a managed service like Amazon EKS. The Workloads section covers how Kubernetes applications are built, stored, run, and managed. The Clusters section lays out the different components that make up Kubernetes clusters and what your responsibilities are for creating and maintaining Kubernetes clusters. [.topiclist] [[Topic List]] -As you go through this content, links will lead you to further descriptions of [.noloc]`Kubernetes` concepts in both Amazon EKS and [.noloc]`Kubernetes` documentation, in case you want to take deep dives into any of the topics we cover here. For details about how Amazon EKS implements [.noloc]`Kubernetes` control plane and compute features, see <>. +As you go through this content, links will lead you to further descriptions of Kubernetes concepts in both Amazon EKS and Kubernetes documentation, in case you want to take deep dives into any of the topics we cover here. For details about how Amazon EKS implements Kubernetes control plane and compute features, see <>. -[[why-kubernetes,why-kubernetes.title]] -== Why [.noloc]`Kubernetes`? +[#why-kubernetes] +== Why Kubernetes? -[.noloc]`Kubernetes` was designed to improve availability and scalability when running mission-critical, production-quality containerized applications. Rather than just running [.noloc]`Kubernetes` on a single machine (although that is possible), [.noloc]`Kubernetes` achieves those goals by allowing you to run applications across sets of computers that can expand or contract to meet demand. [.noloc]`Kubernetes` includes features that make it easier for you to: +Kubernetes was designed to improve availability and scalability when running mission-critical, production-quality containerized applications. Rather than just running Kubernetes on a single machine (although that is possible), Kubernetes achieves those goals by allowing you to run applications across sets of computers that can expand or contract to meet demand. Kubernetes includes features that make it easier for you to: @@ -38,134 +33,134 @@ As you go through this content, links will lead you to further descriptions of [ * Allocate resources between containers * Balance traffic across machines -Having [.noloc]`Kubernetes` automate these types of complex tasks allows an application developer to focus on building and improving their application workloads, rather than worrying about infrastructure. The developer typically creates configuration files, formatted as YAML files, that describe the desired state of the application. This could include which containers to run, resource limits, number of Pod replicas, CPU/memory allocation, affinity rules, and more. +Having Kubernetes automate these types of complex tasks allows an application developer to focus on building and improving their application workloads, rather than worrying about infrastructure. The developer typically creates configuration files, formatted as YAML files, that describe the desired state of the application. This could include which containers to run, resource limits, number of Pod replicas, CPU/memory allocation, affinity rules, and more. -[[attributes-of-kubernetes,attributes-of-kubernetes.title]] -=== Attributes of [.noloc]`Kubernetes` +[#attributes-of-kubernetes] +=== Attributes of Kubernetes -To achieve its goals, [.noloc]`Kubernetes` has the following attributes: +To achieve its goals, Kubernetes has the following attributes: -* *Containerized* -- [.noloc]`Kubernetes` is a container orchestration tool. To use [.noloc]`Kubernetes`, you must first have your applications containerized. Depending on the type of application, this could be as a set of _microservices,_ as batch jobs or in other forms. Then, your applications can take advantage of a [.noloc]`Kubernetes` workflow that encompasses a huge ecosystem of tools, where containers can be stored as https://kubernetes.io/docs/concepts/containers/images/#multi-architecture-images-with-image-indexes[images in a container registry], deployed to a [.noloc]`Kubernetes` https://kubernetes.io/docs/concepts/architecture/[cluster], and run on an available https://kubernetes.io/docs/concepts/architecture/nodes/[node]. You can build and test individual containers on your local computer with [.noloc]`Docker` or another https://kubernetes.io/docs/setup/production-environment/container-runtimes/[container runtime], before deploying them to your [.noloc]`Kubernetes` cluster. -* *Scalable* -- If the demand for your applications exceeds the capacity of the running instances of those applications, [.noloc]`Kubernetes` is able to scale up. As needed, [.noloc]`Kubernetes` can tell if applications require more CPU or memory and respond by either automatically expanding available capacity or using more of existing capacity. Scaling can be done at the Pod level, if there is enough compute available to just run more instances of the application (https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/[horizontal Pod autoscaling]), or at the node level, if more nodes need to be brought up to handle the increased capacity (https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler[Cluster Autoscaler] or https://karpenter.sh/[Karpenter]). As capacity is no longer needed, these services can delete unnecessary Pods and shut down unneeded nodes. -* *Available* -- If an application or node becomes unhealthy or unavailable, [.noloc]`Kubernetes` can move running workloads to another available node. You can force the issue by simply deleting a running instance of a workload or node that's running your workloads. The bottom line here is that workloads can be brought up in other locations if they can no longer run where they are. -* *Declarative* -- [.noloc]`Kubernetes` uses active reconciliation to constantly check that the state that you declare for your cluster matches the actual state. By applying https://kubernetes.io/docs/concepts/overview/working-with-objects/[Kubernetes objects] to a cluster, typically through YAML-formatted configuration files, you can, for example, ask to start up the workloads you want to run on your cluster. You can later change the configurations to do something like use a later version of a container or allocate more memory. [.noloc]`Kubernetes` will do what it needs to do to establish the desired state. This can include bringing nodes up or down, stopping and restarting workloads, or pulling updated containers. -* *Composable* -- Because an application typically consists of multiple components, you want to be able to manage a set of these components (often represented by multiple containers) together. While [.noloc]`Docker` Compose offers a way to do this directly with [.noloc]`Docker`, the [.noloc]`Kubernetes` http://kompose.io/[Kompose] command can help you do that with [.noloc]`Kubernetes`. See https://kubernetes.io/docs/tasks/configure-pod-container/translate-compose-kubernetes/[Translate a Docker Compose File to Kubernetes Resources] for an example of how to do this. -* *Extensible* -- Unlike proprietary software, the open source [.noloc]`Kubernetes` project is designed to be open to you extending [.noloc]`Kubernetes` any way that you like to meet your needs. APIs and configuration files are open to direct modifications. Third-parties are encouraged to write their own https://kubernetes.io/docs/concepts/architecture/controller/[Controllers], to extend both infrastructure and end-user [.noloc]`Kubernetes` featues. https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/[Webhooks] let you set up cluster rules to enforce policies and adapt to changing conditions. For more ideas on how to extend [.noloc]`Kubernetes` clusters, see https://kubernetes.io/docs/concepts/extend-kubernetes/[Extending Kubernetes]. -* *Portable* -- Many organizations have standardized their operations on [.noloc]`Kubernetes` because it allows them to manage all of their application needs in the same way. Developers can use the same pipelines to build and store containerized applications. Those applications can then be deployed to [.noloc]`Kubernetes` clusters running on-premises, in clouds, on point-of-sales terminals in restaurants, or on IOT devices dispersed across company's remote sites. Its open source nature makes it possible for people to develop these special [.noloc]`Kubernetes` distributions, along will tools needed to manage them. +* *Containerized* -- Kubernetes is a container orchestration tool. To use Kubernetes, you must first have your applications containerized. Depending on the type of application, this could be as a set of _microservices,_ as batch jobs or in other forms. Then, your applications can take advantage of a Kubernetes workflow that encompasses a huge ecosystem of tools, where containers can be stored as https://kubernetes.io/docs/concepts/containers/images/#multi-architecture-images-with-image-indexes[images in a container registry], deployed to a Kubernetes https://kubernetes.io/docs/concepts/architecture/[cluster], and run on an available https://kubernetes.io/docs/concepts/architecture/nodes/[node]. You can build and test individual containers on your local computer with Docker or another https://kubernetes.io/docs/setup/production-environment/container-runtimes/[container runtime], before deploying them to your Kubernetes cluster. +* *Scalable* -- If the demand for your applications exceeds the capacity of the running instances of those applications, Kubernetes is able to scale up. As needed, Kubernetes can tell if applications require more CPU or memory and respond by either automatically expanding available capacity or using more of existing capacity. Scaling can be done at the Pod level, if there is enough compute available to just run more instances of the application (https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/[horizontal Pod autoscaling]), or at the node level, if more nodes need to be brought up to handle the increased capacity (https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler[Cluster Autoscaler] or https://karpenter.sh/[Karpenter]). As capacity is no longer needed, these services can delete unnecessary Pods and shut down unneeded nodes. +* *Available* -- If an application or node becomes unhealthy or unavailable, Kubernetes can move running workloads to another available node. You can force the issue by simply deleting a running instance of a workload or node that's running your workloads. The bottom line here is that workloads can be brought up in other locations if they can no longer run where they are. +* *Declarative* -- Kubernetes uses active reconciliation to constantly check that the state that you declare for your cluster matches the actual state. By applying https://kubernetes.io/docs/concepts/overview/working-with-objects/[Kubernetes objects] to a cluster, typically through YAML-formatted configuration files, you can, for example, ask to start up the workloads you want to run on your cluster. You can later change the configurations to do something like use a later version of a container or allocate more memory. Kubernetes will do what it needs to do to establish the desired state. This can include bringing nodes up or down, stopping and restarting workloads, or pulling updated containers. +* *Composable* -- Because an application typically consists of multiple components, you want to be able to manage a set of these components (often represented by multiple containers) together. While Docker Compose offers a way to do this directly with Docker, the Kubernetes http://kompose.io/[Kompose] command can help you do that with Kubernetes. See https://kubernetes.io/docs/tasks/configure-pod-container/translate-compose-kubernetes/[Translate a Docker Compose File to Kubernetes Resources] for an example of how to do this. +* *Extensible* -- Unlike proprietary software, the open source Kubernetes project is designed to be open to you extending Kubernetes any way that you like to meet your needs. APIs and configuration files are open to direct modifications. Third-parties are encouraged to write their own https://kubernetes.io/docs/concepts/architecture/controller/[Controllers], to extend both infrastructure and end-user Kubernetes features. https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/[Webhooks] let you set up cluster rules to enforce policies and adapt to changing conditions. For more ideas on how to extend Kubernetes clusters, see https://kubernetes.io/docs/concepts/extend-kubernetes/[Extending Kubernetes]. +* *Portable* -- Many organizations have standardized their operations on Kubernetes because it allows them to manage all of their application needs in the same way. Developers can use the same pipelines to build and store containerized applications. Those applications can then be deployed to Kubernetes clusters running on-premises, in clouds, on point-of-sales terminals in restaurants, or on IOT devices dispersed across company's remote sites. Its open source nature makes it possible for people to develop these special Kubernetes distributions, along will tools needed to manage them. -[[managing-kubernetes,managing-kubernetes.title]] -=== Managing [.noloc]`Kubernetes` +[#managing-kubernetes] +=== Managing Kubernetes -[.noloc]`Kubernetes` source code is freely available, so with your own equipment you could install and manage [.noloc]`Kubernetes` yourself. However, self-managing [.noloc]`Kubernetes` requires deep operational expertise and takes time and effort to maintain. For those reasons, most people deploying production workloads choose a cloud provider (such as Amazon EKS) or on-premises provider (such as Amazon EKS Anywhere) with its own tested [.noloc]`Kubernetes` distribution and support of [.noloc]`Kubernetes` experts. This allows you to offload much of the undifferentiated heavy lifting needed to maintain your clusters, including: +Kubernetes source code is freely available, so with your own equipment you could install and manage Kubernetes yourself. However, self-managing Kubernetes requires deep operational expertise and takes time and effort to maintain. For those reasons, most people deploying production workloads choose a cloud provider (such as Amazon EKS) or on-premises provider (such as Amazon EKS Anywhere) with its own tested Kubernetes distribution and support of Kubernetes experts. This allows you to offload much of the undifferentiated heavy lifting needed to maintain your clusters, including: -* *Hardware* -- If you don't have hardware available to run [.noloc]`Kubernetes` per your requirements, a cloud provider such as {aws} Amazon EKS can save you on upfront costs. With Amazon EKS, this means that you can consume the best cloud resources offered by {aws}, including computer instances (Amazon Elastic Compute Cloud), your own private environment (Amazon VPC), central identity and permissions management (IAM), and storage (Amazon EBS). {aws} manages the computers, networks, data centers, and all the other physical components needed to run [.noloc]`Kubernetes`. Likewise, you don't have to plan your datacenter to handle the maximum capacity on your highest-demand days. For Amazon EKS Anywhere, or other on premises [.noloc]`Kubernetes` clusters, you are responsible for managing the infrastructure used in your [.noloc]`Kubernetes` deployments, but you can still rely on {aws} to help you keep [.noloc]`Kubernetes` up to date. -* *Control plane management* -- Amazon EKS manages the security and availability of the {aws}-hosted [.noloc]`Kubernetes` control plane, which is responsible for scheduling containers, managing the availability of applications, and other key tasks, so you can focus on your application workloads. If your cluster breaks, {aws} should have the means to restore your cluster to a running state. For Amazon EKS Anywhere, you would manage the control plane yourself. -* *Tested upgrades* -- When you upgrade your clusters, you can rely on Amazon EKS or Amazon EKS Anywhere to provide tested versions of their [.noloc]`Kubernetes` distributions. -* *Add-ons* -- There are hundreds of projects built to extend and work with [.noloc]`Kubernetes` that you can add to your cluster's infrastructure or use to aid the running of your workloads. Instead of building and managing those add-ons yourself, {aws} provides <> that you can use with your clusters. Amazon EKS Anywhere provides https://anywhere.eks.amazonaws.com/docs/packages/[Curated Packages] that include builds of many popular open source projects. So you don't have to build the software yourself or manage critical security patches, bug fixes, or upgrades. Likewise, if the defaults meet your needs, it's typical for very little configuration of those add-ons to be needed. See <> for details on extending your cluster with add-ons. +* *Hardware* -- If you don't have hardware available to run Kubernetes per your requirements, a cloud provider such as {aws} Amazon EKS can save you on upfront costs. With Amazon EKS, this means that you can consume the best cloud resources offered by {aws}, including computer instances (Amazon Elastic Compute Cloud), your own private environment (Amazon VPC), central identity and permissions management (IAM), and storage (Amazon EBS). {aws} manages the computers, networks, data centers, and all the other physical components needed to run Kubernetes. Likewise, you don't have to plan your datacenter to handle the maximum capacity on your highest-demand days. For Amazon EKS Anywhere, or other on premises Kubernetes clusters, you are responsible for managing the infrastructure used in your Kubernetes deployments, but you can still rely on {aws} to help you keep Kubernetes up to date. +* *Control plane management* -- Amazon EKS manages the security and availability of the {aws}-hosted Kubernetes control plane, which is responsible for scheduling containers, managing the availability of applications, and other key tasks, so you can focus on your application workloads. If your cluster breaks, {aws} should have the means to restore your cluster to a running state. For Amazon EKS Anywhere, you would manage the control plane yourself. +* *Tested upgrades* -- When you upgrade your clusters, you can rely on Amazon EKS or Amazon EKS Anywhere to provide tested versions of their Kubernetes distributions. +* *Add-ons* -- There are hundreds of projects built to extend and work with Kubernetes that you can add to your cluster's infrastructure or use to aid the running of your workloads. Instead of building and managing those add-ons yourself, {aws} provides <> that you can use with your clusters. Amazon EKS Anywhere provides https://anywhere.eks.amazonaws.com/docs/packages/[Curated Packages] that include builds of many popular open source projects. So you don't have to build the software yourself or manage critical security patches, bug fixes, or upgrades. Likewise, if the defaults meet your needs, it's typical for very little configuration of those add-ons to be needed. See <> for details on extending your cluster with add-ons. -[[kubernetes-in-action,kubernetes-in-action.title]] -=== [.noloc]`Kubernetes` in action +[#kubernetes-in-action] +=== Kubernetes in action -The following diagram shows key activities you would do as a [.noloc]`Kubernetes` Admin or Application Developer to create and use a [.noloc]`Kubernetes` cluster. In the process, it illustrates how [.noloc]`Kubernetes` components interact with each other, using the {aws} cloud as the example of the underlying cloud provider. +The following diagram shows key activities you would do as a Kubernetes Admin or Application Developer to create and use a Kubernetes cluster. In the process, it illustrates how Kubernetes components interact with each other, using the {aws} cloud as the example of the underlying cloud provider. image::images/k8sinaction.png[A Kubernetes cluster in action.,scaledwidth=100%] -A [.noloc]`Kubernetes` Admin creates the [.noloc]`Kubernetes` cluster using a tool specific to the type of provider on which the cluster will be built. This example uses the {aws} cloud as the provider, which offers the managed [.noloc]`Kubernetes` service called Amazon EKS. The managed service automatically allocates the resources needed to create the cluster, including creating two new Virtual Private Clouds (Amazon VPCs) for the cluster, setting up networking, and mapping [.noloc]`Kubernetes` permissions directly into the new VPCs for cloud asset management. The managed service also sees that the control plane services have places to run and allocates zero or more Amazon EC2 instances as [.noloc]`Kubernetes` nodes for running workloads. {aws} manages one Amazon VPC itself for the control plane, while the other Amazon VPC contains the customer nodes that run workloads. +A Kubernetes Admin creates the Kubernetes cluster using a tool specific to the type of provider on which the cluster will be built. This example uses the {aws} cloud as the provider, which offers the managed Kubernetes service called Amazon EKS. The managed service automatically allocates the resources needed to create the cluster, including creating two new Virtual Private Clouds (Amazon VPCs) for the cluster, setting up networking, and mapping Kubernetes permissions directly into the new VPCs for cloud asset management. The managed service also sees that the control plane services have places to run and allocates zero or more Amazon EC2 instances as Kubernetes nodes for running workloads. {aws} manages one Amazon VPC itself for the control plane, while the other Amazon VPC contains the customer nodes that run workloads. -Many of the [.noloc]`Kubernetes` Admin's tasks going forward are done using [.noloc]`Kubernetes` tools such as `kubectl`. That tool makes requests for services directly to the cluster's control plane. The ways that queries and changes are made to the cluster are then very similar to the ways you would do them on any [.noloc]`Kubernetes` cluster. +Many of the Kubernetes Admin's tasks going forward are done using Kubernetes tools such as `kubectl`. That tool makes requests for services directly to the cluster's control plane. The ways that queries and changes are made to the cluster are then very similar to the ways you would do them on any Kubernetes cluster. -An application developer wanting to deploy workloads to this cluster can perform several tasks. The developer needs to build the application into one or more container images, then push those images to a container registry that is accessible to the [.noloc]`Kubernetes` cluster. {aws} offers the Amazon Elastic Container Registry (Amazon ECR) for that purpose. +An application developer wanting to deploy workloads to this cluster can perform several tasks. The developer needs to build the application into one or more container images, then push those images to a container registry that is accessible to the Kubernetes cluster. {aws} offers the Amazon Elastic Container Registry (Amazon ECR) for that purpose. To run the application, the developer can create YAML-formatted configuration files that tell the cluster how to run the application, including which containers to pull from the registry and how to wrap those containers in Pods. The control plane (scheduler) schedules the containers to one or more nodes and the container runtime on each node actually pulls and runs the needed containers. The developer can also set up an application load balancer to balance traffic to available containers running on each node and expose the application so it is available on a public network to the outside world. With that all done, someone wanting to use the application can connect to the application endpoint to access it. -The following sections go through details of each of these features, from the perspective of [.noloc]`Kubernetes` Clusters and Workloads. +The following sections go through details of each of these features, from the perspective of Kubernetes Clusters and Workloads. -[[concepts-clusters,concepts-clusters.title]] +[#concepts-clusters] == Clusters -If your job is to start and manage [.noloc]`Kubernetes` clusters, you should know how [.noloc]`Kubernetes` clusters are created, enhanced, managed, and deleted. You should also know what the components are that make up a cluster and what you need to do to maintain those components. +If your job is to start and manage Kubernetes clusters, you should know how Kubernetes clusters are created, enhanced, managed, and deleted. You should also know what the components are that make up a cluster and what you need to do to maintain those components. -Tools for managing clusters handle the overlap between the [.noloc]`Kubernetes` services and the underlying hardware provider. For that reason, automation of these tasks tend to be done by the [.noloc]`Kubernetes` provider (such as Amazon EKS or Amazon EKS Anywhere) using tools that are specific to the provider. For example, to start an Amazon EKS cluster you can use `eksctl create cluster`, while for Amazon EKS Anywhere you can use `eksctl anywhere create cluster`. Note that while these commands create a [.noloc]`Kubernetes` cluster, they are specific to the provider and are not part of the [.noloc]`Kubernetes` project itself. +Tools for managing clusters handle the overlap between the Kubernetes services and the underlying hardware provider. For that reason, automation of these tasks tend to be done by the Kubernetes provider (such as Amazon EKS or Amazon EKS Anywhere) using tools that are specific to the provider. For example, to start an Amazon EKS cluster you can use `eksctl create cluster`, while for Amazon EKS Anywhere you can use `eksctl anywhere create cluster`. Note that while these commands create a Kubernetes cluster, they are specific to the provider and are not part of the Kubernetes project itself. -[[cluster-creation-and-management-tools,cluster-creation-and-management-tools.title]] +[#cluster-creation-and-management-tools] === Cluster creation and management tools -The [.noloc]`Kubernetes` project offers tools for creating a [.noloc]`Kubernetes` cluster manually. So if you want to install [.noloc]`Kubernetes` on a single machine, or run the control plane on a machine and add nodes manually, you can use CLI tools like https://kind.sigs.k8s.io/[kind], https://kubernetes.io/docs/tutorials/hello-minikube/[minikube], or https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/[kubeadm] that are listed under [.noloc]`Kubernetes` https://kubernetes.io/docs/tasks/tools/[Install Tools]. To simplify and automate the full lifecycle of cluster creation and management, it is much easier to use tools supported by an established [.noloc]`Kubernetes` provider, such as Amazon EKS or Amazon EKS Anywhere. +The Kubernetes project offers tools for creating a Kubernetes cluster manually. So if you want to install Kubernetes on a single machine, or run the control plane on a machine and add nodes manually, you can use CLI tools like https://kind.sigs.k8s.io/[kind], https://kubernetes.io/docs/tutorials/hello-minikube/[minikube], or https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/[kubeadm] that are listed under Kubernetes https://kubernetes.io/docs/tasks/tools/[Install Tools]. To simplify and automate the full lifecycle of cluster creation and management, it is much easier to use tools supported by an established Kubernetes provider, such as Amazon EKS or Amazon EKS Anywhere. -In {aws} Cloud, you can create link:eks/[Amazon EKS,type="documentation"] clusters using CLI tools, such as https://eksctl.io/[eksctl], or more declarative tools, such as Terraform (see https://github.com/aws-ia/terraform-aws-eks-blueprints[Amazon EKS Blueprints for Terraform]). You can also create a cluster from the {aws-management-console}. See link:eks/features/[Amazon EKS features,type="marketing"] for a list what you get with Amazon EKS. [.noloc]`Kubernetes` responsibilities that Amazon EKS takes on for you include: +In {aws} Cloud, you can create link:eks/[Amazon EKS,type="documentation"] clusters using CLI tools, such as https://eksctl.io/[eksctl], or more declarative tools, such as Terraform (see https://github.com/aws-ia/terraform-aws-eks-blueprints[Amazon EKS Blueprints for Terraform]). You can also create a cluster from the {aws-management-console}. See link:eks/features/[Amazon EKS features,type="marketing"] for a list what you get with Amazon EKS. Kubernetes responsibilities that Amazon EKS takes on for you include: * *Managed control plane* -- {aws} makes sure that the Amazon EKS cluster is available and scalable because it manages the control plane for you and makes it available across {aws} Availability Zones. -* *Node management* -- Instead of manually adding nodes, you can have Amazon EKS create nodes automatically as needed, using Managed Node Groups (see <>) or https://karpenter.sh/[Karpenter]. Managed Node Groups have integrations with [.noloc]`Kubernetes` https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md[Cluster Autoscaling]. Using node management tools, you can take advantage of cost savings, with things like link:AWSEC2/latest/UserGuide/using-spot-instances.html[Spot Instances,type="documentation"] and node consolidation, and availability, using https://karpenter.sh/docs/concepts/scheduling/[Scheduling]features to set how workloads are deployed and nodes are selected. -* *Cluster networking* -- Using CloudFormation templates, `eksctl` sets up networking between control plane and data plane (node) components in the [.noloc]`Kubernetes` cluster. It also sets up endpoints through which internal and external communications can take place. See link:containers/de-mystifying-cluster-networking-for-amazon-eks-worker-nodes[De-mystifying cluster networking for Amazon EKS worker nodes,type="blog"] for details. Communications between Pods in Amazon EKS is done using Amazon EKS Pod Identities (see <>), which provides a means of letting Pods tap into {aws} cloud methods of managing credentials and permissions. -* *Add-Ons* -- Amazon EKS saves you from having to build and add software components that are commonly used to support [.noloc]`Kubernetes` clusters. For example, when you create an Amazon EKS cluster from the {aws} Management console, it automatically adds the Amazon EKS kube-proxy (<>), Amazon VPC CNI plugin for [.noloc]`Kubernetes` (<>), and CoreDNS (<>) add-ons. See <> for more on these add-ons, including a list of which are available. +* *Node management* -- Instead of manually adding nodes, you can have Amazon EKS create nodes automatically as needed, using Managed Node Groups (see <>) or https://karpenter.sh/[Karpenter]. Managed Node Groups have integrations with Kubernetes https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md[Cluster Autoscaling]. Using node management tools, you can take advantage of cost savings, with things like link:AWSEC2/latest/UserGuide/using-spot-instances.html[Spot Instances,type="documentation"] and node consolidation, and availability, using https://karpenter.sh/docs/concepts/scheduling/[Scheduling] features to set how workloads are deployed and nodes are selected. +* *Cluster networking* -- Using CloudFormation templates, `eksctl` sets up networking between control plane and data plane (node) components in the Kubernetes cluster. It also sets up endpoints through which internal and external communications can take place. See link:containers/de-mystifying-cluster-networking-for-amazon-eks-worker-nodes[De-mystifying cluster networking for Amazon EKS worker nodes,type="blog"] for details. Communications between Pods in Amazon EKS is done using Amazon EKS Pod Identities (see <>), which provides a means of letting Pods tap into {aws} cloud methods of managing credentials and permissions. +* *Add-Ons* -- Amazon EKS saves you from having to build and add software components that are commonly used to support Kubernetes clusters. For example, when you create an Amazon EKS cluster from the {aws-management-console}, it automatically adds the Amazon EKS kube-proxy (<>), Amazon VPC CNI plugin for Kubernetes (<>), and CoreDNS (<>) add-ons. See <> for more on these add-ons, including a list of which are available. To run your clusters on your own on-premises computers and networks, Amazon offers https://anywhere.eks.amazonaws.com/[Amazon EKS Anywhere]. Instead of the {aws} Cloud being the provider, you have the choice of running Amazon EKS Anywhere on https://anywhere.eks.amazonaws.com/docs/getting-started/vsphere/[VMWare vSphere], https://anywhere.eks.amazonaws.com/docs/getting-started/baremetal/[bare metal] (https://tinkerbell.org[Tinkerbell provider]), https://anywhere.eks.amazonaws.com/docs/getting-started/snow/[Snow], https://anywhere.eks.amazonaws.com/docs/getting-started/cloudstack/[CloudStack], or https://anywhere.eks.amazonaws.com/docs/getting-started/nutanix/[Nutanix] platforms using your own equipment. Amazon EKS Anywhere is based on the same https://distro.eks.amazonaws.com/[Amazon EKS Distro] software that is used by Amazon EKS. However, Amazon EKS Anywhere relies on different implementations of the https://cluster-api.sigs.k8s.io/[Kubernetes Cluster API] (CAPI) interface to manage the full lifecycle of the machines in an Amazon EKS Anywhere cluster (such as https://github.com/kubernetes-sigs/cluster-api-provider-vsphere[CAPV] for vSphere and https://github.com/kubernetes-sigs/cluster-api-provider-cloudstack[CAPC] for CloudStack). Because the entire cluster is running on your equipment, you take on the added responsibility of managing the control plane and backing up its data (see `etcd` later in this document). -[[cluster-components,cluster-components.title]] +[#cluster-components] === Cluster components -[.noloc]`Kubernetes` cluster components are divided into two major areas: control plane and worker nodes. https://kubernetes.io/docs/concepts/overview/components/#control-plane-components[Control Plane Components] manage the cluster and provide access to its APIs. Worker nodes (sometimes just referred to as Nodes) provide the places where the actual workloads are run. https://kubernetes.io/docs/concepts/overview/components/#node-components[Node Components] consist of services that run on each node to communicate with the control plane and run containers. The set of worker nodes for your cluster is referred to as the _Data Plane_. +Kubernetes cluster components are divided into two major areas: control plane and worker nodes. https://kubernetes.io/docs/concepts/overview/components/#control-plane-components[Control Plane Components] manage the cluster and provide access to its APIs. Worker nodes (sometimes just referred to as Nodes) provide the places where the actual workloads are run. https://kubernetes.io/docs/concepts/overview/components/#node-components[Node Components] consist of services that run on each node to communicate with the control plane and run containers. The set of worker nodes for your cluster is referred to as the _Data Plane_. -[[concepts-control-plane,concepts-control-plane.title]] +[#concepts-control-plane] ==== Control plane The control plane consists of a set of services that manage the cluster. These services may all be running on a single computer or may be spread across multiple computers. Internally, these are referred to as Control Plane Instances (CPIs). How CPIs are run depends on the size of the cluster and requirements for high availability. As demand increase in the cluster, a control plane service can scale to provide more instances of that service, with requests being load balanced between the instances. -Tasks that components of the [.noloc]`Kubernetes` control plane performs include: +Tasks that components of the Kubernetes control plane performs include: -* *Communicating with cluster components (API server)* -- The API server (https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/[kube-apiserver]) exposes the [.noloc]`Kubernetes` API so requests to the cluster can be made from both inside and outside of the cluster. In other words, requests to add or change a cluster's objects (Pods, Services, Nodes, and so on) can come from outside commands, such as requests from `kubectl` to run a Pod. Likewise, requests can be made from the API server to components within the cluster, such as a query to the `kubelet` service for the status of a Pod. +* *Communicating with cluster components (API server)* -- The API server (https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/[kube-apiserver]) exposes the Kubernetes API so requests to the cluster can be made from both inside and outside of the cluster. In other words, requests to add or change a cluster's objects (Pods, Services, Nodes, and so on) can come from outside commands, such as requests from `kubectl` to run a Pod. Likewise, requests can be made from the API server to components within the cluster, such as a query to the `kubelet` service for the status of a Pod. * *Store data about the cluster (`etcd` key value store)* -- The `etcd` service provides the critical role of keeping track of the current state of the cluster. If the `etcd` service became inaccessible, you would be unable to update or query the status of the cluster, though workloads would continue to run for a while. For that reason, critical clusters typically have multiple, load-balanced instances of the `etcd` service running at a time and do periodic backups of the `etcd` key value store in case of data loss or corruption. Keep in mind that, in Amazon EKS, this is all handled for you automatically by default. Amazon EKS Anywhere provides instruction for https://anywhere.eks.amazonaws.com/docs/clustermgmt/etcd-backup-restore/[etcd backup and restore]. See the https://etcd.io/docs/v3.5/learning/data_model/[etcd Data Model] to learn how `etcd` manages data. -* *Schedule Pods to nodes (Scheduler)* -- Requests to start or stop a Pod in [.noloc]`Kubernetes` are directed to the https://kubernetes.io/docs/concepts/scheduling-eviction/kube-scheduler/[Kubernetes Scheduler] (https://kubernetes.io/docs/reference/command-line-tools-reference/kube-scheduler/[kube-scheduler]). Because a cluster could have multiple nodes that are capable of running the Pod, it is up to the Scheduler to choose which node (or nodes, in the case of replicas) the Pod should run on. If there is not enough available capacity to run the requested Pod on an existing node, the request will fail, unless you have made other provisions. Those provisions could include enabling services such as Managed Node Groups (<>) or https://karpenter.sh/[Karpenter] that can automatically start up new nodes to handle the workloads. -* *Keep components in desired state (Controller Manager)* -- The [.noloc]`Kubernetes` Controller Manager runs as a daemon process (https://kubernetes.io/docs/reference/command-line-tools-reference/kube-controller-manager/[kube-controller-manager]) to watch the state of the cluster and make changes to the cluster to reestablish the expected states. In particular, there are several controllers that watch over different [.noloc]`Kubernetes` objects, which includes a `statefulset-controller`, `endpoint-controller`, `cronjob-controller`, `node-controller`, and others. -* *Manage cloud resources (Cloud Controller Manager)* -- Interactions between [.noloc]`Kubernetes` and the cloud provider that carries out requests for the underlying data center resources are handled by the https://kubernetes.io/docs/concepts/architecture/cloud-controller/[Cloud Controller Manager] (https://github.com/kubernetes/kubernetes/tree/master/cmd/cloud-controller-manager[cloud-controller-manager]). Controllers managed by the Cloud Controller Manager can include a route controller (for setting up cloud network routes), service controller (for using cloud load balancing services), and node lifecycle controller (to keep nodes in sync with Kubernetes throughout their lifecycles). +* *Schedule Pods to nodes (Scheduler)* -- Requests to start or stop a Pod in Kubernetes are directed to the https://kubernetes.io/docs/concepts/scheduling-eviction/kube-scheduler/[Kubernetes Scheduler] (https://kubernetes.io/docs/reference/command-line-tools-reference/kube-scheduler/[kube-scheduler]). Because a cluster could have multiple nodes that are capable of running the Pod, it is up to the Scheduler to choose which node (or nodes, in the case of replicas) the Pod should run on. If there is not enough available capacity to run the requested Pod on an existing node, the request will fail, unless you have made other provisions. Those provisions could include enabling services such as Managed Node Groups (<>) or https://karpenter.sh/[Karpenter] that can automatically start up new nodes to handle the workloads. +* *Keep components in desired state (Controller Manager)* -- The Kubernetes Controller Manager runs as a daemon process (https://kubernetes.io/docs/reference/command-line-tools-reference/kube-controller-manager/[kube-controller-manager]) to watch the state of the cluster and make changes to the cluster to reestablish the expected states. In particular, there are several controllers that watch over different Kubernetes objects, which includes a `statefulset-controller`, `endpoint-controller`, `cronjob-controller`, `node-controller`, and others. +* *Manage cloud resources (Cloud Controller Manager)* -- Interactions between Kubernetes and the cloud provider that carries out requests for the underlying data center resources are handled by the https://kubernetes.io/docs/concepts/architecture/cloud-controller/[Cloud Controller Manager] (https://github.com/kubernetes/kubernetes/tree/master/cmd/cloud-controller-manager[cloud-controller-manager]). Controllers managed by the Cloud Controller Manager can include a route controller (for setting up cloud network routes), service controller (for using cloud load balancing services), and node lifecycle controller (to keep nodes in sync with Kubernetes throughout their lifecycles). -[[worker-nodes-data-plane,worker-nodes-data-plane.title]] +[#worker-nodes-data-plane] ==== Worker Nodes (data plane) -For a single-node [.noloc]`Kubernetes` cluster, workloads run on the same machine as the control plane. However, a more standard configuration is to have one or more separate computer systems (https://kubernetes.io/docs/concepts/architecture/nodes/[Nodes]) that are dedicated to running [.noloc]`Kubernetes` workloads. +For a single-node Kubernetes cluster, workloads run on the same machine as the control plane. However, a more standard configuration is to have one or more separate computer systems (https://kubernetes.io/docs/concepts/architecture/nodes/[Nodes]) that are dedicated to running Kubernetes workloads. -When you first create a [.noloc]`Kubernetes` cluster, some cluster creation tools allow you to configure a certain number nodes to be added to the cluster (either by identifying existing computer systems or by having the provider create new ones). Before any workloads are added to those systems, services are added to each node to implement these features: +When you first create a Kubernetes cluster, some cluster creation tools allow you to configure a certain number nodes to be added to the cluster (either by identifying existing computer systems or by having the provider create new ones). Before any workloads are added to those systems, services are added to each node to implement these features: * *Manage each node (`kubelet`)* -- The API server communicates with the https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/[kubelet] service running on each node to make sure that the node is properly registered and Pods requested by the Scheduler are running. The kubelet can read the Pod manifests and set up storage volumes or other features needed by the Pods on the local system. It can also check on the health of the locally running containers. -* *Run containers on a node (container runtime)* -- The https://kubernetes.io/docs/setup/production-environment/container-runtimes/[Container Runtime] on each node manages the containers requested for each Pod assigned to the node. That means that it can pull container images from the appropriate registry, run the container, stop it, and responds to queries about the container. The default container runtime is https://github.com/containerd/containerd/blob/main/docs/getting-started.md[containerd]. As of [.noloc]`Kubernetes` 1.24, the special integration of [.noloc]`Docker` (`dockershim`) that could be used as the container runtime was dropped from [.noloc]`Kubernetes`. While you can still use [.noloc]`Docker` to test and run containers on your local system, to use [.noloc]`Docker` with [.noloc]`Kubernetes` you would now have to https://docs.docker.com/engine/install/#server[Install Docker Engine] on each node to use it with [.noloc]`Kubernetes`. -* *Manage networking between containers (kube-proxy)* -- To be able to support communication between Pods, [.noloc]`Kubernetes` uses a feature referred to as a https://kubernetes.io/docs/concepts/services-networking/service/[Service] to set up Pod networks that track IP addresses and ports associated with those Pods. The https://kubernetes.io/docs/reference/command-line-tools-reference/kube-proxy/[kube-proxy] service runs on every node to allow that communication between Pods to take place. +* *Run containers on a node (container runtime)* -- The https://kubernetes.io/docs/setup/production-environment/container-runtimes/[Container Runtime] on each node manages the containers requested for each Pod assigned to the node. That means that it can pull container images from the appropriate registry, run the container, stop it, and responds to queries about the container. The default container runtime is https://github.com/containerd/containerd/blob/main/docs/getting-started.md[containerd]. As of Kubernetes 1.24, the special integration of Docker (`dockershim`) that could be used as the container runtime was dropped from Kubernetes. While you can still use Docker to test and run containers on your local system, to use Docker with Kubernetes you would now have to https://docs.docker.com/engine/install/#server[Install Docker Engine] on each node to use it with Kubernetes. +* *Manage networking between containers (`kube-proxy`)* -- To be able to support communication between Pods, Kubernetes uses a feature referred to as a https://kubernetes.io/docs/concepts/services-networking/service/[Service] to set up Pod networks that track IP addresses and ports associated with those Pods. The https://kubernetes.io/docs/reference/command-line-tools-reference/kube-proxy/[kube-proxy] service runs on every node to allow that communication between Pods to take place. -[[extend-clusters,extend-clusters.title]] +[#extend-clusters] === Extend Clusters -There are some services you can add to [.noloc]`Kubernetes` to support the cluster, but are not run in the control plane. These services often run directly on nodes in the kube-system namespace or in its own namespace (as is often done with third-party service providers). A common example is the CoreDNS service, which provides DNS services to the cluster. Refer to https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster-services/[Discovering builtin services] for information on how to see which cluster services are running in kube-system on your cluster. +There are some services you can add to Kubernetes to support the cluster, but are not run in the control plane. These services often run directly on nodes in the kube-system namespace or in its own namespace (as is often done with third-party service providers). A common example is the CoreDNS service, which provides DNS services to the cluster. Refer to https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster-services/[Discovering builtin services] for information on how to see which cluster services are running in kube-system on your cluster. -There are different types of add-ons you can consider adding to your clusters. To keep your clusters healthy, you can add observability features (see <>) that allow you to do things like logging, auditing, and metrics. With this information, you can troubleshoot problems that occur, often through the same observability interfaces. Examples of these types of services include link:guardduty/latest/ug/runtime-monitoring.html[Amazon GuardDuty,type="documentation"], CloudWatch (see <>), https://aws-otel.github.io/[{aws} Distro for OpenTelemetry], Amazon VPC CNI plugin for [.noloc]`Kubernetes` (see <>), and https://grafana.com/docs/grafana-cloud/monitor-infrastructure/kubernetes-monitoring/configuration/config-aws-eks/[Grafana Kubernetes Monitoring]. For storage (see <>), add-ons to Amazon EKS include Amazon Elastic Block Store CSI Driver (see <>), Amazon Elastic File System CSI Driver (see <>), and several third-party storage add-ons such as Amazon FSx for NetApp ONTAP CSI driver <>). +There are different types of add-ons you can consider adding to your clusters. To keep your clusters healthy, you can add observability features (see <>) that allow you to do things like logging, auditing, and metrics. With this information, you can troubleshoot problems that occur, often through the same observability interfaces. Examples of these types of services include link:guardduty/latest/ug/runtime-monitoring.html[Amazon GuardDuty,type="documentation"], CloudWatch (see <>), https://aws-otel.github.io/[{aws} Distro for OpenTelemetry], Amazon VPC CNI plugin for Kubernetes (see <>), and https://grafana.com/docs/grafana-cloud/monitor-infrastructure/kubernetes-monitoring/configuration/config-aws-eks/[Grafana Kubernetes Monitoring]. For storage (see <>), add-ons to Amazon EKS include Amazon Elastic Block Store CSI Driver (see <>), Amazon Elastic File System CSI Driver (see <>), and several third-party storage add-ons such as Amazon FSx for NetApp ONTAP CSI driver <>). For a more complete list of available Amazon EKS add-ons, see <>. -[[workloads,workloads.title]] +[#workloads] == Workloads -[.noloc]`Kubernetes` defines a https://kubernetes.io/docs/concepts/workloads/[Workload] as "`an application running on [.noloc]`Kubernetes`.`" That application can consist of a set of microservices run as https://kubernetes.io/docs/reference/glossary/?fundamental=true#term-container[Containers] in https://kubernetes.io/docs/reference/glossary/?fundamental=true#term-pod[Pods], or could be run as a batch job or other type of applications. The job of [.noloc]`Kubernetes` is to make sure that the requests that you make for those objects to be set up or deployed are carried out. As someone deploying applications, you should learn about how containers are built, how Pods are defined, and what methods you can use for deploying them. +Kubernetes defines a https://kubernetes.io/docs/concepts/workloads/[Workload] as "`an application running on Kubernetes.`" That application can consist of a set of microservices run as https://kubernetes.io/docs/reference/glossary/?fundamental=true#term-container[Containers] in https://kubernetes.io/docs/reference/glossary/?fundamental=true#term-pod[Pods], or could be run as a batch job or other type of applications. The job of Kubernetes is to make sure that the requests that you make for those objects to be set up or deployed are carried out. As someone deploying applications, you should learn about how containers are built, how Pods are defined, and what methods you can use for deploying them. -[[containers,containers.title]] +[#containers] === Containers -The most basic element of an application workload that you deploy and manage in [.noloc]`Kubernetes` is a _https://kubernetes.io/docs/concepts/workloads/pods/[Pod]_. A Pod represents a way of holding the components of an application as well as defining specifications that describe the Pod's attributes. Contrast this to something like an RPM or Deb package, which packages together software for a Linux system, but does not itself run as an entity. +The most basic element of an application workload that you deploy and manage in Kubernetes is a _https://kubernetes.io/docs/concepts/workloads/pods/[Pod]_. A Pod represents a way of holding the components of an application as well as defining specifications that describe the Pod's attributes. Contrast this to something like an RPM or Deb package, which packages together software for a Linux system, but does not itself run as an entity. Because the Pod is the smallest deployable unit, it typically holds a single container. However, multiple containers can be in a Pod in cases where the containers are tightly coupled. For example, a web server container might be packaged in a Pod with a https://kubernetes.io/docs/concepts/workloads/pods/sidecar-containers/[sidecar] type of container that may provide logging, monitoring, or other service that is closely tied to the web server container. In this case, being in the same Pod ensures that for each running instance of the Pod, both containers always run on the same node. Likewise, all containers in a Pod share the same environment, with the containers in a Pod running as though they are in the same isolated host. The effect of this is that the containers share a single IP address that provides access to the Pod and the containers can communicate with each other as though they were running on their own localhost. @@ -173,12 +168,12 @@ Pod specifications (https://kubernetes.io/docs/reference/kubernetes-api/workload While a Pod is the smallest unit you deploy, a container is the smallest unit that you build and manage. -[[building-containers,building-containers.title]] +[#building-containers] ==== Building Containers -The Pod is really just a structure around one or more containers, with each container itself holding the file system, executables, configuration files, libraries, and other components to actually run the application. Because a company called [.noloc]`Docker` Inc. first popularized containers, some people refer to containers as [.noloc]`Docker` Containers. However, the https://opencontainers.org/[Open Container Initiative] has since defined container runtimes, images, and distribution methods for the industry. Add to that the fact that containers were created from many existing Linux features, others often refer to containers as OCI Containers, Linux Containers, or just Containers. +The Pod is really just a structure around one or more containers, with each container itself holding the file system, executables, configuration files, libraries, and other components to actually run the application. Because a company called Docker Inc. first popularized containers, some people refer to containers as Docker Containers. However, the https://opencontainers.org/[Open Container Initiative] has since defined container runtimes, images, and distribution methods for the industry. Add to that the fact that containers were created from many existing Linux features, others often refer to containers as OCI Containers, Linux Containers, or just Containers. -When you build a container, you typically start with a [.noloc]`Dockerfile` (literally named that). Inside that Dockerfile, you identify: +When you build a container, you typically start with a Dockerfile (literally named that). Inside that Dockerfile, you identify: @@ -188,72 +183,72 @@ When you build a container, you typically start with a [.noloc]`Dockerfile` (lit While the `docker` command and service have traditionally been used to build containers (`docker build`), other tools that are available to build container images include https://docs.podman.io/en/stable/markdown/podman-build.1.html[podman] and https://github.com/containerd/nerdctl[nerdctl]. See link:containers/building-better-container-images[Building Better Container Images,type="blog"] or https://docs.docker.com/build/[Overview of Docker Build] to learn about building containers. -[[storing-containers,storing-containers.title]] +[#storing-containers] ==== Storing Containers Once you've built your container image, you can store it in a container https://distribution.github.io/distribution/[distribution registry] on your workstation or on a public container registry. Running a private container registry on your workstation allows you to store container images locally, making them readily available to you. To store container images in a more public manner, you can push them to a public container registry. Public container registries provide a central location for storing and distributing container images. Examples of public container registries include the link:ecr/[Amazon Elastic Container Registry,type="marketing"], https://quay.io/[Red Hat Quay] registry, and https://hub.docker.com/[Docker Hub] registry. -When running containerized workloads on Amazon Elastic Kubernetes Service (Amazon EKS) we recommend pulling copies of [.noloc]`Docker` Official Images that are stored in Amazon Elastic Container Registry. Amazon ECR has been storing these images since 2021. You can search for popular container images in the https://gallery.ecr.aws/[Amazon ECR Public Gallery], and specifically for the [.noloc]`Docker` Hub images, you can search the https://gallery.ecr.aws/docker/[Amazon ECR Docker Gallery]. +When running containerized workloads on Amazon Elastic Kubernetes Service (Amazon EKS) we recommend pulling copies of Docker Official Images that are stored in Amazon Elastic Container Registry. Amazon ECR has been storing these images since 2021. You can search for popular container images in the https://gallery.ecr.aws/[Amazon ECR Public Gallery], and specifically for the Docker Hub images, you can search the https://gallery.ecr.aws/docker/[Amazon ECR Docker Gallery]. -[[running-containers,running-containers.title]] +[#running-containers] ==== Running containers -Because containers are built in a standard format, a container can run on any machine that can run a container runtime (such as [.noloc]`Docker`) and whose contents match the local machine's architecture (such as `x86_64` or `arm`). To test a container or just run it on your local desktop, you can use `docker run` or `podman run` commands to start up a container on the localhost. For [.noloc]`Kubernetes`, however, each worker node has a container runtime deployed and it is up to [.noloc]`Kubernetes` to request that a node run a container. +Because containers are built in a standard format, a container can run on any machine that can run a container runtime (such as Docker) and whose contents match the local machine's architecture (such as `x86_64` or `arm`). To test a container or just run it on your local desktop, you can use `docker run` or `podman run` commands to start up a container on the localhost. For Kubernetes, however, each worker node has a container runtime deployed and it is up to Kubernetes to request that a node run a container. -Once a container has been assigned to run on a node, the node looks to see if the requested version of the container image already exists on the node. If it doesn't, [.noloc]`Kubernetes` tells the container runtime to pull that container from the appropriate container registry, then run that container locally. Keep in mind that a _container image_ refers to the software package that is moved around between your laptop, the container registry, and [.noloc]`Kubernetes` nodes. A _container_ refers to a running instance of that image. +Once a container has been assigned to run on a node, the node looks to see if the requested version of the container image already exists on the node. If it doesn't, Kubernetes tells the container runtime to pull that container from the appropriate container registry, then run that container locally. Keep in mind that a _container image_ refers to the software package that is moved around between your laptop, the container registry, and Kubernetes nodes. A _container_ refers to a running instance of that image. -[[pods,pods.title]] +[#pods] === Pods Once your containers are ready, working with Pods includes configuring, deploying, and making the Pods accessible. -[[configuring-pods,configuring-pods.title]] +[#configuring-pods] ==== Configuring Pods When you define a Pod, you assign a set of attributes to it. Those attributes must include at least the Pod name and the container image to run. However, there are many other things you want to configure with your Pod definitions as well (see the https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#PodSpec[PodSpec] page for details on what can go into a Pod). These include: -* *Storage* -- When a running container is stopped and deleted, data storage in that container will disappear, unless you set up more permanent storage. [.noloc]`Kubernetes` supports many different storage types and abstracts them under the umbrella of https://kubernetes.io/docs/concepts/storage/volumes/[Volumes]. Storage types include https://kubernetes.io/docs/concepts/storage/volumes/#cephfs[CephFS], https://kubernetes.io/docs/concepts/storage/volumes/#nfs[NFS], https://kubernetes.io/docs/concepts/storage/volumes/#iscsi[iSCSI], and others. You can even use a https://kubernetes.io/docs/concepts/storage/volumes/#local[local block device] from the local computer. With one of those storage types available from your cluster, you can mount the storage volume to a selected mount point in your container's file system. A https://kubernetes.io/docs/concepts/storage/persistent-volumes/[Persistent Volume] is one that continues to exist after the Pod is deleted, while an https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/[Ephemeral Volume] is deleted when the Pod is deleted. If your cluster administrator created different https://kubernetes.io/docs/concepts/storage/storage-classes/[StorageClasses] for your cluster, you might have the option for choosing the attributes of the storage you use, such as whether the volume is deleted or reclaimed after use, whether it will expand if more space is needed, and even whether it meets certain performance requirements. +* *Storage* -- When a running container is stopped and deleted, data storage in that container will disappear, unless you set up more permanent storage. Kubernetes supports many different storage types and abstracts them under the umbrella of https://kubernetes.io/docs/concepts/storage/volumes/[Volumes]. Storage types include https://kubernetes.io/docs/concepts/storage/volumes/#cephfs[CephFS], https://kubernetes.io/docs/concepts/storage/volumes/#nfs[NFS], https://kubernetes.io/docs/concepts/storage/volumes/#iscsi[iSCSI], and others. You can even use a https://kubernetes.io/docs/concepts/storage/volumes/#local[local block device] from the local computer. With one of those storage types available from your cluster, you can mount the storage volume to a selected mount point in your container's file system. A https://kubernetes.io/docs/concepts/storage/persistent-volumes/[Persistent Volume] is one that continues to exist after the Pod is deleted, while an https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/[Ephemeral Volume] is deleted when the Pod is deleted. If your cluster administrator created different https://kubernetes.io/docs/concepts/storage/storage-classes/[storage classes] for your cluster, you might have the option for choosing the attributes of the storage you use, such as whether the volume is deleted or reclaimed after use, whether it will expand if more space is needed, and even whether it meets certain performance requirements. * *Secrets* -- By making https://kubernetes.io/docs/concepts/configuration/secret/[Secrets] available to containers in Pod specs, you can provide the permissions those containers need to access file systems, data bases, or other protected assets. Keys, passwords, and tokens are among the items that can be stored as secrets. Using secrets makes it so you don't have to store this information in container images, but need only make the secrets available to running containers. Similar to Secrets are https://kubernetes.io/docs/concepts/configuration/configmap/[ConfigMaps]. A `ConfigMap` tends to hold less critical information, such as key-value pairs for configuring a service. * *Container resources* -- Objects for further configuring containers can take the form of resource configuration. For each container, you can request the amount of memory and CPU that it can use, as well as place limits of the total amount of those resources that the container can use. See https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/[Resource Management for Pods and Containers] for examples. * *Disruptions* -- Pods can be disrupted involuntarily (a node goes down) or voluntarily (an upgrade is desired). By configuring a https://kubernetes.io/docs/concepts/workloads/pods/disruptions/#pod-disruption-budgets[Pod disruption budget], you can exert some control over how available your application remains when disruptions occur. See https://kubernetes.io/docs/tasks/run-application/configure-pdb/[Specifying a Disruption Budget] for your application for examples. -* *Namespaces* -- [.noloc]`Kubernetes` provides different ways to isolate [.noloc]`Kubernetes` components and workloads from each other. Running all the Pods for a particular application in the same https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/[Namespace] is a common way to secure and manage those Pods together. You can create your own namespaces to use or choose to not indicate a namespace (which causes [.noloc]`Kubernetes` to use the `default` namespace). [.noloc]`Kubernetes` control plane components typically run in the https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/[kube-system] namespace. +* *Namespaces* -- Kubernetes provides different ways to isolate Kubernetes components and workloads from each other. Running all the Pods for a particular application in the same https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/[Namespace] is a common way to secure and manage those Pods together. You can create your own namespaces to use or choose to not indicate a namespace (which causes Kubernetes to use the `default` namespace). Kubernetes control plane components typically run in the https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/[kube-system] namespace. -The configuration just described is typically gathered together in a YAML file to be applied to the [.noloc]`Kubernetes` cluster. For personal [.noloc]`Kubernetes` clusters, you might just store these YAML files on your local system. However, with more critical clusters and workloads, https://www.eksworkshop.com/docs/automation/gitops/[GitOps] is a popular way to automate storage and updates to both workload and [.noloc]`Kubernetes` infrastructure resources. +The configuration just described is typically gathered together in a YAML file to be applied to the Kubernetes cluster. For personal Kubernetes clusters, you might just store these YAML files on your local system. However, with more critical clusters and workloads, https://www.eksworkshop.com/docs/automation/gitops/[GitOps] is a popular way to automate storage and updates to both workload and Kubernetes infrastructure resources. The objects used to gather together and deploy Pod information is defined by one of the following deployment methods. -[[deploying-pods,deploying-pods.title]] +[#deploying-pods] ==== Deploying Pods The method you would choose for deploying Pods depends on the type of application you plan to run with those Pods. Here are some of your choices: -* *Stateless applications* -- A stateless application doesn't save a client's session data, so another session doesn't need to refer back to what happened to a previous session. This makes is easier to just replace Pods with new ones if they become unhealthy or move them around without saving state. If you are running a stateless application (such as a web server), you can use a https://kubernetes.io/docs/concepts/workloads/controllers/deployment/[Deployment] to deploy https://kubernetes.io/docs/concepts/workloads/pods/[Pods]and https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/[ReplicaSets]. A ReplicaSet defines how many instances of a Pod that you want running concurrently. Although you can run a ReplicaSet directly, it is common to run replicas directly within a Deployment, to define how many replicas of a Pod should be running at a time. -* *Stateful applications* -- A stateful application is one where the identity of the Pod and the order in which Pods are launched are important. These applications need persistent storage that is stable and need to be deployed and scaled in a consistent manner. To deploy a stateful application in [.noloc]`Kubernetes`, you can use https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/[StatefulSets]. An example of an application that is typically run as a StatefulSet is a database. Within a StatefulSet, you could define replicas, the Pod and its containers, storage volumes to mount, and locations in the container where data are stored. See https://kubernetes.io/docs/tasks/run-application/run-replicated-stateful-application/[Run a Replicated Stateful Application] for an example of a database being deployed as a ReplicaSet. -* *Per-node applications* -- There are times when you want to run an application on every node in your [.noloc]`Kubernetes` cluster. For example, your data center might require that every computer run a monitoring application or a particular remote access service. For [.noloc]`Kubernetes`, you can use a https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/[DaemonSet] to ensure that the selected application runs on every node in your cluster. +* *Stateless applications* -- A stateless application doesn't save a client's session data, so another session doesn't need to refer back to what happened to a previous session. This makes it easier to just replace Pods with new ones if they become unhealthy or move them around without saving state. If you are running a stateless application (such as a web server), you can use a https://kubernetes.io/docs/concepts/workloads/controllers/deployment/[Deployment] to deploy https://kubernetes.io/docs/concepts/workloads/pods/[Pods]and https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/[ReplicaSets]. A ReplicaSet defines how many instances of a Pod that you want running concurrently. Although you can run a ReplicaSet directly, it is common to run replicas directly within a Deployment, to define how many replicas of a Pod should be running at a time. +* *Stateful applications* -- A stateful application is one where the identity of the Pod and the order in which Pods are launched are important. These applications need persistent storage that is stable and need to be deployed and scaled in a consistent manner. To deploy a stateful application in Kubernetes, you can use https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/[StatefulSets]. An example of an application that is typically run as a StatefulSet is a database. Within a StatefulSet, you could define replicas, the Pod and its containers, storage volumes to mount, and locations in the container where data are stored. See https://kubernetes.io/docs/tasks/run-application/run-replicated-stateful-application/[Run a Replicated Stateful Application] for an example of a database being deployed as a ReplicaSet. +* *Per-node applications* -- There are times when you want to run an application on every node in your Kubernetes cluster. For example, your data center might require that every computer run a monitoring application or a particular remote access service. For Kubernetes, you can use a https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/[DaemonSet] to ensure that the selected application runs on every node in your cluster. * *Applications run to completion* -- There are some applications you want to run to complete a particular task. This could include one that runs monthly status reports or cleans out old data. A https://kubernetes.io/docs/concepts/workloads/controllers/job/[Job] object can be used to set up an application to start up and run, then exit when the task is done. A https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/[CronJob] object lets you set up an application to run at a specific hour, minute, day of the month, month, or day of the week, using a structure defined by the Linux https://man7.org/linux/man-pages/man5/crontab.5.html[crontab] format. -[[making-applications-accessible-from-the-network,making-applications-accessible-from-the-network.title]] +[#making-applications-accessible-from-the-network] ==== Making applications accessible from the network -With applications often deployed as a set of microservices that moved around to different places, [.noloc]`Kubernetes` needed a way for those microservices to be able to find each other. Also, for others to access an application outside of the [.noloc]`Kubernetes` cluster, [.noloc]`Kubernetes` needed a way to expose that application on outside addresses and ports. These networking-related features are done with Service and Ingress objects, respectively: +With applications often deployed as a set of microservices that moved around to different places, Kubernetes needed a way for those microservices to be able to find each other. Also, for others to access an application outside of the Kubernetes cluster, Kubernetes needed a way to expose that application on outside addresses and ports. These networking-related features are done with Service and Ingress objects, respectively: -* *Services* -- Because a Pod can move around to different nodes and addresses, another Pod that needed to communicate with the first Pod could find it difficult to find where it is. To solve this problem, [.noloc]`Kubernetes` lets you represent an application as a https://kubernetes.io/docs/concepts/services-networking/service/[Service]. With a Service, you can identify a Pod or set of Pods with a particular name, then indicate what port exposes that application's service from the Pod and what ports another application could use to contact that service. Another Pod within a cluster can simply request a Service by name and [.noloc]`Kubernetes` will direct that request to the proper port for an instance of the Pod running that service. -* *Ingress* -- https://kubernetes.io/docs/concepts/services-networking/ingress/[Ingress] is what can make applications represented by [.noloc]`Kubernetes` Services available to clients that are outside of the cluster. Basic features of Ingress include a load balancer (managed by Ingress), the Ingress controller, and rules for routing requests from the controller to the Service. There are several https://kubernetes.io/docs/concepts/services-networking/ingress-controllers/[Ingress Controllers] that you can choose from with [.noloc]`Kubernetes`. +* *Services* -- Because a Pod can move around to different nodes and addresses, another Pod that needs to communicate with the first Pod could find it difficult to locate where it is. To solve this problem, Kubernetes lets you represent an application as a https://kubernetes.io/docs/concepts/services-networking/service/[Service]. With a Service, you can identify a Pod or set of Pods with a particular name, then indicate what port exposes that application's service from the Pod and what ports another application could use to contact that service. Another Pod within a cluster can simply request a Service by name and Kubernetes will direct that request to the proper port for an instance of the Pod running that service. +* *Ingress* -- https://kubernetes.io/docs/concepts/services-networking/ingress/[Ingress] is what can make applications represented by Kubernetes Services available to clients that are outside of the cluster. Basic features of Ingress include a load balancer (managed by Ingress), the Ingress controller, and rules for routing requests from the controller to the Service. There are several https://kubernetes.io/docs/concepts/services-networking/ingress-controllers/[Ingress Controllers] that you can choose from with Kubernetes. -[[next-steps,next-steps.title]] +[#next-steps] == Next steps -Understanding basic [.noloc]`Kubernetes` concepts and how they relate to Amazon EKS will help you navigate both the link:eks/[Amazon EKS documentation,type="documentation"] and https://kubernetes.io/docs[Kubernetes documentation] to find the information you need to manage Amazon EKS clusters and deploy workloads to those clusters. To begin using Amazon EKS, choose from the following: +Understanding basic Kubernetes concepts and how they relate to Amazon EKS will help you navigate both the link:eks/[Amazon EKS documentation,type="documentation"] and https://kubernetes.io/docs[Kubernetes documentation] to find the information you need to manage Amazon EKS clusters and deploy workloads to those clusters. To begin using Amazon EKS, choose from the following: @@ -261,4 +256,3 @@ Understanding basic [.noloc]`Kubernetes` concepts and how they relate to Amazon * <> * <> * <> - diff --git a/latest/ug/what-is/what-is-eks.adoc b/latest/ug/what-is/what-is-eks.adoc index ad3c14ea1..401123ef0 100644 --- a/latest/ug/what-is/what-is-eks.adoc +++ b/latest/ug/what-is/what-is-eks.adoc @@ -1,91 +1,123 @@ -//!!NODE_ROOT include::../attributes.txt[] + [.topic] -[[what-is-eks,what-is-eks.title]] +[#what-is-eks] = What is Amazon EKS? -:doctype: book -:sectnums: -:toc: left -:icons: font -:experimental: -:idprefix: -:idseparator: - -:sourcedir: . -:info_doctype: chapter -:info_title: What is Amazon EKS? -:info_titleabbrev: What is Amazon EKS? -:keywords: Amazon Elastic Kubernetes Service, Amazon EKS, about, summary, description -:info_abstract: Learn to manage containerized applications with Amazon EKS [abstract] -- Learn to manage containerized applications with Amazon EKS -- -Amazon Elastic Kubernetes Service (Amazon EKS) is a managed Kubernetes service that eliminates the need to operate and maintain the availability and scalability of Kubernetes clusters in Amazon Web Services ({aws}) and in your own data centers. https://kubernetes.io/docs/concepts/overview/[Kubernetes] is an open source system that automates the management, scaling, and deployment of containerized applications. To get started, see the <> page in the Amazon EKS User Guide. -[[eks-features,eks-features.title]] -== Features of Amazon EKS +## Amazon EKS: Simplified Kubernetes Management + +Amazon Elastic Kubernetes Service (EKS) provides a fully managed Kubernetes service that eliminates the complexity of operating Kubernetes clusters. With EKS, you can: -*Fully Managed Kubernetes* -Amazon EKS provides a scalable and highly-available Kubernetes control plane running across multiple {aws} Availability Zones (AZs). Amazon EKS automatically manages availability and scalability of Kubernetes API servers and etcd persistence layer. Amazon EKS runs the Kubernetes control plane across multiple AZs to ensure high availability, and automatically detects and replaces unhealthy control plane nodes. +* Deploy applications faster with less operational overhead +* Scale seamlessly to meet changing workload demands +* Improve security through {aws} integration and automated updates +* Choose between standard EKS or fully automated EKS Auto Mode -Amazon EKS Auto Mode fully automates Kubernetes cluster infrastructure management for compute, storage, and networking on {aws}. It simplifies Kubernetes management by automatically provisioning infrastructure, selecting optimal compute instances, dynamically scaling resources, continuously optimizing costs, patching operating systems, and integrating with {aws} security services. -*Kubernetes Compatibility and Support* -Amazon EKS runs upstream Kubernetes and is certified Kubernetes-conformant, so you can use all the existing plug-ins and tooling from the Kubernetes community. Applications running on Amazon EKS are fully compatible with applications running on any standard Kubernetes environment, whether running in on-premises data centers or public clouds. This means that you can easily migrate any standard Kubernetes application to Amazon EKS without refactoring your code. Amazon EKS supports Kubernetes versions longer than they are supported upstream, with standard support for Kubernetes minor versions for 14 months from the time they are released in Amazon EKS, and extended support for Kubernetes minor versions for an additional 12 months of support (26 total months per version). See <> for more information. +Amazon Elastic Kubernetes Service (Amazon EKS) is the premiere platform for running https://kubernetes.io/docs/concepts/overview/[Kubernetes] clusters, both in the Amazon Web Services ({aws}) cloud and in your own data centers (https://anywhere.eks.amazonaws.com/[EKS Anywhere] and <>). -*Machine Learning* +Amazon EKS simplifies building, securing, and maintaining Kubernetes clusters. It can be more cost effective at providing enough resources to meet peak demand than maintaining your own data centers. Two of the main approaches to using Amazon EKS are as follows: -Amazon EKS has become a cornerstone for deploying and managing AI/ML workloads in the cloud. With its ability to handle complex, resource-intensive tasks, Amazon EKS provides a scalable and flexible foundation for running AI/ML models, making it an ideal choice for organizations aiming to harness the full potential of machine learning. Whether you're training large language models that require vast amounts of compute power or deploying inference pipelines that need to handle unpredictable traffic patterns, Amazon EKS scales up and down efficiently, optimizing resource use and cost. Amazon EKS supports a wide range of compute options including GPU-powered instances and {aws} Neuron, allowing for high-performance training and low-latency inference, ensuring that models run efficiently in production environments. See the https://docs.aws.amazon.com/eks/latest/userguide/machine-learning-on-eks.html[Machine Learning on Amazon EKS Overview] for more information. +* **EKS standard**: {aws} manages the https://kubernetes.io/docs/concepts/overview/components/#control-plane-components[Kubernetes control plane] when you create a cluster with EKS. Components that manage nodes, schedule workloads, integrate with the {aws} cloud, and store and scale control plane information to keep your clusters up and running, are handled for you automatically. -*Hybrid Deployments* +* **EKS Auto Mode**: Using the <> feature, EKS extends its control to manage https://kubernetes.io/docs/concepts/overview/components/#node-components[Nodes] (Kubernetes data plane) as well. +It simplifies Kubernetes management by automatically provisioning infrastructure, selecting optimal compute instances, dynamically scaling resources, continuously optimizing costs, patching operating systems, and integrating with {aws} security services. -You can use the same Amazon EKS clusters to run nodes on {aws}-hosted infrastructure in {aws} https://aws.amazon.com/about-aws/global-infrastructure/regions_az/[Regions], https://aws.amazon.com/about-aws/global-infrastructure/localzones/[{aws} Local Zones], https://aws.amazon.com/wavelength/[{aws} Wavelength Zones], or in your own on-premises environments with https://aws.amazon.com/outposts/[{aws} Outposts] and link:eks/latest/userguide/hybrid-nodes-overview.html[Amazon EKS Hybrid Nodes,type="documentation"]. {aws} Outposts is {aws}-managed infrastructure that you run in your data centers or co-location facilities, whereas Amazon EKS Hybrid Nodes runs on virtual machines or bare metal infrastructure that you manage in your on-premises or edge environments. If you need to run in isolated or air-gapped environments, you can use https://aws.amazon.com/eks/eks-anywhere/[Amazon EKS Anywhere], which is {aws}-supported Kubernetes management software that runs on infrastructure you manage. With Amazon EKS Anywhere, you are responsible for cluster lifecycle operations and maintenance of your Amazon EKS Anywhere clusters. The _Amazon EKS Connector_ can be used to view any Kubernetes cluster and their resources in the Amazon EKS console. _Amazon EKS Distro_ is the {aws} distribution of the underlying Kubernetes components that power all Amazon EKS offerings. +The following diagram illustrates how Amazon EKS integrates your Kubernetes clusters with the {aws} cloud, depending on which method of cluster creation you choose: -*Compute* +image::images/whatis.png[Amazon EKS standard and EKS Auto Mode,scaledwidth=100%] + +Amazon EKS helps you accelerate time to production, improve performance, availability and resiliency, and enhance system security. +For more information, see https://aws.amazon.com/eks/[Amazon Elastic Kubernetes Service]. + +[#eks-features] +== Features of Amazon EKS +Amazon EKS provides the following high-level features: -You can use the full range of Amazon EC2 instance types and {aws} innovations such as Nitro and Graviton with Amazon EKS for you to optimize the compute for your workloads. You can use on-demand or Spot instances and your savings plans with compute you use with your Amazon EKS clusters. See <> for more information. +*Management interfaces*:: +EKS offers multiple interfaces to provision, manage, and maintain clusters, including {aws-management-console}, Amazon EKS API/SDKs, CDK, {aws} CLI, eksctl CLI, {aws} CloudFormation, and Terraform. +For more information, see <> and <>. -*Networking* +*Access control tools*:: +EKS relies on both Kubernetes and {aws} Identity and Access Management ({aws} IAM) features to <> +from users and workloads. +For more information, see <> and <>. -Amazon EKS integrates with Amazon VPC allowing you to use your own Amazon VPC security groups and link:vpc/latest/userguide/vpc-network-acls[network access control lists,type="documentation"] (ACLs) with Amazon EKS clusters. Amazon EKS provides the https://github.com/aws/amazon-vpc-cni-k8s[Amazon VPC container network interface] (CNI), allowing Kubernetes pods to receive IP addresses directly from the VPC. Amazon EKS supports IPv4 and IPv6 for workloads and dual-stack endpoints for the Amazon EKS APIs and Kubernetes API. You can use Application Load Balancers (ALB) and Network Load Balancers (NLB) managed by the {aws} Load Balancer Controller for application ingress and load balancing. You can also use Amazon VPC Lattice, a managed application networking service built directly into the {aws} networking infrastructure, for cross-cluster connectivity with standard Kubernetes semantics in a simple and consistent manner. See <> for more information. +*Compute resources*:: +For <>, EKS allows the full range of Amazon EC2 instance types and {aws} innovations such as Nitro and Graviton with Amazon EKS for you to optimize the compute for your workloads. For more information, see <>. -*Security* +*Storage*:: +EKS Auto Mode automatically creates storage classes using <>. +Using Container Storage Interface (CSI) drivers, you can also use Amazon S3, Amazon EFS, Amazon FSX, and Amazon File Cache for your application storage needs. For more information, see <>. -Amazon EKS integrates with {aws} Identity and Access Management (IAM) for you to secure your clusters and applications. Amazon EKS makes it easy to map {aws} IAM permissions to Kubernetes Role Based Access Control (RBAC). You can use {aws} IAM for cluster authentication and authorization with Amazon EKS Cluster Access Management, for access and permissions of operational software running on your clusters, and for granular application access to other {aws} services with Amazon EKS Pod Identity. Amazon EKS is certified by multiple compliance programs for regulated and sensitive applications. Amazon EKS is compliant with https://aws.amazon.com/compliance/soc-faqs/[SOC], https://aws.amazon.com/compliance/pci-dss-level-1-faqs/[PCI], https://aws.amazon.com/compliance/iso-certified/[ISO], https://aws.amazon.com/compliance/fedramp/[FedRAMP-Moderate], https://aws.amazon.com/compliance/irap/[IRAP], https://aws.amazon.com/compliance/bsi-c5/[C5], https://aws.amazon.com/compliance/k-isms/[K-ISMS], https://aws.amazon.com/compliance/esquema-nacional-de-seguridad/[ENS High], https://aws.amazon.com/compliance/OSPAR/[OSPAR], https://aws.amazon.com/compliance/hitrust/[HITRUST CSF], and is a https://aws.amazon.com/compliance/hipaa-compliance/[HIPAA] eligible service. See <> for more information. +*Security*:: +The shared responsibility model is employed as it relates to <>. +For more information, see <>, <>, +and <>. -*Observability* +*Monitoring tools*:: +Use the <> to monitor Amazon EKS clusters. +Monitoring tools include <>, <>, <>, +and <>. +For more information on dashboards, metrics servers, and other tools, see <> and <>. -Amazon EKS integrates with {aws} Managed Service for Prometheus (AMP), Amazon CloudWatch, Amazon CloudTrail, and Amazon GuardDuty for monitoring, logging, and auditing capabilities. You can also view performance insights for your Amazon EKS clusters directly in the Amazon EKS console. You can use AMP agent-less scrapers or the {aws} Distro for OpenTelemetry add-on to monitor and collect logs for your clusters, infrastructure, and applications. You can use Amazon CloudWatch Container Insights, the CloudWatch Observability Agent add-on, and Amazon EKS control plane logging to monitor, collect logs, and analyze issues with your clusters, infrastructure, and applications. Amazon EKS also integrates with Amazon CloudTrail for auditing cluster API activity, and Amazon GuardDuty for audit log threat analysis and runtime threat detection. See <> for more information. +*Kubernetes compatibility and support*:: -*Storage* +Amazon EKS is certified Kubernetes-conformant, so you can deploy Kubernetes-compatible applications without refactoring and use Kubernetes community tooling and plugins. +EKS offers both <> and <> for Kubernetes. +For more information, see <>. -You can use a range of {aws} storage services with Amazon EKS for the storage needs of your applications. Through an {aws}-supported breadth of Container Storage Interface (CSI) drivers, you can easily use Amazon EBS, Amazon S3, Amazon EFS, Amazon FSX, and Amazon File Cache for the storage needs of your applications running on Amazon EKS. See <> for more information. +[#eks-related-services] +== Related services +**Services to use with Amazon EKS** -*Add-ons* +You can use other {aws} services with the clusters that you deploy using Amazon EKS: -Amazon EKS offers a curated set of {aws}-vended Kubernetes software, also known as Amazon EKS add-ons, that provide key operational capabilities for Kubernetes clusters and integration with various {aws} services for cluster and pod networking, load balancing, storage, observability, and security. Amazon EKS provides a unified management experience for finding, selecting, installing, managing, and configuring third-party Kubernetes operational software (add-ons) from independent software vendors on Amazon EKS clusters. See <> for more information. +*Amazon EC2*:: +Obtain on-demand, scalable compute capacity with link:AWSEC2/latest/UserGuide/concepts.html[Amazon EC2,type="documentation"]. -*Management interfaces* +*Amazon EBS*:: +Attach scalable, high-performance block storage resources with link:ebs/latest/userguide/what-is-ebs.html[Amazon EBS,type="documentation"]. -Amazon EKS supports a range of interfaces to provision, manage, and maintain clusters including the Amazon EKS console, Amazon EKS API/SDKs, CDK, {aws} CLI, eksctl CLI, {aws} CloudFormation, and Terraform. You can also use {aws} Controllers for Kubernetes (ACK) to provision and manage {aws} services from within your Kubernetes environment using Kubernetes interfaces. ACK makes it simple to build scalable and highly available Kubernetes applications utilizing {aws} services. See <> for more information. +*Amazon ECR*:: +Store container images securely with link:AmazonECR/latest/userguide/what-is-ecr.html[Amazon ECR,type="documentation"]. -*Operating systems* +*Amazon CloudWatch*:: +Monitor {aws} resources and applications in real time with link:AmazonCloudWatch/latest/monitoring/WhatIsCloudWatch.html[Amazon CloudWatch,type="documentation"]. -Amazon EKS supports a range of operating systems and you can use pre-built, Amazon EKS-optimized Amazon Machine Images (AMIs) for the base images of your compute nodes. Amazon EKS maintains optimized images for Amazon Linux 2, Amazon Linux 2023, Bottlerocket, Windows, and there are Ubuntu images maintained by Canonical. You can also use your own custom AMIs for other operating system variants. The Amazon EKS AMIs for Amazon Linux have built-in support for NVIDIA and {aws} Neuron accelerated instance types. See <> for more information. +*Amazon Prometheus*:: +Track metrics for containerized applications with link:prometheus/latest/userguide/what-is-Amazon-Managed-Service-Prometheus.html[Amazon Managed Service for Prometheus,type="documentation"]. -[[eks-pricing,eks-pricing.title]] +*Elastic Load Balancing*:: +Distribute incoming traffic across multiple targets with link:elasticloadbalancing/latest/userguide/what-is-load-balancing.html[Elastic Load Balancing,type="documentation"]. + +*Amazon GuardDuty*:: +Detect threats to EKS clusters with <>. + +*{aws} Resilience Hub*:: +Assess EKS cluster resiliency with <>. + +[#eks-pricing] == Amazon EKS Pricing -Amazon EKS has per cluster pricing based on Kubernetes cluster version support, pricing for Amazon EKS Auto Mode, and per vCPU pricing for Amazon EKS Hybrid Nodes. When using Amazon EKS, you pay separately for the {aws} resources you use to run your applications on Kubernetes worker nodes. For example, if you are running Kubernetes worker nodes as Amazon EC2 instances with Amazon EBS volumes and public IPv4 addresses, you are charged for the instance capacity through Amazon EC2, the volume capacity through Amazon EBS, and the IPv4 address through Amazon VPC. Visit the respective pricing pages of the {aws} services you are using with your Kubernetes applications for detailed pricing information. +Amazon EKS has per cluster pricing based on Kubernetes cluster version support, pricing for Amazon EKS Auto Mode, and per vCPU pricing for Amazon EKS Hybrid Nodes. + +When using Amazon EKS, you pay separately for the {aws} resources you use to run your applications on Kubernetes worker nodes. For example, if you are running Kubernetes worker nodes as Amazon EC2 instances with Amazon EBS volumes and public IPv4 addresses, you are charged for the instance capacity through Amazon EC2, the volume capacity through Amazon EBS, and the IPv4 address through Amazon VPC. + +Visit the respective pricing pages of the {aws} services you are using with your Kubernetes applications for detailed pricing information. -* For Amazon EKS cluster, Amazon EKS Auto Mode, and Amazon EKS Hybrid Nodes pricing, see https://aws.amazon.com/eks/pricing/[Amazon EKS Pricing]. -* For Amazon EC2 pricing, see https://aws.amazon.com/ec2/pricing/on-demand/[Amazon EC2 On-Demand Pricing] and https://aws.amazon.com/ec2/spot/pricing/[Amazon EC2 Spot Pricing]. -* For {aws} Fargate pricing, see https://aws.amazon.com/fargate/pricing[{aws} Fargate Pricing]. -* You can use your savings plans for compute used in Amazon EKS clusters. For more information, see https://aws.amazon.com/savingsplans/pricing/[Pricing with Savings Plans]. +* For Amazon EKS cluster, Amazon EKS Auto Mode, and Amazon EKS Hybrid Nodes pricing, see link:eks/pricing/[Amazon EKS Pricing,type="marketing"]. +* For Amazon EC2 pricing, see link:ec2/pricing/on-demand/[Amazon EC2 On-Demand Pricing,type="marketing"] and link:ec2/spot/pricing/[Amazon EC2 Spot Pricing,type="marketing"]. +* For {aws} Fargate pricing, see link:fargate/pricing[{aws} Fargate Pricing,type="marketing"]. +* You can use your savings plans for compute used in Amazon EKS clusters. For more information, see link:savingsplans/pricing/[Pricing with Savings Plans,type="marketing"]. include::common-use-cases.adoc[leveloffset=+1] diff --git a/latest/ug/workloads/add-ons-iam.adoc b/latest/ug/workloads/add-ons-iam.adoc index 79e807814..95db03f00 100644 --- a/latest/ug/workloads/add-ons-iam.adoc +++ b/latest/ug/workloads/add-ons-iam.adoc @@ -1,19 +1,10 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[add-ons-iam,add-ons-iam.title]] +[#add-ons-iam] = IAM roles for Amazon EKS add-ons :info_titleabbrev: IAM roles -include::../attributes.txt[] - -include::retreive-iam-info.adoc[leveloffset=+1] - -include::update-addon-role.adoc[leveloffset=+1] - -include::remove-addon-role.adoc[leveloffset=+1] - -include::addon-id-troubleshoot.adoc[leveloffset=+1] - [abstract] -- Grant an Amazon EKS add-on permission to call {aws} APIs. Create a Pod Identity Association for an Amazon EKS add-on. @@ -29,9 +20,18 @@ Amazon EKS add-ons can help manage the life cycle of pod identity associations c . Determine if the add-on you want to install requires IAM permissions using the `describe-addon-versions` {aws} CLI operation. If the `requiresIamPermissions` flag is `true`, then you should use the `describe-addon-configurations` operation to determine the permissions needed by the addon. The response includes a list of suggested managed IAM policies. . Retrieve the name of the Kubernetes Service Account and the IAM policy using the `describe-addon-configuration` CLI operation. Evaluate the scope of the suggested policy against your security requirements. . Create an IAM role using the suggested permissions policy, and the trust policy required by Pod Identity. For more information, see <>. -. Create or update an Amazon EKS add-on using the CLI. Specify at least one pod identity association. A pod identity association is the name of a [.noloc]`Kubernetes` service account, and the ARN of the IAM role. +. Create or update an Amazon EKS add-on using the CLI. Specify at least one pod identity association. A pod identity association is the name of a Kubernetes service account, and the ARN of the IAM role. * Pod identity associations created using the add-on APIs are owned by the respective add-on. If you delete the add-on, the pod identity association is also deleted. You can prevent this cascading delete by using the `preserve` option when deleting an addon using the {aws} CLI or API. You also can directly update or delete the pod identity association if necessary. Add-ons can't assume ownership of existing pod identity associations. You must delete the existing association and re-create it using an add-on create or update operation. * Amazon EKS recommends using pod identity associations to manage IAM permissions for add-ons. The previous method, IAM roles for service accounts (IRSA), is still supported. You can specify both an IRSA `serviceAccountRoleArn` and a pod identity association for an add-on. If the EKS pod identity agent is installed on the cluster, the `serviceAccountRoleArn` will be ignored, and EKS will use the provided pod identity association. If Pod Identity is not enabled, the `serviceAccountRoleArn` will be used. * If you update the pod identity associations for an existing add-on, Amazon EKS initiates a rolling restart of the add-on pods. + +include::retreive-iam-info.adoc[leveloffset=+1] + +include::update-addon-role.adoc[leveloffset=+1] + +include::remove-addon-role.adoc[leveloffset=+1] + +include::addon-id-troubleshoot.adoc[leveloffset=+1] + diff --git a/latest/ug/workloads/add-ons-images.adoc b/latest/ug/workloads/add-ons-images.adoc index 001cfc823..1b35265fe 100644 --- a/latest/ug/workloads/add-ons-images.adoc +++ b/latest/ug/workloads/add-ons-images.adoc @@ -1,15 +1,16 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[add-ons-images,add-ons-images.title]] +[#add-ons-images] = View Amazon container image registries for Amazon EKS add-ons -:info_titleabbrev: View Amazon container image registries - -include::../attributes.txt[] +:info_titleabbrev: View Amazon image registries -When you deploy <> to your cluster, your nodes pull the required container images from the registry specified in the installation mechanism for the add-on, such as an installation manifest or a Helm `values.yaml` file. The images are pulled from an Amazon EKS Amazon ECR private repository. Amazon EKS replicates the images to a repository in each Amazon EKS supported {aws} Region. Your nodes can pull the container image over the internet from any of the following registries. Alternatively, your nodes can pull the image over Amazon's network if you created an link:AmazonECR/latest/userguide/vpc-endpoints.html[interface VPC endpoint for Amazon ECR ({aws} PrivateLink),type="documentation"] in your VPC. The registries require authentication with an {aws} IAM account. Your nodes authenticate using the <>, which has the permissions in the link:aws-managed-policy/latest/reference/AmazonEC2ContainerRegistryReadOnly.html[AmazonEC2ContainerRegistryReadOnly,type="documentation"] managed IAM policy associated to it. +When you deploy <> to your cluster, your nodes pull the required container images from the registry specified in the installation mechanism for the add-on, such as an installation manifest or a Helm `values.yaml` file. The images are pulled from an Amazon EKS Amazon ECR private repository. Amazon EKS replicates the images to a repository in each Amazon EKS supported {aws} Region. Your nodes can pull the container image over the internet from any of the following registries. Alternatively, your nodes can pull the image over Amazon's network if you created an link:AmazonECR/latest/userguide/vpc-endpoints.html[interface VPC endpoint for Amazon ECR ({aws} PrivateLink),type="documentation"] in your VPC. The registries require authentication with an {aws} IAM account. Your nodes authenticate using the <>, which has the permissions in the link:aws-managed-policy/latest/reference/AmazonEC2ContainerRegistryReadOnly.html[AmazonEC2ContainerRegistryReadOnly,type="documentation"] managed IAM policy associated to it. -[cols="1,1", options="header"] +[role="no-scroll"] +[%header,cols="2"] |=== + |{aws} Region |Registry @@ -20,20 +21,29 @@ When you deploy <> to |ap-east-1 |800184023465.dkr.ecr.ap-east-1.amazonaws.com -|ap-northeast-1 -|602401143452.dkr.ecr.ap-northeast-1.amazonaws.com +|ap-east-2 +|533267051163.dkr.ecr.ap-east-1.amazonaws.com -|ap-northeast-2 -|602401143452.dkr.ecr.ap-northeast-2.amazonaws.com +|ap-southeast-3 +|296578399912.dkr.ecr.ap-southeast-3.amazonaws.com -|ap-northeast-3 -|602401143452.dkr.ecr.ap-northeast-3.amazonaws.com +|ap-south-2 +|900889452093.dkr.ecr.ap-south-2.amazonaws.com + +|ap-southeast-4 +|491585149902.dkr.ecr.ap-southeast-4.amazonaws.com + +|ap-southeast-6 +|333609536671.dkr.ecr.ap-southeast-6.amazonaws.com |ap-south-1 |602401143452.dkr.ecr.ap-south-1.amazonaws.com -|ap-south-2 -|900889452093.dkr.ecr.ap-south-2.amazonaws.com +|ap-northeast-3 +|602401143452.dkr.ecr.ap-northeast-3.amazonaws.com + +|ap-northeast-2 +|602401143452.dkr.ecr.ap-northeast-2.amazonaws.com |ap-southeast-1 |602401143452.dkr.ecr.ap-southeast-1.amazonaws.com @@ -41,17 +51,11 @@ When you deploy <> to |ap-southeast-2 |602401143452.dkr.ecr.ap-southeast-2.amazonaws.com -|ap-southeast-3 -|296578399912.dkr.ecr.ap-southeast-3.amazonaws.com +|ap-southeast-7 +|121268973566.dkr.ecr.ap-southeast-7.amazonaws.com -|ap-southeast-4 -|491585149902.dkr.ecr.ap-southeast-4.amazonaws.com - -|ca-central-1 -|602401143452.dkr.ecr.ca-central-1.amazonaws.com - -|ca-west-1 -|761377655185.dkr.ecr.ca-west-1.amazonaws.com +|ap-northeast-1 +|602401143452.dkr.ecr.ap-northeast-1.amazonaws.com |cn-north-1 |918309763551.dkr.ecr.cn-north-1.amazonaws.com.cn @@ -62,54 +66,64 @@ When you deploy <> to |eu-central-1 |602401143452.dkr.ecr.eu-central-1.amazonaws.com -|eu-central-2 -|900612956339.dkr.ecr.eu-central-2.amazonaws.com +|eu-west-1 +|602401143452.dkr.ecr.eu-west-1.amazonaws.com -|eu-north-1 -|602401143452.dkr.ecr.eu-north-1.amazonaws.com +|eu-west-2 +|602401143452.dkr.ecr.eu-west-2.amazonaws.com |eu-south-1 |590381155156.dkr.ecr.eu-south-1.amazonaws.com +|eu-west-3 +|602401143452.dkr.ecr.eu-west-3.amazonaws.com + |eu-south-2 |455263428931.dkr.ecr.eu-south-2.amazonaws.com -|eu-west-1 -|602401143452.dkr.ecr.eu-west-1.amazonaws.com - -|eu-west-2 -|602401143452.dkr.ecr.eu-west-2.amazonaws.com +|eu-north-1 +|602401143452.dkr.ecr.eu-north-1.amazonaws.com -|eu-west-3 -|602401143452.dkr.ecr.eu-west-3.amazonaws.com +|eu-central-2 +|900612956339.dkr.ecr.eu-central-2.amazonaws.com |il-central-1 |066635153087.dkr.ecr.il-central-1.amazonaws.com +|mx-central-1 +|730335286997.dkr.ecr.mx-central-1.amazonaws.com + |me-south-1 |558608220178.dkr.ecr.me-south-1.amazonaws.com |me-central-1 |759879836304.dkr.ecr.me-central-1.amazonaws.com -|sa-east-1 -|602401143452.dkr.ecr.sa-east-1.amazonaws.com - |us-east-1 |602401143452.dkr.ecr.us-east-1.amazonaws.com |us-east-2 |602401143452.dkr.ecr.us-east-2.amazonaws.com +|us-west-1 +|602401143452.dkr.ecr.us-west-1.amazonaws.com + +|us-west-2 +|602401143452.dkr.ecr.us-west-2.amazonaws.com + +|ca-central-1 +|602401143452.dkr.ecr.ca-central-1.amazonaws.com + +|ca-west-1 +|761377655185.dkr.ecr.ca-west-1.amazonaws.com + +|sa-east-1 +|602401143452.dkr.ecr.sa-east-1.amazonaws.com + |us-gov-east-1 |151742754352.dkr.ecr.us-gov-east-1.amazonaws.com |us-gov-west-1 |013241004608.dkr.ecr.us-gov-west-1.amazonaws.com -|us-west-1 -|602401143452.dkr.ecr.us-west-1.amazonaws.com - -|us-west-2 -|602401143452.dkr.ecr.us-west-2.amazonaws.com |=== diff --git a/latest/ug/workloads/addon-compat.adoc b/latest/ug/workloads/addon-compat.adoc index 0c874a011..734cc0131 100644 --- a/latest/ug/workloads/addon-compat.adoc +++ b/latest/ug/workloads/addon-compat.adoc @@ -1,13 +1,10 @@ -//!!NODE_ROOT
+include::../attributes.txt[] [.topic] -[[addon-compat,addon-compat.title]] +[#addon-compat] = Verify Amazon EKS add-on version compatibility with a cluster :info_titleabbrev: Verify compatibility -include::../attributes.txt[] - - [abstract] -- Learn how to verify the Amazon EKS add-on compatibility with your cluster before you create or update an Amazon EKS add-on. @@ -15,7 +12,7 @@ Learn how to verify the Amazon EKS add-on compatibility with your cluster before Before you create an Amazon EKS add-on you need to verify that the Amazon EKS add-on version is compatible with your cluster. -Use the link:eks/latest/APIReference/API_DescribeAddonVersions.html[describe-addon-verisions API,type="documentation"] to list the available versions of EKS add-ons, and which Kubernetes versions each addon version supports. +Use the link:eks/latest/APIReference/API_DescribeAddonVersions.html[describe-addon-versions API,type="documentation"] to list the available versions of EKS add-ons, and which Kubernetes versions each addon version supports. . Verify the {aws} CLI is installed and working with `aws sts get-caller-identity`. If this command doesn't work, learn how to link:cli/latest/userguide/cli-chap-getting-started.html[Get started with the {aws} CLI.,type="documentation"] . Determine the name of the add-on you want to retrieve version compatibility information for, such as `amazon-cloudwatch-observability`. @@ -71,5 +68,3 @@ This output shows that addon version `vX.X.X-eksbuild.X` is compatible with Kube The `computeTypes` field in the `describe-addon-versions` output indicates an add-on's compatibility with EKS Auto Mode Managed Nodes or Hybrid Nodes. Add-ons marked `auto` work with EKS Auto Mode's cloud-based, {aws}-managed infrastructure, while those marked `hybrid` can run on on-premises nodes connected to the EKS cloud control plane. For more information, see <>. - - diff --git a/latest/ug/workloads/addon-id-troubleshoot.adoc b/latest/ug/workloads/addon-id-troubleshoot.adoc index 24686340f..4f2694371 100644 --- a/latest/ug/workloads/addon-id-troubleshoot.adoc +++ b/latest/ug/workloads/addon-id-troubleshoot.adoc @@ -1,10 +1,9 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[addon-id-troubleshoot,addon-id-troubleshoot.title]] +[#addon-id-troubleshoot] = Troubleshoot Pod Identities for EKS add-ons -:info_titleabbrev: Troubleshoot Pod Identities - -include::../attributes.txt[] +:info_titleabbrev: Troubleshoot Identities [abstract] -- @@ -40,3 +39,8 @@ aws iam get-role --role-name --query Role.AssumeRolePolicyDocument * The service account name in the pod identity association matches the service account name used by the add-on. + ** For information about the available add-ons, see <>. ++ +* Check configuration of MutatingWebhookConfiguration named `pod-identity-webhook` +** `admissionReviewVersions` of the webhook needs to be `v1beta1` and doesn't work with `v1`. + + diff --git a/latest/ug/workloads/alb-ingress.adoc b/latest/ug/workloads/alb-ingress.adoc index d097b58ef..c9d680d3f 100644 --- a/latest/ug/workloads/alb-ingress.adoc +++ b/latest/ug/workloads/alb-ingress.adoc @@ -1,13 +1,9 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[alb-ingress,alb-ingress.title]] -= Route application and [.noloc]`HTTP` traffic with [.noloc]`Application Load Balancers` -:info_doctype: section -:info_title: Route application and HTTP traffic with Application Load Balancers +[#alb-ingress] += Route application and HTTP traffic with Application Load Balancers :info_titleabbrev: Application load balancing -:info_abstract: Learn how to use Application Load Balancing on Amazon EKS to load balance application traffic at L7 with {aws} Load Balancer Controller. - -include::../attributes.txt[] [abstract] -- @@ -23,9 +19,9 @@ Learn how to use Application Load Balancing on Amazon EKS to load balance applic ==== -When you create a [.noloc]`Kubernetes` `ingress`, an {aws} Application Load Balancer (ALB) is provisioned that load balances application traffic. To learn more, see link:elasticloadbalancing/latest/application/introduction.html[What is an Application Load Balancer?,type="documentation"] in the _Application Load Balancers User Guide_ and https://kubernetes.io/docs/concepts/services-networking/ingress/[Ingress] in the [.noloc]`Kubernetes` documentation. ALBs can be used with [.noloc]`Pods` that are deployed to nodes or to {aws} Fargate. You can deploy an ALB to public or private subnets. +When you create a Kubernetes `ingress`, an {aws} Application Load Balancer (ALB) is provisioned that load balances application traffic. To learn more, see link:elasticloadbalancing/latest/application/introduction.html[What is an Application Load Balancer?,type="documentation"] in the _Application Load Balancers User Guide_ and https://kubernetes.io/docs/concepts/services-networking/ingress/[Ingress] in the Kubernetes documentation. ALBs can be used with Pods that are deployed to nodes or to {aws} Fargate. You can deploy an ALB to public or private subnets. -Application traffic is balanced at `L7` of the OSI model. To load balance network traffic at `L4`, you deploy a [.noloc]`Kubernetes` `service` of the `LoadBalancer` type. This type provisions an {aws} Network Load Balancer. For more information, see <>. To learn more about the differences between the two types of load balancing, see link:elasticloadbalancing/features/[Elastic Load Balancing features,type="marketing"] on the {aws} website. +Application traffic is balanced at `L7` of the OSI model. To load balance network traffic at `L4`, you deploy a Kubernetes `service` of the `LoadBalancer` type. This type provisions an {aws} Network Load Balancer. For more information, see <>. To learn more about the differences between the two types of load balancing, see link:elasticloadbalancing/features/[Elastic Load Balancing features,type="marketing"] on the {aws} website. == Prerequisites @@ -33,8 +29,8 @@ Application traffic is balanced at `L7` of the OSI model. To load balance networ Before you can load balance application traffic to an application, you must meet the following requirements. * Have an existing cluster. If you don't have an existing cluster, see <>. If you need to update the version of an existing cluster, see <>. -* Have the [.noloc]`{aws} Load Balancer Controller` deployed on your cluster. For more information, see <>. We recommend version `2.7.2` or later. -* At least two subnets in different Availability Zones. The [.noloc]`{aws} Load Balancer Controller` chooses one subnet from each Availability Zone. When multiple tagged subnets are found in an Availability Zone, the controller chooses the subnet whose subnet ID comes first lexicographically. Each subnet must have at least eight available IP addresses. +* Have the {aws} Load Balancer Controller deployed on your cluster. For more information, see <>. We recommend version `2.7.2` or later. +* At least two subnets in different Availability Zones. The {aws} Load Balancer Controller chooses one subnet from each Availability Zone. When multiple tagged subnets are found in an Availability Zone, the controller chooses the subnet whose subnet ID comes first lexicographically. Each subnet must have at least eight available IP addresses. + If you're using multiple security groups attached to worker node, exactly one security group must be tagged as follows. Replace [.replaceable]`my-cluster` with your cluster name. + @@ -42,23 +38,23 @@ If you're using multiple security groups attached to worker node, exactly one se – `kubernetes.io/cluster/` ** *Value* – `shared` or `owned` -* If you're using the [.noloc]`{aws} Load Balancer Controller` version `2.1.1` or earlier, subnets must be tagged in the format that follows. If you're using version `2.1.2` or later, tagging is optional. However, we recommend that you tag a subnet if any of the following is the case. You have multiple clusters that are running in the same VPC, or have multiple {aws} services that share subnets in a VPC. Or, you want more control over where load balancers are provisioned for each cluster. Replace [.replaceable]`my-cluster` with your cluster name. +* If you're using the {aws} Load Balancer Controller version `2.1.1` or earlier, subnets must be tagged in the format that follows. If you're using version `2.1.2` or later, tagging is optional. However, we recommend that you tag a subnet if any of the following is the case. You have multiple clusters that are running in the same VPC, or have multiple {aws} services that share subnets in a VPC. Or, you want more control over where load balancers are provisioned for each cluster. Replace [.replaceable]`my-cluster` with your cluster name. + ** *Key* – `kubernetes.io/cluster/` ** *Value* – `shared` or `owned` -* Your public and private subnets must meet the following requirements. This is unless you explicitly specify subnet IDs as an annotation on a service or ingress object. Assume that you provision load balancers by explicitly specifying subnet IDs as an annotation on a service or ingress object. In this situation, [.noloc]`Kubernetes` and the {aws} load balancer controller use those subnets directly to create the load balancer and the following tags aren't required. +* Your public and private subnets must meet the following requirements. This is unless you explicitly specify subnet IDs as an annotation on a service or ingress object. Assume that you provision load balancers by explicitly specifying subnet IDs as an annotation on a service or ingress object. In this situation, Kubernetes and the {aws} load balancer controller use those subnets directly to create the load balancer and the following tags aren't required. + ** *Private subnets* - – Must be tagged in the following format. This is so that [.noloc]`Kubernetes` and the {aws} load balancer controller know that the subnets can be used for internal load balancers. If you use `eksctl` or an Amazon EKS {aws} CloudFormation template to create your VPC after March 26, 2020, the subnets are tagged appropriately when created. For more information about the Amazon EKS {aws} CloudFormation VPC templates, see <>. + – Must be tagged in the following format. This is so that Kubernetes and the {aws} load balancer controller know that the subnets can be used for internal load balancers. If you use `eksctl` or an Amazon EKS {aws} CloudFormation template to create your VPC after March 26, 2020, the subnets are tagged appropriately when created. For more information about the Amazon EKS {aws} CloudFormation VPC templates, see <>. + *** *Key* – `kubernetes.io/role/internal-elb` *** *Value* – `1` ** *Public subnets* - – Must be tagged in the following format. This is so that [.noloc]`Kubernetes` knows to use only the subnets that were specified for external load balancers. This way, [.noloc]`Kubernetes` doesn't choose a public subnet in each Availability Zone (lexicographically based on their subnet ID). If you use `eksctl` or an Amazon EKS {aws} CloudFormation template to create your VPC after March 26, 2020, the subnets are tagged appropriately when created. For more information about the Amazon EKS {aws} CloudFormation VPC templates, see <>. + – Must be tagged in the following format. This is so that Kubernetes knows to use only the subnets that were specified for external load balancers. This way, Kubernetes doesn't choose a public subnet in each Availability Zone (lexicographically based on their subnet ID). If you use `eksctl` or an Amazon EKS {aws} CloudFormation template to create your VPC after March 26, 2020, the subnets are tagged appropriately when created. For more information about the Amazon EKS {aws} CloudFormation VPC templates, see <>. + *** *Key* – `kubernetes.io/role/elb` @@ -66,10 +62,10 @@ If you're using multiple security groups attached to worker node, exactly one se – `1` + -If the subnet role tags aren't explicitly added, the [.noloc]`Kubernetes` service controller examines the route table of your cluster VPC subnets. This is to determine if the subnet is private or public. We recommend that you don't rely on this behavior. Rather, explicitly add the private or public role tags. The [.noloc]`{aws} Load Balancer Controller` doesn't examine route tables. It also requires the private and public tags to be present for successful auto discovery. +If the subnet role tags aren't explicitly added, the Kubernetes service controller examines the route table of your cluster VPC subnets. This is to determine if the subnet is private or public. We recommend that you don't rely on this behavior. Rather, explicitly add the private or public role tags. The {aws} Load Balancer Controller doesn't examine route tables. It also requires the private and public tags to be present for successful auto discovery. -* The https://github.com/kubernetes-sigs/aws-load-balancer-controller[{aws} Load Balancer Controller] creates ALBs and the necessary supporting {aws} resources whenever a [.noloc]`Kubernetes` ingress resource is created on the cluster with the `kubernetes.io/ingress.class: alb` annotation. The ingress resource configures the ALB to route HTTP or HTTPS traffic to different [.noloc]`Pods` within the cluster. To ensure that your ingress objects use the [.noloc]`{aws} Load Balancer Controller`, add the following annotation to your [.noloc]`Kubernetes` ingress specification. For more information, see https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/ingress/spec/[Ingress specification] on [.noloc]`GitHub`. +* The https://github.com/kubernetes-sigs/aws-load-balancer-controller[{aws} Load Balancer Controller] creates ALBs and the necessary supporting {aws} resources whenever a Kubernetes ingress resource is created on the cluster with the `kubernetes.io/ingress.class: alb` annotation. The ingress resource configures the ALB to route HTTP or HTTPS traffic to different Pods within the cluster. To ensure that your ingress objects use the {aws} Load Balancer Controller, add the following annotation to your Kubernetes ingress specification. For more information, see https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/guide/ingress/spec/[Ingress specification] on GitHub. + [source,yaml,subs="verbatim,attributes"] ---- @@ -77,29 +73,29 @@ annotations: kubernetes.io/ingress.class: alb ---- + -NOTE: If you're load balancing to `IPv6` [.noloc]`Pods`, add the following annotation to your ingress spec. You can only load balance over `IPv6` to IP targets, not instance targets. Without this annotation, load balancing is over `IPv4`. - +NOTE: If you're load balancing to `IPv6` Pods, add the following annotation to your ingress spec. You can only load balance over `IPv6` to IP targets, not instance targets. Without this annotation, load balancing is over `IPv4`. ++ [source,yaml,subs="verbatim,attributes"] ---- alb.ingress.kubernetes.io/ip-address-type: dualstack ---- -* The [.noloc]`{aws} Load Balancer Controller` supports the following traffic modes: +* The {aws} Load Balancer Controller supports the following traffic modes: + ** *Instance* - – Registers nodes within your cluster as targets for the ALB. Traffic reaching the ALB is routed to `NodePort` for your service and then proxied to your [.noloc]`Pods`. This is the default traffic mode. You can also explicitly specify it with the `alb.ingress.kubernetes.io/target-type: instance` annotation. + – Registers nodes within your cluster as targets for the ALB. Traffic reaching the ALB is routed to `NodePort` for your service and then proxied to your Pods. This is the default traffic mode. You can also explicitly specify it with the `alb.ingress.kubernetes.io/target-type: instance` annotation. + -NOTE: Your [.noloc]`Kubernetes` service must specify the `NodePort` or "LoadBalancer" type to use this traffic mode. +NOTE: Your Kubernetes service must specify the `NodePort` or `LoadBalancer` type to use this traffic mode. ** *IP* - – Registers [.noloc]`Pods` as targets for the ALB. Traffic reaching the ALB is directly routed to [.noloc]`Pods` for your service. You must specify the `alb.ingress.kubernetes.io/target-type: ip` annotation to use this traffic mode. The IP target type is required when target [.noloc]`Pods` are running on Fargate or Amazon EKS Hybrid Nodes. -* To tag ALBs created by the controller, add the following annotation to the controller: `alb.ingress.kubernetes.io/tags`. For a list of all available annotations supported by the [.noloc]`{aws} Load Balancer Controller`, see https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/ingress/annotations/[Ingress annotations] on [.noloc]`GitHub`. -* Upgrading or downgrading the ALB controller version can introduce breaking changes for features that rely on it. For more information about the breaking changes that are introduced in each release, see the https://github.com/kubernetes-sigs/aws-load-balancer-controller/releases[ALB controller release notes] on [.noloc]`GitHub`. + – Registers Pods as targets for the ALB. Traffic reaching the ALB is directly routed to Pods for your service. You must specify the `alb.ingress.kubernetes.io/target-type: ip` annotation to use this traffic mode. The IP target type is required when target Pods are running on Fargate or Amazon EKS Hybrid Nodes. +* To tag ALBs created by the controller, add the following annotation to the controller: `alb.ingress.kubernetes.io/tags`. For a list of all available annotations supported by the {aws} Load Balancer Controller, see https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/guide/ingress/annotations/[Ingress annotations] on GitHub. +* Upgrading or downgrading the ALB controller version can introduce breaking changes for features that rely on it. For more information about the breaking changes that are introduced in each release, see the https://github.com/kubernetes-sigs/aws-load-balancer-controller/releases[ALB controller release notes] on GitHub. == Reuse ALBs with Ingress Groups You can share an application load balancer across multiple service resources using `IngressGroups`. -To join an ingress to a group, add the following annotation to a [.noloc]`Kubernetes` ingress resource specification. +To join an ingress to a group, add the following annotation to a Kubernetes ingress resource specification. [source,yaml,subs="verbatim,attributes"] ---- @@ -121,7 +117,7 @@ The controller automatically merges ingress rules for all ingresses in the same *Potential security risk* -Specify an ingress group for an ingress only when all the [.noloc]`Kubernetes` users that have RBAC permission to create or modify ingress resources are within the same trust boundary. If you add the annotation with a group name, other [.noloc]`Kubernetes` users might create or modify their ingresses to belong to the same ingress group. Doing so can cause undesirable behavior, such as overwriting existing rules with higher priority rules. +Specify an ingress group for an ingress only when all the Kubernetes users that have RBAC permission to create or modify ingress resources are within the same trust boundary. If you add the annotation with a group name, other Kubernetes users might create or modify their ingresses to belong to the same ingress group. Doing so can cause undesirable behavior, such as overwriting existing rules with higher priority rules. ==== @@ -141,15 +137,15 @@ Ensure that each ingress in the same ingress group has a unique priority number. ==== -[[application-load-balancer-sample-application,application-load-balancer-sample-application.title]] +[#application-load-balancer-sample-application] == (Optional) Deploy a sample application * At least one public or private subnet in your cluster VPC. -* Have the [.noloc]`{aws} Load Balancer Controller` deployed on your cluster. For more information, see <>. We recommend version `2.7.2` or later. +* Have the {aws} Load Balancer Controller deployed on your cluster. For more information, see <>. We recommend version `2.7.2` or later. -You can run the sample application on a cluster that has Amazon EC2 nodes, Fargate [.noloc]`Pods`, or both. +You can run the sample application on a cluster that has Amazon EC2 nodes, Fargate Pods, or both. -. If you're not deploying to Fargate, skip this step. If you're deploying to Fargate, create a Fargate profile. You can create the profile by running the following command or in the <> using the same values for `name` and `namespace` that are in the command. Replace the [.replaceable]`example values` with your own. +. If you're not deploying to Fargate, skip this step. If you're deploying to Fargate, create a Fargate profile. You can create the profile by running the following command or in the <> using the same values for `name` and `namespace` that are in the command. Replace the example values with your own. + [source,bash,subs="verbatim,attributes"] ---- @@ -159,16 +155,16 @@ eksctl create fargateprofile \ --name alb-sample-app \ --namespace game-2048 ---- -. Deploy the game https://play2048.co/[2048] as a sample application to verify that the [.noloc]`{aws} Load Balancer Controller` creates an {aws} ALB as a result of the ingress object. Complete the steps for the type of subnet you're deploying to. +. Deploy the game https://play2048.co/[2048] as a sample application to verify that the {aws} Load Balancer Controller creates an {aws} ALB as a result of the ingress object. Complete the steps for the type of subnet you're deploying to. + -.. If you're deploying to [.noloc]`Pods` in a cluster that you created with the `IPv6` family, skip to the next step. +.. If you're deploying to Pods in a cluster that you created with the `IPv6` family, skip to the next step. + *** *Public*:: + [source,bash,subs="verbatim,attributes"] ---- -kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.11.0/docs/examples/2048/2048_full.yaml +kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.14.1/docs/examples/2048/2048_full.yaml ---- *** *Private*:: @@ -177,7 +173,7 @@ kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/aws-load-bala + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.11.0/docs/examples/2048/2048_full.yaml +curl -O https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.14.1/docs/examples/2048/2048_full.yaml ---- .... Edit the file and find the line that says `alb.ingress.kubernetes.io/scheme: internet-facing`. .... Change [.replaceable]`internet-facing` to `internal` and save the file. @@ -187,13 +183,13 @@ curl -O https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-cont ---- kubectl apply -f 2048_full.yaml ---- -.. If you're deploying to [.noloc]`Pods` in a cluster that you created with the <>, complete the following steps. +.. If you're deploying to Pods in a cluster that you created with the <>, complete the following steps. + ... Download the manifest. + [source,bash,subs="verbatim,attributes"] ---- -curl -O https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.11.0/docs/examples/2048/2048_full.yaml +curl -O https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.14.1/docs/examples/2048/2048_full.yaml ---- ... Open the file in an editor and add the following line to the annotations in the ingress spec. + @@ -201,7 +197,7 @@ curl -O https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-cont ---- alb.ingress.kubernetes.io/ip-address-type: dualstack ---- -... If you're load balancing to internal [.noloc]`Pods`, rather than internet facing [.noloc]`Pods`, change the line that says `alb.ingress.kubernetes.io/scheme: [.replaceable]``internet-facing``` to `alb.ingress.kubernetes.io/scheme: internal` +... If you're load balancing to internal Pods, rather than internet facing Pods, change the line that says `alb.ingress.kubernetes.io/scheme: [.replaceable]``internet-facing``` to `alb.ingress.kubernetes.io/scheme: internal` ... Save the file. ... Apply the manifest to your cluster. + @@ -226,7 +222,7 @@ ingress-2048 * k8s-game2048-ingress2-xxxxxxxxxx-yyyyyyyyyy.regi + NOTE: If you created the load balancer in a private subnet, the value under `ADDRESS` in the previous output is prefaced with `internal-`. -If your ingress wasn't successfully created after several minutes, run the following command to view the [.noloc]`{aws} Load Balancer Controller` logs. These logs might contain error messages that you can use to diagnose issues with your deployment. +If your ingress wasn't successfully created after several minutes, run the following command to view the {aws} Load Balancer Controller logs. These logs might contain error messages that you can use to diagnose issues with your deployment. [source,bash,subs="verbatim,attributes"] ---- @@ -241,7 +237,7 @@ image::images/2048.png[2048 sample application,scaledwidth=100%] + [source,bash,subs="verbatim,attributes"] ---- -kubectl delete -f https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.11.0/docs/examples/2048/2048_full.yaml +kubectl delete -f https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.14.1/docs/examples/2048/2048_full.yaml ---- ** If you downloaded and edited the manifest, use the following command. + diff --git a/latest/ug/workloads/community-addons.adoc b/latest/ug/workloads/community-addons.adoc index 6172eae11..2199c6b3a 100644 --- a/latest/ug/workloads/community-addons.adoc +++ b/latest/ug/workloads/community-addons.adoc @@ -1,10 +1,8 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[community-addons,community-addons.title]] +[#community-addons] = Community add-ons -:info_doctype: section - -include::../attributes.txt[] You can use {aws} APIs to install community add-ons, such as the Kubernetes Metrics Server. You may choose to install community add-ons as Amazon EKS Add-ons to reduce the complexity of maintaining the software on multiple clusters. @@ -12,7 +10,11 @@ For example, you can use the {aws} API, CLI, or Management Console to install co You manage community add-ons just like existing Amazon EKS Add-ons. Community add-ons are different from existing add-ons in that they have a unique scope of support. -Community add-ons are built and validated by {aws}. Importantly, {aws} does not provide full support for community add-ons. {aws} supports only lifecycle operations done using {aws} APIs, such as installing add-ons or deleting add-ons. +NOTE: Using community add-ons is at your discretion. As part of the xref:security[shared responsibility model] between you and {aws}, you are expected to understand what you are installing into your cluster for these third party plugins. You are also responsible for the community add-ons meeting your cluster security needs. For more information, see <>. + +Community add-ons are not built by {aws}. {aws} only validates community add-ons for version compatibility. For example, if you install a community add-on on a cluster, {aws} checks if it is compatible with the Kubernetes version of your cluster. + +Importantly, {aws} does not provide full support for community add-ons. {aws} supports only lifecycle operations done using {aws} APIs, such as installing add-ons or deleting add-ons. If you require support for a community add-on, utilize the existing project resources. For example, you may create a GitHub issue on the repo for the project. @@ -51,26 +53,230 @@ You install or update community add-ons in the same way as other Amazon EKS Add- == Available community add-ons -The following community add-ons are availalbe from Amazon EKS. +The following community add-ons are available from Amazon EKS. -=== [.noloc]`Kubernetes Metrics Server` +* <> +* <> +* <> +* <> +* <> +* <> + +[#kubernetes-metrics-server] +=== Kubernetes Metrics Server The Kubernetes Metrics Server is a scalable and efficient source of container resource metrics for Kubernetes built-in autoscaling pipelines. It collects resource metrics from Kubelets and exposes them in Kubernetes apiserver through Metrics API for use by Horizontal Pod Autoscaler and Vertical Pod Autoscaler. +[%header,cols="2"] |=== -|Property | Value +|Property +|Value + + +|Add-on name +|`metrics-server` + +|Namespace +|`kube-system` + +|Documentation +|https://github.com/kubernetes-sigs/metrics-server[GitHub Readme] -|Add-on name | `metrics-server` +|Service account name +|None -|Namespace | `kube-system` +|Managed IAM policy +|None -|Documentation | https://github.com/kubernetes-sigs/metrics-server[GitHub Readme] +|Custom IAM permissions +|None -|Service account name | None +|=== + + +[#kube-state-metrics] +=== kube-state-metrics -|Managed IAM policy | None +Add-on agent to generate and expose cluster-level metrics. -|Custom IAM permissions | None +The state of Kubernetes objects in the Kubernetes API can be exposed as metrics. An add-on agent called kube-state-metrics can connect to the Kubernetes API server and expose a HTTP endpoint with metrics generated from the state of individual objects in the cluster. It exposes various information about the state of objects like labels and annotations, startup and termination times, status or the phase the object currently is in. + +[%header,cols="2"] |=== +|Property +|Value + + +|Add-on name +|`kube-state-metrics` + +|Namespace +|`kube-state-metrics` + +|Documentation +|https://kubernetes.io/docs/concepts/cluster-administration/kube-state-metrics/[Metrics for Kubernetes Object States] in Kubernetes Docs + +|Service account name +|None + +|Managed IAM policy +|None + +|Custom IAM permissions +|None + +|=== + + +[#prometheus-node-exporter] +=== Prometheus Node Exporter + +Prometheus exporter for hardware and OS metrics exposed by *NIX kernels, written in Go with pluggable metric collectors. The Prometheus Node Exporter exposes a wide variety of hardware- and kernel-related metrics. + +[%header,cols="2"] +|=== +|Property +|Value + + +|Add-on name +|`prometheus-node-exporter` + +|Namespace +|`prometheus-node-exporter` + +|Documentation +|https://prometheus.io/docs/guides/node-exporter/#monitoring-linux-host-metrics-with-the-node-exporter[Monitoring Linux host metrics with the Node Exporter] in Prometheus Docs + +|Service account name +|None + +|Managed IAM policy +|None + +|Custom IAM permissions +|None + +|=== + +[#addon-cert-manager] +=== Cert Manager + +Cert Manager can be used to manage the creation and renewal of certificates. + +[%header,cols="2"] +|=== +|Property +|Value + +|Add-on name +|`cert-manager` + +|Namespace +|`cert-manager` + +|Documentation +|https://cert-manager.io/docs/[Cert Manager Docs] + +|Service account name +|None + +|Managed IAM policy +|None + +|Custom IAM permissions +|None + +|=== + + +[#external-dns] +=== External DNS + +The External DNS EKS add-on can be used to manage Route53 DNS records through Kubernetes resources. + +External DNS permissions can be reduced to `route53:ChangeResourceRecordSets`, `route53:ListHostedZones`, and `route53:ListResourceRecordSets` on the hosted zones you wish to manage. + +[%header,cols="2"] +|=== +|Property +|Value + + +|Add-on name +|`external-dns` + +|Namespace +|`external-dns` + +|Documentation +|https://github.com/kubernetes-sigs/external-dns[GitHub Readme] + +|Service account name +|`external-dns` + +|Managed IAM policy +|`{arn-aws}iam::aws:policy/AmazonRoute53FullAccess` + +|Custom IAM permissions +|None + +|=== + + + +[#fluent-bit] +=== Fluent Bit + +Fluent Bit is a lightweight and high-performance log processor and forwarder. It allows you to collect data/logs from different sources, unify them, and send them to multiple destinations including Amazon CloudWatch Logs, Amazon S3, and Amazon Kinesis Data Firehose. Fluent Bit is designed with performance and resource efficiency in mind, making it ideal for Kubernetes environments. + +This add-on does not require IAM permissions in the default configuration. However, you may need to grant this add-on IAM permissions if you configure an {aws} output location. For more information, see <>. + +[%header,cols="2"] +|=== +|Property +|Value + + +|Add-on name +|`fluent-bit` + +|Namespace +|`fluent-bit` + +|Documentation +|https://docs.fluentbit.io/manual/[Fluent Bit Documentation] + +|Service account name +|`fluent-bit` + +|Managed IAM policy +|None + +|Custom IAM permissions +|None + +|=== + + +== View Attributions + +You can download the open source attributions and license information for community add-ons. + +. Determine the name and version of the add-on you want to download attributions for. +. Update the following command with the name and version: ++ +[source,cli] +---- +curl -O https://amazon-eks-docs.s3.amazonaws.com/attributions///attributions.zip +---- ++ +For example: ++ +[source,cli] +---- +curl -O https://amazon-eks-docs.s3.amazonaws.com/attributions/kube-state-metrics/v2.14.0-eksbuild.1/attributions.zip +---- +. Use the command to download the file. -link:samples/attributions-md.zip[View license attributions for this add-on. ] +Use this zip file to view information about the license attributions. diff --git a/latest/ug/workloads/copy-image-to-repository.adoc b/latest/ug/workloads/copy-image-to-repository.adoc index 2e05fd705..8ac421b91 100644 --- a/latest/ug/workloads/copy-image-to-repository.adoc +++ b/latest/ug/workloads/copy-image-to-repository.adoc @@ -1,23 +1,19 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[copy-image-to-repository,copy-image-to-repository.title]] +[#copy-image-to-repository] = Copy a container image from one repository to another repository -:info_doctype: section -:info_title: Copy a container image from one repository to \ - another repository :info_titleabbrev: Copy an image to a repository This topic describes how to pull a container image from a repository that your nodes don't have access to and push the image to a repository that your nodes have access to. You can push the image to Amazon ECR or an alternative repository that your nodes have access to. -* The [.noloc]`Docker` engine installed and configured on your computer. For instructions, see https://docs.docker.com/engine/install/[Install Docker Engine] in the [.noloc]`Docker` documentation. -* Version `2.12.3` or later or version `1.27.160` or later of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device or {aws} CloudShell. To check your current version, use `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or [.noloc]`Homebrew` for [.noloc]`macOS` are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the _{aws} Command Line Interface User Guide_. The {aws} CLI version that is installed in {aws} CloudShell might also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the _{aws} CloudShell User Guide_. -* An interface VPC endpoint for Amazon ECR if you want your nodes to pull container images from or push container images to a private Amazon ECR repository over Amazon's network. For more information, see link:AmazonECR/latest/userguide/vpc-endpoints.html#ecr-setting-up-vpc-create[Create the VPC endpoints for Amazon ECR,type="documentation"] in the Amazon Elastic Container Registry User Guide. +* The Docker engine installed and configured on your computer. For instructions, see https://docs.docker.com/engine/install/[Install Docker Engine] in the Docker documentation. +* Version `2.12.3` or later or version `1.27.160` or later of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device or {aws} CloudShell. To check your current version, use `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or Homebrew for macOS are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the _{aws} Command Line Interface User Guide_. The {aws} CLI version that is installed in {aws} CloudShell might also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the _{aws} CloudShell User Guide_. +* An interface VPC endpoint for Amazon ECR if you want your nodes to pull container images from or push container images to a private Amazon ECR repository over Amazon's network. For more information, see link:AmazonECR/latest/userguide/vpc-endpoints.html#ecr-setting-up-vpc-create[Create the VPC endpoints for Amazon ECR,type="documentation"] in the Amazon Elastic Container Registry User Guide. -Complete the following steps to pull a container image from a repository and push it to your own repository. In the following examples that are provided in this topic, the image for the https://github.com/aws/amazon-vpc-cni-k8s/blob/master/cmd/cni-metrics-helper/README.md[Amazon VPC CNI plugin for Kubernetes metrics helper] is pulled. When you follow these steps, make sure to replace the [.replaceable]`example values` with your own values. +Complete the following steps to pull a container image from a repository and push it to your own repository. In the following examples that are provided in this topic, the image for the https://github.com/aws/amazon-vpc-cni-k8s/blob/master/cmd/cni-metrics-helper/README.md[Amazon VPC CNI plugin for Kubernetes metrics helper] is pulled. When you follow these steps, make sure to replace the example values with your own values. . If you don't already have an Amazon ECR repository or another repository, then create one that your nodes have access to. The following command creates an Amazon ECR private repository. An Amazon ECR private repository name must start with a letter. It can only contain lowercase letters, numbers, hyphens (-), underscores (_), and forward slashes (/). For more information, see link:AmazonECR/latest/userguide/repository-create.html[Creating a private repository,type="documentation"] in the Amazon Elastic Container Registry User Guide. + @@ -40,10 +36,10 @@ image: "602401143452.dkr.ecr.us-west-2.amazonaws.com/cni-metrics-helper:v1.12.6" + You may see the following variations for an image location: + -** Only `repository-name:tag`. In this case, `docker.io` is usually the registry, but not specified since [.noloc]`Kubernetes` prepends it to a repository name by default if no registry is specified. +** Only `repository-name:tag`. In this case, `docker.io` is usually the registry, but not specified since Kubernetes prepends it to a repository name by default if no registry is specified. ** `repository-name/repository-namespace/repository:tag`. A repository namespace is optional, but is sometimes specified by the repository owner for categorizing images. For example, all https://gallery.ecr.aws/aws-ec2/[Amazon EC2 images in the Amazon ECR Public Gallery] use the `aws-ec2` namespace. + -Before installing an image with Helm, view the Helm `values.yaml` file to determine the image location. For example, the https://github.com/aws/amazon-vpc-cni-k8s/blob/master/charts/cni-metrics-helper/values.yaml#L5-L9[values.yaml] file for the https://github.com/aws/amazon-vpc-cni-k8s/blob/master/cmd/cni-metrics-helper/README.md[Amazon VPC CNI plugin for Kubernetes metrics helper] includes the following lines. +Before installing an image with Helm, view the Helm `values.yaml` file to determine the image location. For example, the https://github.com/aws/amazon-vpc-cni-k8s/blob/master/charts/cni-metrics-helper/values.yaml#L5-L9[values.yaml] file for the https://github.com/aws/amazon-vpc-cni-k8s/blob/master/cmd/cni-metrics-helper/README.md[Amazon VPC CNI plugin for Kubernetes metrics helper] includes the following lines. + [source,yaml,subs="verbatim,attributes"] ---- @@ -56,7 +52,7 @@ image: . Pull the container image specified in the manifest file. + [loweralpha] -.. If you're pulling from a public registry, such as the https://gallery.ecr.aws/[Amazon ECR Public Gallery], you can skip to the next sub-step, because authentication isn't required. In this example, you authenticate to an Amazon ECR private registry that contains the repository for the CNI metrics helper image. Amazon EKS maintains the image in each registry listed in <>. You can authenticate to any of the registries by replacing [.replaceable]`602401143452` and [.replaceable]`region-code` with the information for a different registry. A separate registry exists for each link:general/latest/gr/eks.html#eks_region[{aws} Region that Amazon EKS is supported in,type="documentation"]. +.. If you're pulling from a public registry, such as the https://gallery.ecr.aws/[Amazon ECR Public Gallery], you can skip to the next sub-step, because authentication isn't required. In this example, you authenticate to an Amazon ECR private registry that contains the repository for the CNI metrics helper image. Amazon EKS maintains the image in each registry listed in <>. You can authenticate to any of the registries by replacing [.replaceable]`602401143452` and [.replaceable]`region-code` with the information for a different registry. A separate registry exists for each link:general/latest/gr/eks.html#eks_region[{aws} Region that Amazon EKS is supported in,type="documentation"]. + [source,bash,subs="verbatim,attributes"] ---- @@ -74,7 +70,7 @@ docker pull 602401143452.dkr.ecr.region-code.amazonaws.com/cni-metrics-helper:v1 ---- docker tag cni-metrics-helper:v1.12.6 111122223333.dkr.ecr.region-code.amazonaws.com/cni-metrics-helper:v1.12.6 ---- -. Authenticate to your registry. In this example, you authenticate to the Amazon ECR private registry that you created in the first step. For more information, see link:AmazonECR/latest/userguide/Registries.html#registry_auth[Registry authentication,type="documentation"] in the Amazon Elastic Container Registry User Guide. +. Authenticate to your registry. In this example, you authenticate to the Amazon ECR private registry that you created in the first step. For more information, see link:AmazonECR/latest/userguide/Registries.html#registry_auth[Registry authentication,type="documentation"] in the Amazon Elastic Container Registry User Guide. + [source,bash,subs="verbatim,attributes"] ---- @@ -86,4 +82,4 @@ aws ecr get-login-password --region region-code | docker login --username {aws} ---- docker push 111122223333.dkr.ecr.region-code.amazonaws.com/cni-metrics-helper:v1.12.6 ---- -. Update the manifest file that you used to determine the image in a previous step with the `registry/repository:tag` for the image that you pushed. If you're installing with a Helm chart, there's often an option to specify the `registry/repository:tag`. When installing the chart, specify the `registry/repository:tag` for the image that you pushed to your repository. +. Update the manifest file that you used to determine the image in a previous step with the `registry/repository:tag` for the image that you pushed. If you're installing with a Helm chart, there's often an option to specify the `registry/repository:tag`. When installing the chart, specify the `registry/repository:tag` for the image that you pushed to your repository. \ No newline at end of file diff --git a/latest/ug/workloads/creating-an-add-on.adoc b/latest/ug/workloads/creating-an-add-on.adoc index 67171b934..4863d9d92 100644 --- a/latest/ug/workloads/creating-an-add-on.adoc +++ b/latest/ug/workloads/creating-an-add-on.adoc @@ -1,13 +1,10 @@ -//!!NODE_ROOT
+include::../attributes.txt[] [.topic] -[[creating-an-add-on,creating-an-add-on.title]] +[#creating-an-add-on] = Create an Amazon EKS add-on :info_titleabbrev: Create an add-on -include::../attributes.txt[] - - [abstract] -- Learn how to create an add-on for your Amazon EKS cluster. @@ -23,7 +20,7 @@ Amazon EKS add-ons are add-on software for Amazon EKS clusters. All Amazon EKS a You can create an Amazon EKS add-on using `eksctl`, the {aws-management-console}, or the {aws} CLI. If the add-on requires an IAM role, see the details for the specific add-on in <> for details about creating the role. -[[creating-an-add-on-prereq,creating-an-add-on-prereq.title]] +[#creating-an-add-on-prereq] == Prerequisites Complete the following before you create an add-on: @@ -36,7 +33,7 @@ Complete the following before you create an add-on: * Verify that version 0.190.0 or later of the `eksctl` command line tool installed on your computer or {aws} CloudShell. For more information, see https://eksctl.io/installation/[Installation] on the `eksctl` website. -[[creating-an-add-on-procedure,creating-an-add-on-procedure.title]] +[#creating-an-add-on-procedure] == Procedure You can create an Amazon EKS add-on using `eksctl`, the {aws-management-console}, or the {aws} CLI. If the add-on requires an IAM role, see the details for the specific add-on in <> for details about creating the role. @@ -45,11 +42,11 @@ You can create an Amazon EKS add-on using `eksctl`, the {aws-management-console} == Create add-on (eksctl) -. View the names of add-ons available for a cluster version. Replace [.replaceable]`1.30` with the version of your cluster. +. View the names of add-ons available for a cluster version. Replace [.replaceable]`{k8s-n}` with the version of your cluster. + [source,bash,subs="verbatim,attributes"] ---- -eksctl utils describe-addon-versions --kubernetes-version 1.30 | grep AddonName +eksctl utils describe-addon-versions --kubernetes-version {k8s-n} | grep AddonName ---- + An example output is as follows. @@ -67,11 +64,11 @@ An example output is as follows. "AddonName": "factorhouse_kpow", [...] ---- -. View the versions available for the add-on that you would like to create. Replace [.replaceable]`1.30` with the version of your cluster. Replace [.replaceable]`name-of-addon` with the name of the add-on you want to view the versions for. The name must be one of the names returned in the previous step. +. View the versions available for the add-on that you would like to create. Replace [.replaceable]`{k8s-n}` with the version of your cluster. Replace [.replaceable]`name-of-addon` with the name of the add-on you want to view the versions for. The name must be one of the names returned in the previous step. + [source,bash,subs="verbatim,attributes"] ---- -eksctl utils describe-addon-versions --kubernetes-version 1.30 --name name-of-addon | grep AddonVersion +eksctl utils describe-addon-versions --kubernetes-version {k8s-n} --name name-of-addon | grep AddonVersion ---- + The following output is an example of what is returned for the add-on named `vpc-cni`. You can see that the add-on has several available versions. @@ -88,7 +85,7 @@ The following output is an example of what is returned for the add-on named `vpc + [source,bash,subs="verbatim,attributes"] ---- -eksctl utils describe-addon-versions --kubernetes-version 1.30 --name name-of-addon | grep ProductUrl +eksctl utils describe-addon-versions --kubernetes-version {k8s-n} --name name-of-addon | grep ProductUrl ---- + If no output is returned, then the add-on is an Amazon EKS. If output is returned, then the add-on is an {aws} Marketplace add-on. The following output is for an add-on named `teleport_teleport`. @@ -98,13 +95,13 @@ If no output is returned, then the add-on is an Amazon EKS. If output is returne "ProductUrl": "https://aws.amazon.com/marketplace/pp?sku=3bda70bb-566f-4976-806c-f96faef18b26" ---- + -You can learn more about the add-on in the {aws} Marketplace with the returned URL. If the add-on requires a subscription, you can subscribe to the add-on through the {aws} Marketplace. If you're going to create an add-on from the {aws} Marketplace, then the link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] that you're using to create the add-on must have permission to create the link:license-manager/latest/userguide/license-manager-role-core.html[AWSServiceRoleForAWSLicenseManagerRole,type="documentation"] service-linked role. For more information about assigning permissions to an IAM entity, see link:IAM/latest/UserGuide/access_policies_manage-attach-detach.html[Adding and removing IAM identity permissions,type="documentation"] in the IAM User Guide. +You can learn more about the add-on in the {aws} Marketplace with the returned URL. If the add-on requires a subscription, you can subscribe to the add-on through the {aws} Marketplace. If you're going to create an add-on from the {aws} Marketplace, then the link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] that you're using to create the add-on must have permission to create the link:license-manager/latest/userguide/license-manager-role-core.html[AWSServiceRoleForAWSLicenseManagerRole,type="documentation"] service-linked role. For more information about assigning permissions to an IAM entity, see link:IAM/latest/UserGuide/access_policies_manage-attach-detach.html[Adding and removing IAM identity permissions,type="documentation"] in the IAM User Guide. . Create an Amazon EKS add-on. Copy the command and replace the [.replaceable]`user-data` as follows: + ** Replace [.replaceable]`my-cluster` with the name of your cluster. ** Replace [.replaceable]`name-of-addon` with the name of the add-on that you want to create. ** If you want a version of the add-on that's earlier than the latest version, then replace [.replaceable]`latest` with the version number returned in the output of a previous step that you want to use. -** If the add-on uses a service account role, replace [.replaceable]`111122223333` with your account ID and replace [.replaceable]`role-name` with the name of the role. For instructions on creating a role for your service account, see the documentation for the add-on that you're creating. For a list of add-ons, see <>. Specifying a service account role requires that you have an IAM [.noloc]`OpenID Connect` ([.noloc]`OIDC`) provider for your cluster. To determine whether you have one for your cluster, or to create one, see <>. +** If the add-on uses a service account role, replace [.replaceable]`111122223333` with your account ID and replace [.replaceable]`role-name` with the name of the role. For instructions on creating a role for your service account, see the documentation for the add-on that you're creating. For a list of add-ons, see <>. Specifying a service account role requires that you have an IAM OpenID Connect (OIDC) provider for your cluster. To determine whether you have one for your cluster, or to create one, see <>. + If the add-on doesn't use a service account role, delete `--service-account-role-arn{arn-aws}iam::111122223333:role/role-name`. ** This example command overwrites the configuration of any existing self-managed version of the add-on, if there is one. If you don't want to overwrite the configuration of an existing self-managed add-on, remove the [.replaceable]`--force` option. If you remove the option, and the Amazon EKS add-on needs to overwrite the configuration of an existing self-managed add-on, then creation of the Amazon EKS add-on fails with an error message to help you resolve the conflict. Before specifying this option, make sure that the Amazon EKS add-on doesn't manage settings that you need to manage, because those settings are overwritten with this option. @@ -124,7 +121,7 @@ eksctl create addon --help + For more information about available options see https://eksctl.io/usage/addons/[Addons] in the `eksctl` documentation. -[[_create_add_on_console, _create_add_on_console.title]] +[#_create_add_on_console] == Create add-on ({aws} Console) . Open the link:eks/home#/clusters[Amazon EKS console,type="console"]. @@ -132,48 +129,50 @@ For more information about available options see https://eksctl.io/usage/addons/ . Choose the name of the cluster that you want to create the add-on for. . Choose the *Add-ons* tab. . Choose *Get more add-ons*. -. On the *Select add-ons* page, choose the add-ons that you want to add to your cluster. You can add as many *Amazon EKS add-ons* and *{aws} Marketplace add-ons* as you require. +. On the *Select add-ons* page, choose the add-ons that you want to add to your cluster. You can add as many *Amazon EKS add-ons* and *{aws} Marketplace add-ons* as you require. + -For *{aws} Marketplace* add-ons the link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] that you're using to create the add-on must have permissions to read entitlements for the add-on from the {aws} LicenseManager. {aws} LicenseManager requires link:license-manager/latest/userguide/license-manager-role-core.html[AWSServiceRoleForAWSLicenseManagerRole,type="documentation"] service-linked role (SLR) that allows {aws} resources to manage licenses on your behalf. The SLR is a one time requirement, per account, and you will not have to create separate SLR's for each add-on nor each cluster. For more information about assigning permissions to an link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] see link:IAM/latest/UserGuide/access_policies_manage-attach-detach.html[Adding and removing IAM identity permissions,type="documentation"] in the IAM User Guide. +For *{aws} Marketplace* add-ons the link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] that you're using to create the add-on must have permissions to read entitlements for the add-on from the {aws} LicenseManager. {aws} LicenseManager requires link:license-manager/latest/userguide/license-manager-role-core.html[AWSServiceRoleForAWSLicenseManagerRole,type="documentation"] service-linked role (SLR) that allows {aws} resources to manage licenses on your behalf. The SLR is a one time requirement, per account, and you will not have to create separate SLR's for each add-on nor each cluster. For more information about assigning permissions to an link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] see link:IAM/latest/UserGuide/access_policies_manage-attach-detach.html[Adding and removing IAM identity permissions,type="documentation"] in the IAM User Guide. + -If the *{aws} Marketplace add-ons* that you want to install aren't listed, you can click the page numbering to view additional page results or search in the search box. In the *Filtering options*, you can also filter by *category*, *vendor*, or *pricing model* and then choose the add-ons from the search results. Once you've selected the add-ons that you want to install, choose *Next*. +If the *{aws} Marketplace add-ons* that you want to install aren't listed, you can click the page numbering to view additional page results or search in the search box. In the *Filtering options*, you can also filter by *category*, *vendor*, or *pricing model* and then choose the add-ons from the search results. Once you've selected the add-ons that you want to install, choose *Next*. . On the *Configure selected add-ons settings* page, do the following: + -.. Choose *View subscription options* to open the *Subscription options* form. Review the *Pricing details* and *Legal* sections, then choose the *Subscribe* button to continue. -.. For *Version*, choose the version that you want to install. We recommend the version marked *latest*, unless the individual add-on that you're creating recommends a different version. To determine whether an add-on has a recommended version, see the documentation for the add-on that you're creating. For a list of add-ons, see <>. -.. You have two options for configuring roles for add-ons: EKS Pod Identities IAM role and IAM roles for service accounts (IRSA). Follow the appropriate step below for your preferred option. If all of the add-ons that you selected have *Requires subscription* under *Status*, choose *Next*. You can't <> further until you've subscribed to them after your cluster is created. For the add-ons that don't have *Requires subscription* under *Status*, do the following: +.. Choose *View subscription options* to open the *Subscription options* form. Review the *Pricing details* and *Legal* sections, then choose the *Subscribe* button to continue. +.. For *Version*, choose the version that you want to install. We recommend the version marked *latest*, unless the individual add-on that you're creating recommends a different version. To determine whether an add-on has a recommended version, see the documentation for the add-on that you're creating. For a list of add-ons, see <>. +.. You have two options for configuring roles for add-ons: EKS Pod Identities IAM role and IAM roles for service accounts (IRSA). Follow the appropriate step below for your preferred option. If all of the add-ons that you selected have *Requires subscription* under *Status*, choose *Next*. You can't <> further until you've subscribed to them after your cluster is created. For the add-ons that don't have *Requires subscription* under *Status*, do the following: + ... For *Pod Identity IAM role for service account*, you can either use an existing EKS Pod Identity IAM role or create one using the *Create Recommended Role* button. This field will only provide options with the appropriate trust policy. If there's no role to select, then you don't have an existing role with a matching trust policy. To configure an EKS Pod Identity IAM role for service accounts of the selected add-on, choose *Create recommended role*. The role creation wizard opens in a separate window. The wizard will automatically populate the role information as follows. For each add-on where you want to create the EKS Pod Identity IAM role, complete the steps in the IAM wizard as follows. * On the *Select trusted entity* step, the {aws} service option for *EKS* and the use case for *EKS - Pod Identity* are preselected, and the appropriate trust policy will be automatically populated for the add-on. For example, the role will be created with the appropriate trust policy containing the pods.eks.amazonaws.com IAM Principal as detailed in <>. Choose *Next*. -* On the *Add permissions* step, the appropriate managed policy for the role policy is preselected for the add-on. For example, for the Amazon VPC CNI add-on, the role will be created with the managed policy ` AmazonEKS_CNI_Policy` as detailed in <>. Choose *Next*. +* On the *Add permissions* step, the appropriate managed policy for the role policy is preselected for the add-on. For example, for the Amazon VPC CNI add-on, the role will be created with the managed policy `AmazonEKS_CNI_Policy` as detailed in <>. Choose *Next*. * On the *Name, review, and create* step, in *Role name*, the default role name is automatically populated for the add-on. For example, for the *Amazon VPC CNI* add-on, the role will be created with the name *AmazonEKSPodIdentityAmazonVPCCNIRole*. In *Description*, the default description is automatically populated with the appropriate description for the add-on. For example, for the Amazon VPC CNI add-on, the role will be created with the description *Allows pods running in Amazon EKS cluster* to access {aws} resources. In *Trust policy*, view the populated trust policy for the add-on. Choose *Create role*. + NOTE: Retaining the default role name enables EKS to pre-select the role for add-ons in new clusters or when adding add-ons to existing clusters. You can still override this name and the role will be available for the add-on across your clusters, but the role will need to be manually selected from the drop down. -... For add-ons that do not have *Requires subscription* under *Status* and where you want to configure roles using IRSA, see the documentation for the add-on that you're creating to create an IAM policy and attach it to a role. For a list of add-ons, see <>. Selecting an IAM role requires that you have an IAM [.noloc]`OpenID Connect` ([.noloc]`OIDC`) provider for your cluster. To determine whether you have one for your cluster, or to create one, see <>. +... For add-ons that do not have *Requires subscription* under *Status* and where you want to configure roles using IRSA, see the documentation for the add-on that you're creating to create an IAM policy and attach it to a role. For a list of add-ons, see <>. Selecting an IAM role requires that you have an IAM OpenID Connect (OIDC) provider for your cluster. To determine whether you have one for your cluster, or to create one, see <>. ... Choose *Optional configuration settings*. ... If the add-on requires configuration, enter it in the *Configuration values* box. To determine whether the add-on requires configuration information, see the documentation for the add-on that you're creating. For a list of add-ons, see <>. -... Choose one of the available options for *Conflict resolution method*. If you choose *Override* for the *Conflict resolution method*, one or more of the settings for the existing add-on can be overwritten with the Amazon EKS add-on settings. If you don't enable this option and there's a conflict with your existing settings, the operation fails. You can use the resulting error message to troubleshoot the conflict. Before choosing this option, make sure that the Amazon EKS add-on doesn't manage settings that you need to self-manage. +... Choose one of the available options for *Conflict resolution method*. If you choose *Override* for the *Conflict resolution method*, one or more of the settings for the existing add-on can be overwritten with the Amazon EKS add-on settings. If you don't enable this option and there's a conflict with your existing settings, the operation fails. You can use the resulting error message to troubleshoot the conflict. Before choosing this option, make sure that the Amazon EKS add-on doesn't manage settings that you need to self-manage. +... If you want to install the add-on into a specific namespace, enter it in the *Namespace* field. For {aws} and community add-ons, you can define a custom Kubernetes namespace to install the add-on into. For more information, see <>. ... Choose *Next*. -. On the *Review and add* page, choose *Create*. After the add-on installation is complete, you see your installed add-ons. +. On the *Review and add* page, choose *Create*. After the add-on installation is complete, you see your installed add-ons. . If any of the add-ons that you installed require a subscription, complete the following steps: + -.. Choose the *Subscribe* button in the lower right corner for the add-on. You're taken to the page for the add-on in the {aws} Marketplace. Read the information about the add-on such as its *Product Overview* and *Pricing Information*. +.. Choose the *Subscribe* button in the lower right corner for the add-on. You're taken to the page for the add-on in the {aws} Marketplace. Read the information about the add-on such as its *Product Overview* and *Pricing Information*. .. Select the *Continue to Subscribe* button on the top right of the add-on page. -.. Read through the *Terms and Conditions*. If you agree to them, choose *Accept Terms*. It may take several minutes to process the subscription. While the subscription is processing, the *Return to Amazon EKS Console* button is grayed out. -.. Once the subscription has finished processing, the *Return to Amazon EKS Console* button is no longer grayed out. Choose the button to go back to the Amazon EKS console *Add-ons* tab for your cluster. -.. For the add-on that you subscribed to, choose *Remove and reinstall* and then choose *Reinstall add-on*. Installation of the add-on can take several minutes. When Installation is complete, you can configure the add-on. +.. Read through the *Terms and Conditions*. If you agree to them, choose *Accept Terms*. It may take several minutes to process the subscription. While the subscription is processing, the *Return to Amazon EKS Console* button is grayed out. +.. Once the subscription has finished processing, the *Return to Amazon EKS Console* button is no longer grayed out. Choose the button to go back to the Amazon EKS console *Add-ons* tab for your cluster. +.. For the add-on that you subscribed to, choose *Remove and reinstall* and then choose *Reinstall add-on*. Installation of the add-on can take several minutes. When Installation is complete, you can configure the add-on. == Create add-on ({aws} CLI) -. You need version `2.12.3` or later or version `1.27.160` or later of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device or {aws} CloudShell. To check your current version, use `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or [.noloc]`Homebrew` for [.noloc]`macOS` are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the _{aws} Command Line Interface User Guide_. The {aws} CLI version that is installed in {aws} CloudShell might also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the _{aws} CloudShell User Guide_. + +. You need version `2.12.3` or later or version `1.27.160` or later of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device or {aws} CloudShell. To check your current version, use `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or Homebrew for macOS are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the _{aws} Command Line Interface User Guide_. The {aws} CLI version that is installed in {aws} CloudShell might also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the _{aws} CloudShell User Guide_. + -. Determine which add-ons are available. You can see all available add-ons, their type, and their publisher. You can also see the URL for add-ons that are available through the {aws} Marketplace. Replace [.replaceable]`1.30` with the version of your cluster. +. Determine which add-ons are available. You can see all available add-ons, their type, and their publisher. You can also see the URL for add-ons that are available through the {aws} Marketplace. Replace [.replaceable]`{k8s-n}` with the version of your cluster. + [source,bash,subs="verbatim,attributes"] ---- -aws eks describe-addon-versions --kubernetes-version 1.30 \ +aws eks describe-addon-versions --kubernetes-version {k8s-n} \ --query 'addons[].{MarketplaceProductUrl: marketplaceInformation.productUrl, Name: addonName, Owner: owner Publisher: publisher, Type: type}' --output table ---- + @@ -200,11 +199,11 @@ An example output is as follows. ---- + Your output might be different. In this example output, there are three different add-ons available of type `networking` and five add-ons with a publisher of type `eks`. The add-ons with `aws-marketplace` in the `Owner` column may require a subscription before you can install them. You can visit the URL to learn more about the add-on and to subscribe to it. -. You can see which versions are available for each add-on. Replace [.replaceable]`1.30` with the version of your cluster and replace [.replaceable]`vpc-cni` with the name of an add-on returned in the previous step. +. You can see which versions are available for each add-on. Replace [.replaceable]`{k8s-n}` with the version of your cluster and replace [.replaceable]`vpc-cni` with the name of an add-on returned in the previous step. + [source,bash,subs="verbatim,attributes"] ---- -aws eks describe-addon-versions --kubernetes-version 1.30 --addon-name vpc-cni \ +aws eks describe-addon-versions --kubernetes-version {k8s-n} --addon-name vpc-cni \ --query 'addons[].addonVersions[].{Version: addonVersion, Defaultversion: compatibilities[0].defaultVersion}' --output table ---- + @@ -275,6 +274,7 @@ Here is an example of valid configuration values, in YAML format, that works wit ** Replace [.replaceable]`my-cluster` with the name of your cluster. ** Replace [.replaceable]`vpc-cni` with an add-on name returned in the output of the previous step that you want to create. ** Replace [.replaceable]`version-number` with the version returned in the output of the previous step that you want to use. +** If you want to install the add-on into a custom Kubernetes namespace, add the `--namespace-config 'namespace=` option. This option is only available for {aws} and community add-ons. For more information, see <> ** If the add-on doesn't require IAM permissions, delete [.replaceable]``. ** Do one of the following: + @@ -284,7 +284,7 @@ Here is an example of valid configuration values, in YAML format, that works wit ---- --pod-identity-associations 'serviceAccount=,roleArn=' ---- -*** If the add-on (1) requires IAM permissions, and (2) your cluster uses IRSA, replace [.replaceable]`` with the following IRSA configuration. Replace [.replaceable]`111122223333` with your account ID and [.replaceable]`role-name` with the name of an existing IAM role that you've created. For instructions on creating the role, see the documentation for the add-on that you're creating. For a list of add-ons, see <>. Specifying a service account role requires that you have an IAM [.noloc]`OpenID Connect` ([.noloc]`OIDC`) provider for your cluster. To determine whether you have one for your cluster, or to create one, see <>. +*** If the add-on (1) requires IAM permissions, and (2) your cluster uses IRSA, replace [.replaceable]`` with the following IRSA configuration. Replace [.replaceable]`111122223333` with your account ID and [.replaceable]`role-name` with the name of an existing IAM role that you've created. For instructions on creating the role, see the documentation for the add-on that you're creating. For a list of add-ons, see <>. Specifying a service account role requires that you have an IAM OpenID Connect (OIDC) provider for your cluster. To determine whether you have one for your cluster, or to create one, see <>. + [source,bash,subs="verbatim,attributes"] ---- @@ -327,4 +327,4 @@ For a full list of available options, see `link:cli/latest/reference/eks/create- ---- + -If you receive an error similar to the error in the previous output, visit the URL in the output of a previous step to subscribe to the add-on. Once subscribed, run the `create-addon` command again. +If you receive an error similar to the error in the previous output, visit the URL in the output of a previous step to subscribe to the add-on. Once subscribed, run the `create-addon` command again. \ No newline at end of file diff --git a/latest/ug/workloads/eks-add-ons.adoc b/latest/ug/workloads/eks-add-ons.adoc index dc97b1b77..bb5d992a8 100644 --- a/latest/ug/workloads/eks-add-ons.adoc +++ b/latest/ug/workloads/eks-add-ons.adoc @@ -1,69 +1,70 @@ -//!!NODE_ROOT
+include::../attributes.txt[] [.topic] -[[eks-add-ons,eks-add-ons.title]] +[#eks-add-ons] = Amazon EKS add-ons -:info_doctype: section -:info_title: Amazon EKS add-ons :info_titleabbrev: Amazon EKS add-ons -:keywords: managed, add-ons, plugins -:info_abstract: Learn how to manage operational software add-ons on Amazon EKS clusters with Amazon EKS add-ons for observability, networking, storage, and security from {aws} and third-party vendors. - -include::../attributes.txt[] [abstract] -- Learn how to manage operational software add-ons on Amazon EKS clusters with Amazon EKS add-ons for observability, networking, storage, and security from {aws} and third-party vendors. -- -An add-on is software that provides supporting operational capabilities to [.noloc]`Kubernetes` applications, but is not specific to the application. This includes software like observability agents or [.noloc]`Kubernetes` drivers that allow the cluster to interact with underlying {aws} resources for networking, compute, and storage. Add-on software is typically built and maintained by the [.noloc]`Kubernetes` community, cloud providers like {aws}, or third-party vendors. Amazon EKS automatically installs self-managed add-ons such as the [.noloc]`Amazon VPC CNI plugin for Kubernetes`, `kube-proxy`, and [.noloc]`CoreDNS` for every cluster. Note that the VPC CNI add-on isn't compatible with Amazon EKS Hybrid Nodes and doesn't deploy to hybrid nodes. You can change the default configuration of the add-ons and update them when desired. +An add-on is software that provides supporting operational capabilities to Kubernetes applications, but is not specific to the application. This includes software like observability agents or Kubernetes drivers that allow the cluster to interact with underlying {aws} resources for networking, compute, and storage. Add-on software is typically built and maintained by the Kubernetes community, cloud providers like {aws}, or third-party vendors. Amazon EKS automatically installs self-managed add-ons such as the Amazon VPC CNI plugin for Kubernetes, `kube-proxy`, and CoreDNS for every cluster. Note that the VPC CNI add-on isn't compatible with Amazon EKS Hybrid Nodes and doesn't deploy to hybrid nodes. You can change the default configuration of the add-ons and update them when desired. Amazon EKS add-ons provide installation and management of a curated set of add-ons for Amazon EKS clusters. All Amazon EKS add-ons include the latest security patches, bug fixes, and are validated by {aws} to work with Amazon EKS. Amazon EKS add-ons allow you to consistently ensure that your Amazon EKS clusters are secure and stable and reduce the amount of work that you need to do in order to install, configure, and update add-ons. If a self-managed add-on, such as `kube-proxy` is already running on your cluster and is available as an Amazon EKS add-on, then you can install the `kube-proxy` Amazon EKS add-on to start benefiting from the capabilities of Amazon EKS add-ons. -You can update specific Amazon EKS managed configuration fields for Amazon EKS add-ons through the Amazon EKS API. You can also modify configuration fields not managed by Amazon EKS directly within the [.noloc]`Kubernetes` cluster once the add-on starts. This includes defining specific configuration fields for an add-on where applicable. These changes are not overridden by Amazon EKS once they are made. This is made possible using the [.noloc]`Kubernetes` server-side apply feature. For more information, see <>. +You can update specific Amazon EKS managed configuration fields for Amazon EKS add-ons through the Amazon EKS API. You can also modify configuration fields not managed by Amazon EKS directly within the Kubernetes cluster once the add-on starts. This includes defining specific configuration fields for an add-on where applicable. These changes are not overridden by Amazon EKS once they are made. This is made possible using the Kubernetes server-side apply feature. For more information, see <>. You can use Amazon EKS add-ons with any Amazon EKS node type. For more information, see <>. You can add, update, or delete Amazon EKS add-ons using the Amazon EKS API, {aws-management-console}, {aws} CLI, and `eksctl`. You can also create Amazon EKS add-ons using link:AWSCloudFormation/latest/UserGuide/aws-resource-eks-addon.html[{aws} CloudFormation,type="documentation"]. -[[eks-add-ons-considerations,eks-add-ons-considerations.title]] +[#eks-add-ons-considerations] == Considerations Consider the following when you use Amazon EKS add-ons: -* To configure add-ons for the cluster your link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] must have IAM permissions to work with add-ons. For more information, see the actions with `Addon` in their name in link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html#amazonelastickubernetesservice-actions-as-permissions[Actions defined by Amazon Elastic Kubernetes Service,type="documentation"]. +* To configure add-ons for the cluster your link:IAM/latest/UserGuide/id_roles.html#iam-term-principal[IAM principal,type="documentation"] must have IAM permissions to work with add-ons. For more information, see the actions with `Addon` in their name in link:service-authorization/latest/reference/list_amazonelastickubernetesservice.html#amazonelastickubernetesservice-actions-as-permissions[Actions defined by Amazon Elastic Kubernetes Service,type="documentation"]. * Amazon EKS add-ons run on the nodes that you provision or configure for your cluster. Node types include Amazon EC2 instances, Fargate, and hybrid nodes. * You can modify fields that aren't managed by Amazon EKS to customize the installation of an Amazon EKS add-on. For more information, see <>. -* If you create a cluster with the {aws-management-console}, the Amazon EKS `kube-proxy`, [.noloc]`Amazon VPC CNI plugin for Kubernetes`, and [.noloc]`CoreDNS` Amazon EKS add-ons are automatically added to your cluster. If you use `eksctl` to create your cluster with a `config` file, `eksctl` can also create the cluster with Amazon EKS add-ons. If you create your cluster using `eksctl` without a `config` file or with any other tool, the self-managed `kube-proxy`, [.noloc]`Amazon VPC CNI plugin for Kubernetes`, and [.noloc]`CoreDNS` add-ons are installed, rather than the Amazon EKS add-ons. You can either manage them yourself or add the Amazon EKS add-ons manually after cluster creation. Regardless of the method that you use to create your cluster, the VPC CNI add-on doesn't install on hybrid nodes. -* The `eks:addon-cluster-admin` `ClusterRoleBinding` binds the `cluster-admin` `ClusterRole` to the `eks:addon-manager` [.noloc]`Kubernetes` identity. The role has the necessary permissions for the `eks:addon-manager` identity to create [.noloc]`Kubernetes` namespaces and install add-ons into namespaces. If the `eks:addon-cluster-admin` `ClusterRoleBinding` is removed, the Amazon EKS cluster will continue to function, however Amazon EKS is no longer able to manage any add-ons. All clusters starting with the following platform versions use the new `ClusterRoleBinding`. +* If you create a cluster with the {aws-management-console}, the Amazon EKS `kube-proxy`, Amazon VPC CNI plugin for Kubernetes, and CoreDNS Amazon EKS add-ons are automatically added to your cluster. If you use `eksctl` to create your cluster with a `config` file, `eksctl` can also create the cluster with Amazon EKS add-ons. If you create your cluster using `eksctl` without a `config` file or with any other tool, the self-managed `kube-proxy`, Amazon VPC CNI plugin for Kubernetes, and CoreDNS add-ons are installed, rather than the Amazon EKS add-ons. You can either manage them yourself or add the Amazon EKS add-ons manually after cluster creation. Regardless of the method that you use to create your cluster, the VPC CNI add-on doesn't install on hybrid nodes. +* The `eks:addon-cluster-admin` `ClusterRoleBinding` binds the `cluster-admin` `ClusterRole` to the `eks:addon-manager` Kubernetes identity. The role has the necessary permissions for the `eks:addon-manager` identity to create Kubernetes namespaces and install add-ons into namespaces. If the `eks:addon-cluster-admin` `ClusterRoleBinding` is removed, the Amazon EKS cluster will continue to function, however Amazon EKS is no longer able to manage any add-ons. All clusters starting with the following platform versions use the new `ClusterRoleBinding`. * A subset of EKS add-ons from {aws} have been validated for compatibility with Amazon EKS Hybrid Nodes. For more information, see the compatibility table on <>. -+ -[cols="1,1", options="header"] -|=== -|Kubernetes version -|EKS platform version +[#custom-namespace] +== Custom namespace for add-ons -|1.20 -|eks.12 +For community and {aws} add-ons, you can optionally specify a custom namespace during add-on creation. Once you install an add-on in a specific namespace, you must remove and re-create the add-on to change its namespace. -|1.21 -|eks.14 +If you don't specify a namespace, it will use the predefined namespace for the add-on. -|1.22 -|eks.9 +Use custom namespaces for better organization and isolation of add-on objects within your EKS cluster. This flexibility helps you align add-ons with your operational needs and existing namespace strategy. -|1.23 -|eks.5 +You can set a custom namespace when creating an add-on. For more information, see <>. -|1.24 -|eks.3 +=== Get predefined namespace for add-on -|=== +The predefined namespace for an add-on is the namespace it will be installed into if you don't specify one. + +To get the predefined namespace for an add-on, use the following command: + +```bash +aws eks describe-addon-versions --addon-name --query "addons[].defaultNamespace" +``` -[[addon-consider-auto,addon-consider-auto.title]] +Example output: + +```json +[ + "kube-system" +] +``` + + +[#addon-consider-auto] == Considerations for Amazon EKS Auto Mode Amazon EKS Auto mode includes capabilities that deliver essential cluster functionality, including: @@ -90,7 +91,7 @@ However, if your cluster combines Auto mode with other compute options like self If you are planning to enable EKS Auto Mode on an existing cluster, you may need to upgrade the version of certain addons. For more information, see <> for EKS Auto Mode. -[[addon-support,addon-support.title]] +[#addon-support] == Support {aws} publishes multiple types of add-ons with different levels of support. @@ -103,12 +104,13 @@ If you are planning to enable EKS Auto Mode on an existing cluster, you may need ** For more information, see <>. * *Community Add-ons*: These add-ons are scanned by {aws} but supported by the open source community. ** Use a community add-on to reduce the complexity of installing common open source software, such as Kubernetes Metrics Server. +** Community add-ons are packaged from source by {aws}. {aws} only validates community add-ons for version compatibility. ** For more information, see <>. The following table details the scope of support for each add-on type: -[cols="1,1,1,1,1", options="header"] +[%header,cols="5"] |=== |Category |Feature |{aws} add-ons |{aws} Marketplace add-ons |Community add-ons |Development @@ -120,7 +122,7 @@ The following table details the scope of support for each add-on type: |Validated by {aws} |Yes |No -|Yes +|Yes* |Development |Validated by {aws} Partner |No @@ -168,6 +170,8 @@ The following table details the scope of support for each add-on type: |No |=== +`*`: Validation for community add-ons only includes Kubernetes version compatibility. For example, if you install a community add-on on a cluster, {aws} checks if it is compatible with the Kubernetes version of your cluster. + {aws} Marketplace add-ons can download additional software dependencies from external sources outside of {aws}. These external dependencies are not scanned or validated by {aws}. Consider your security requirements when deploying {aws} Marketplace add-ons that fetch external dependencies. include::workloads-add-ons-available-eks.adoc[leveloffset=+1] @@ -187,4 +191,3 @@ include::removing-an-addon.adoc[leveloffset=+1] include::add-ons-iam.adoc[leveloffset=+1] include::kubernetes-field-management.adoc[leveloffset=+1] - diff --git a/latest/ug/workloads/eks-workloads.adoc b/latest/ug/workloads/eks-workloads.adoc index 4a7e965a2..e43f6b339 100644 --- a/latest/ug/workloads/eks-workloads.adoc +++ b/latest/ug/workloads/eks-workloads.adoc @@ -1,29 +1,19 @@ -//!!NODE_ROOT include::../attributes.txt[] -[[eks-workloads,eks-workloads.title]] + +[#eks-workloads] = Learn how to deploy workloads and add-ons to Amazon EKS -:doctype: book -:sectnums: -:toc: left -:icons: font -:experimental: -:idprefix: -:idseparator: - -:sourcedir: . -:info_doctype: chapter -:info_title: Learn how to deploy workloads and add-ons to Amazon EKS :info_titleabbrev: Workloads -Your workloads are deployed in containers, which are deployed in [.noloc]`Pods` in [.noloc]`Kubernetes`. A [.noloc]`Pod` includes one or more containers. Typically, one or more [.noloc]`Pods` that provide the same service are deployed in a [.noloc]`Kubernetes` service. Once you've deployed multiple [.noloc]`Pods` that provide the same service, you can: +Your workloads are deployed in containers, which are deployed in Pods in Kubernetes. A Pod includes one or more containers. Typically, one or more Pods that provide the same service are deployed in a Kubernetes service. Once you've deployed multiple Pods that provide the same service, you can: * <> running on each of your clusters using the {aws-management-console}. -* Vertically scale [.noloc]`Pods` up or down with the [.noloc]`Kubernetes` <>. -* Horizontally scale the number of [.noloc]`Pods` needed to meet demand up or down with the [.noloc]`Kubernetes` <>. -* Create an external (for internet-accessible [.noloc]`Pods`) or an internal (for private [.noloc]`Pods`) <> to balance network traffic across [.noloc]`Pods`. The load balancer routes traffic at Layer 4 of the OSI model. -* Create an <> to balance application traffic across [.noloc]`Pods`. The application load balancer routes traffic at Layer 7 of the OSI model. -* If you're new to [.noloc]`Kubernetes`, this topic helps you <>. +* Vertically scale Pods up or down with the Kubernetes <>. +* Horizontally scale the number of Pods needed to meet demand up or down with the Kubernetes <>. +* Create an external (for internet-accessible Pods) or an internal (for private Pods) <> to balance network traffic across Pods. The load balancer routes traffic at Layer 4 of the OSI model. +* Create an <> to balance application traffic across Pods. The application load balancer routes traffic at Layer 7 of the OSI model. +* If you're new to Kubernetes, this topic helps you <>. * You can <> with `externalIPs`. @@ -57,4 +47,4 @@ include::add-ons-images.adoc[leveloffset=+1] include::eks-add-ons.adoc[leveloffset=+1] -include::image-verification.adoc[leveloffset=+1] +include::image-verification.adoc[leveloffset=+1] \ No newline at end of file diff --git a/latest/ug/workloads/horizontal-pod-autoscaler.adoc b/latest/ug/workloads/horizontal-pod-autoscaler.adoc index ce25b2c06..f983406b0 100644 --- a/latest/ug/workloads/horizontal-pod-autoscaler.adoc +++ b/latest/ug/workloads/horizontal-pod-autoscaler.adoc @@ -1,38 +1,34 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[horizontal-pod-autoscaler,horizontal-pod-autoscaler.title]] -= Scale pod deployments with [.noloc]`Horizontal Pod Autoscaler` -:info_doctype: section -:info_title: Scale pod deployments with Horizontal Pod Autoscaler +[#horizontal-pod-autoscaler] += Scale pod deployments with Horizontal Pod Autoscaler :info_titleabbrev: Horizontal Pod Autoscaler -:info_abstract: Learn how to use the Kubernetes Horizontal Pod Autoscaler to automatically scale your Amazon EKS deployments based on CPU utilization for efficient resource management. [abstract] -- Learn how to use the Kubernetes Horizontal Pod Autoscaler to automatically scale your Amazon EKS deployments based on CPU utilization for efficient resource management. -- -The [.noloc]`Kubernetes` https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/[Horizontal Pod Autoscaler] automatically scales the number of [.noloc]`Pods` in a deployment, replication controller, or replica set based on that resource's CPU utilization. This can help your applications scale out to meet increased demand or scale in when resources are not needed, thus freeing up your nodes for other applications. When you set a target CPU utilization percentage, the Horizontal Pod Autoscaler scales your application in or out to try to meet that target. +The Kubernetes https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/[Horizontal Pod Autoscaler] automatically scales the number of Pods in a deployment, replication controller, or replica set based on that resource's CPU utilization. This can help your applications scale out to meet increased demand or scale in when resources are not needed, thus freeing up your nodes for other applications. When you set a target CPU utilization percentage, the Horizontal Pod Autoscaler scales your application in or out to try to meet that target. -The [.noloc]`Horizontal Pod Autoscaler` is a standard API resource in [.noloc]`Kubernetes` that simply requires that a metrics source (such as the [.noloc]`Kubernetes` metrics server) is installed on your Amazon EKS cluster to work. You do not need to deploy or install the [.noloc]`Horizontal Pod Autoscaler` on your cluster to begin scaling your applications. For more information, see https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/[Horizontal Pod Autoscaler] in the [.noloc]`Kubernetes` documentation. +The Horizontal Pod Autoscaler is a standard API resource in Kubernetes that simply requires that a metrics source (such as the Kubernetes metrics server) is installed on your Amazon EKS cluster to work. You do not need to deploy or install the Horizontal Pod Autoscaler on your cluster to begin scaling your applications. For more information, see https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/[Horizontal Pod Autoscaler] in the Kubernetes documentation. -Use this topic to prepare the [.noloc]`Horizontal Pod Autoscaler` for your Amazon EKS cluster and to verify that it is working with a sample application. +Use this topic to prepare the Horizontal Pod Autoscaler for your Amazon EKS cluster and to verify that it is working with a sample application. [NOTE] ==== -This topic is based on the https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/[Horizontal Pod autoscaler walkthrough] in the [.noloc]`Kubernetes` documentation. +This topic is based on the https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/[Horizontal Pod autoscaler walkthrough] in the Kubernetes documentation. ==== * You have an existing Amazon EKS cluster. If you don't, see <>. -* You have the [.noloc]`Kubernetes` Metrics Server installed. For more information, see <>. +* You have the Kubernetes Metrics Server installed. For more information, see <>. * You are using a `kubectl` client that is <>. -[[hpa-sample-app,hpa-sample-app.title]] +[#hpa-sample-app] == Run a Horizontal Pod Autoscaler test application In this section, you deploy a sample application to verify that the Horizontal Pod Autoscaler is working. @@ -40,7 +36,7 @@ In this section, you deploy a sample application to verify that the Horizontal P [NOTE] ==== -This example is based on the https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/[Horizontal Pod autoscaler walkthrough] in the [.noloc]`Kubernetes` documentation. +This example is based on the https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/[Horizontal Pod autoscaler walkthrough] in the Kubernetes documentation. ==== . Deploy a simple Apache web server application with the following command. @@ -50,7 +46,7 @@ This example is based on the https://kubernetes.io/docs/tasks/run-application/ho kubectl apply -f https://k8s.io/examples/application/php-apache.yaml ---- + -This Apache web server [.noloc]`Pod` is given a 500 millicpu CPU limit and it is serving on port 80. +This Apache web server Pod is given a 500 millicpu CPU limit and it is serving on port 80. . Create a Horizontal Pod Autoscaler resource for the `php-apache` deployment. + [source,bash,subs="verbatim,attributes"] @@ -58,7 +54,7 @@ This Apache web server [.noloc]`Pod` is given a 500 millicpu CPU limit and it is kubectl autoscale deployment php-apache --cpu-percent=50 --min=1 --max=10 ---- + -This command creates an autoscaler that targets 50 percent CPU utilization for the deployment, with a minimum of one [.noloc]`Pod` and a maximum of ten [.noloc]`Pods`. When the average CPU load is lower than 50 percent, the autoscaler tries to reduce the number of [.noloc]`Pods` in the deployment, to a minimum of one. When the load is greater than 50 percent, the autoscaler tries to increase the number of [.noloc]`Pods` in the deployment, up to a maximum of ten. For more information, see https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#how-does-a-horizontalpodautoscaler-work[How does a HorizontalPodAutoscaler work?] in the [.noloc]`Kubernetes` documentation. +This command creates an autoscaler that targets 50 percent CPU utilization for the deployment, with a minimum of one Pod and a maximum of ten Pods. When the average CPU load is lower than 50 percent, the autoscaler tries to reduce the number of Pods in the deployment, to a minimum of one. When the load is greater than 50 percent, the autoscaler tries to increase the number of Pods in the deployment, up to a maximum of ten. For more information, see https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#how-does-a-horizontalpodautoscaler-work[How does a HorizontalPodAutoscaler work?] in the Kubernetes documentation. . Describe the autoscaler with the following command to view its details. + [source,bash,subs="verbatim,attributes"] @@ -74,7 +70,7 @@ NAME REFERENCE TARGETS MINPODS MAXPODS REPLICAS AG php-apache Deployment/php-apache 0%/50% 1 10 1 51s ---- + -As you can see, the current CPU load is `0%`, because there's no load on the server yet. The [.noloc]`Pod` count is already at its lowest boundary (one), so it cannot scale in. +As you can see, the current CPU load is `0%`, because there's no load on the server yet. The Pod count is already at its lowest boundary (one), so it cannot scale in. . [[hpa-create-load]]Create a load for the web server by running a container. + [source,bash,subs="verbatim,attributes"] @@ -118,10 +114,10 @@ NAME REFERENCE TARGETS MINPODS MAXPODS REPLICAS AG php-apache Deployment/php-apache 0%/50% 1 10 1 25m ---- + -NOTE: The default timeframe for scaling back down is five minutes, so it will take some time before you see the replica count reach 1 again, even when the current CPU percentage is 0 percent. The timeframe is modifiable. For more information, see https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/[Horizontal Pod Autoscaler] in the [.noloc]`Kubernetes` documentation. +NOTE: The default timeframe for scaling back down is five minutes, so it will take some time before you see the replica count reach 1 again, even when the current CPU percentage is 0 percent. The timeframe is modifiable. For more information, see https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/[Horizontal Pod Autoscaler] in the Kubernetes documentation. . When you are done experimenting with your sample application, delete the `php-apache` resources. + [source,bash,subs="verbatim,attributes"] ---- kubectl delete deployment.apps/php-apache service/php-apache horizontalpodautoscaler.autoscaling/php-apache ----- +---- \ No newline at end of file diff --git a/latest/ug/workloads/image-verification.adoc b/latest/ug/workloads/image-verification.adoc index 8557afaf6..c02d9dc7e 100644 --- a/latest/ug/workloads/image-verification.adoc +++ b/latest/ug/workloads/image-verification.adoc @@ -1,13 +1,9 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[image-verification,image-verification.title]] +[#image-verification] = Validate container image signatures during deployment -:info_doctype: section -:info_title: Validate container image signatures during deployment :info_titleabbrev: Verify container images -:info_abstract: Learn how to verify signed container images during deployment on Amazon EKS using admission controllers like Gatekeeper with Ratify or Kyverno configured with {aws} Signer plugins for validating image signatures. [abstract] -- @@ -19,7 +15,7 @@ If you use link:signer/latest/developerguide/Welcome.html[{aws} Signer,type="doc * https://ratify.dev/docs/1.0/quickstarts/ratify-on-aws[Gatekeeper and Ratify] – Use Gatekeeper as the admission controller and Ratify configured with an {aws} Signer plugin as a web hook for validating signatures. -* https://github.com/nirmata/kyverno-notation-aws[Kyverno] – A [.noloc]`Kubernetes` policy engine configured with an {aws} Signer plugin for validating signatures. +* https://github.com/nirmata/kyverno-notation-aws[Kyverno] – A Kubernetes policy engine configured with an {aws} Signer plugin for validating signatures. [NOTE] @@ -27,4 +23,4 @@ If you use link:signer/latest/developerguide/Welcome.html[{aws} Signer,type="doc Before verifying container image signatures, configure the https://github.com/notaryproject/notation#readme[Notation] trust store and trust policy, as required by your selected admission controller. -==== +==== \ No newline at end of file diff --git a/latest/ug/workloads/kubernetes-field-management.adoc b/latest/ug/workloads/kubernetes-field-management.adoc index d59cee57d..e5466a7d8 100644 --- a/latest/ug/workloads/kubernetes-field-management.adoc +++ b/latest/ug/workloads/kubernetes-field-management.adoc @@ -1,10 +1,9 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[kubernetes-field-management,kubernetes-field-management.title]] +[#kubernetes-field-management] = Determine fields you can customize for Amazon EKS add-ons -:info_titleabbrev: Determine fields you can customize - -include::../attributes.txt[] +:info_titleabbrev: Fields you can customize [abstract] -- @@ -13,7 +12,7 @@ Learn how to manage Amazon EKS add-on configurations using Kubernetes field mana Amazon EKS add-ons are installed to your cluster using standard, best practice configurations. For more information about adding an Amazon EKS add-on to your cluster, see <>. -You may want to customize the configuration of an Amazon EKS add-on to enable advanced features. Amazon EKS uses the [.noloc]`Kubernetes` server-side apply feature to enable management of an add-on by Amazon EKS without overwriting your configuration for settings that aren't managed by Amazon EKS. For more information, see https://kubernetes.io/docs/reference/using-api/server-side-apply/[Server-Side Apply] in the [.noloc]`Kubernetes` documentation. To achieve this, Amazon EKS manages a minimum set of fields for every add-on that it installs. You can modify all fields that aren't managed by Amazon EKS, or another [.noloc]`Kubernetes` control plane process such as `kube-controller-manager`, without issue. +You may want to customize the configuration of an Amazon EKS add-on to enable advanced features. Amazon EKS uses the Kubernetes server-side apply feature to enable management of an add-on by Amazon EKS without overwriting your configuration for settings that aren't managed by Amazon EKS. For more information, see https://kubernetes.io/docs/reference/using-api/server-side-apply/[Server-Side Apply] in the Kubernetes documentation. To achieve this, Amazon EKS manages a minimum set of fields for every add-on that it installs. You can modify all fields that aren't managed by Amazon EKS, or another Kubernetes control plane process such as `kube-controller-manager`, without issue. [IMPORTANT] ==== @@ -22,10 +21,10 @@ Modifying a field managed by Amazon EKS prevents Amazon EKS from managing the ad ==== -[[add-on-config-management-understanding-field-management,add-on-config-management-understanding-field-management.title]] +[#add-on-config-management-understanding-field-management] == Field management syntax -When you view details for a [.noloc]`Kubernetes` object, both managed and unmanaged fields are returned in the output. Managed fields can be either of the following types: +When you view details for a Kubernetes object, both managed and unmanaged fields are returned in the output. Managed fields can be either of the following types: @@ -45,7 +44,7 @@ Each key is either a `.` representing the field itself, which always maps to an * `v:[.replaceable]``value```, where [.replaceable]`value` is the exact JSON formatted value of a list item. * `i:[.replaceable]``index```, where [.replaceable]`index` is position of an item in the list. -The following portions of output for the [.noloc]`CoreDNS` add-on illustrate the previous declarations: +The following portions of output for the CoreDNS add-on illustrate the previous declarations: @@ -107,14 +106,14 @@ manager: eks ---- -[[view-field-management,view-field-management.title]] +[#view-field-management] == Procedure You can use `kubectl` to see which fields are managed by Amazon EKS for any Amazon EKS add-on. -You can modify all fields that aren't managed by Amazon EKS, or another [.noloc]`Kubernetes` control plane process such as `kube-controller-manager`, without issue. +You can modify all fields that aren't managed by Amazon EKS, or another Kubernetes control plane process such as `kube-controller-manager`, without issue. -. Determine which add-on that you want to examine. To see all of the `deployments` and [.noloc]`DaemonSets` deployed to your cluster, see <>. +. Determine which add-on that you want to examine. To see all of the `deployments` and DaemonSets deployed to your cluster, see <>. . View the managed fields for an add-on by running the following command: + [source,bash,subs="verbatim,attributes"] @@ -122,7 +121,7 @@ You can modify all fields that aren't managed by Amazon EKS, or another [.noloc] kubectl get type/add-on-name -n add-on-namespace -o yaml ---- + -For example, you can see the managed fields for the [.noloc]`CoreDNS` add-on with the following command. +For example, you can see the managed fields for the CoreDNS add-on with the following command. + [source,bash,subs="verbatim,attributes"] ---- @@ -144,7 +143,7 @@ managedFields: NOTE: If you don't see `managedFields` in the output, add `--show-managed-fields` to the command and run it again. The version of `kubectl` that you're using determines whether managed fields are returned by default. -[[view-field-management-next-steps,view-field-management-next-steps.title]] +[#view-field-management-next-steps] == Next steps -Customize the fields not owned by {aws} for you add-on. +Customize the fields not owned by {aws} for you add-on. \ No newline at end of file diff --git a/latest/ug/workloads/network-load-balancing.adoc b/latest/ug/workloads/network-load-balancing.adoc index 3b71b4f1b..ed36593c2 100644 --- a/latest/ug/workloads/network-load-balancing.adoc +++ b/latest/ug/workloads/network-load-balancing.adoc @@ -1,13 +1,9 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[network-load-balancing,network-load-balancing.title]] -= Route [.noloc]`TCP` and [.noloc]`UDP` traffic with [.noloc]`Network Load Balancers` -:info_doctype: section -:info_title: Route TCP and UDP traffic with Network Load Balancers +[#network-load-balancing] += Route TCP and UDP traffic with Network Load Balancers :info_titleabbrev: Network load balancing -:info_abstract: Use the {aws} Load Balancer Controller to create network load balancers for Amazon EKS workloads, supporting IP and instance targets with {aws} Network Load Balancers. - -include::../attributes.txt[] [abstract] -- @@ -23,83 +19,83 @@ Use the {aws} Load Balancer Controller to create network load balancers for Amaz ==== -Network traffic is load balanced at `L4` of the OSI model. To load balance application traffic at `L7`, you deploy a [.noloc]`Kubernetes` `ingress`, which provisions an {aws} Application Load Balancer. For more information, see <>. To learn more about the differences between the two types of load balancing, see link:elasticloadbalancing/features/[Elastic Load Balancing features,type="marketing"] on the {aws} website. +Network traffic is load balanced at `L4` of the OSI model. To load balance application traffic at `L7`, you deploy a Kubernetes `ingress`, which provisions an {aws} Application Load Balancer. For more information, see <>. To learn more about the differences between the two types of load balancing, see link:elasticloadbalancing/features/[Elastic Load Balancing features,type="marketing"] on the {aws} website. -When you create a [.noloc]`Kubernetes` `Service` of type `LoadBalancer`, the {aws} cloud provider load balancer controller creates {aws} link:elasticloadbalancing/latest/classic/introduction.html[Classic Load Balancers,type="documentation"] by default, but can also create {aws} link:elasticloadbalancing/latest/network/introduction.html[Network Load Balancers,type="documentation"]. This controller is only receiving critical bug fixes in the future. For more information about using the {aws} cloud provider load balancer , see https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer[{aws} cloud provider load balancer controller] in the [.noloc]`Kubernetes` documentation. Its use is not covered in this topic. +When you create a Kubernetes `Service` of type `LoadBalancer`, the {aws} cloud provider load balancer controller creates {aws} link:elasticloadbalancing/latest/classic/introduction.html[Classic Load Balancers,type="documentation"] by default, but can also create {aws} link:elasticloadbalancing/latest/network/introduction.html[Network Load Balancers,type="documentation"]. This controller is only receiving critical bug fixes in the future. For more information about using the {aws} cloud provider load balancer , see https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer[{aws} cloud provider load balancer controller] in the Kubernetes documentation. Its use is not covered in this topic. -We recommend that you use version `2.7.2` or later of the <> instead of the {aws} cloud provider load balancer controller. The [.noloc]`{aws} Load Balancer Controller` creates {aws} Network Load Balancers, but doesn't create {aws} Classic Load Balancers. The remainder of this topic is about using the {aws} Load Balancer Controller. +We recommend that you use version `2.7.2` or later of the <> instead of the {aws} cloud provider load balancer controller. The {aws} Load Balancer Controller creates {aws} Network Load Balancers, but doesn't create {aws} Classic Load Balancers. The remainder of this topic is about using the {aws} Load Balancer Controller. -An {aws} Network Load Balancer can load balance network traffic to [.noloc]`Pods` deployed to Amazon EC2 IP and instance link:elasticloadbalancing/latest/network/load-balancer-target-groups.html#target-type[targets,type="documentation"], to {aws} Fargate IP targets, or to Amazon EKS Hybrid Nodes as IP targets. For more information, see https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/targetgroupbinding/targetgroupbinding/#targettype[{aws} Load Balancer Controller] on [.noloc]`GitHub`. +An {aws} Network Load Balancer can load balance network traffic to Pods deployed to Amazon EC2 IP and instance link:elasticloadbalancing/latest/network/load-balancer-target-groups.html#target-type[targets,type="documentation"], to {aws} Fargate IP targets, or to Amazon EKS Hybrid Nodes as IP targets. For more information, see https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/guide/targetgroupbinding/targetgroupbinding/#targettype[{aws} Load Balancer Controller] on GitHub. == Prerequisites -Before you can load balance network traffic using the [.noloc]`{aws} Load Balancer Controller`, you must meet the following requirements. +Before you can load balance network traffic using the {aws} Load Balancer Controller, you must meet the following requirements. * Have an existing cluster. If you don't have an existing cluster, see <>. If you need to update the version of an existing cluster, see <>. -* Have the [.noloc]`{aws} Load Balancer Controller` deployed on your cluster. For more information, see <>. We recommend version `2.7.2` or later. +* Have the {aws} Load Balancer Controller deployed on your cluster. For more information, see <>. We recommend version `2.7.2` or later. * At least one subnet. If multiple tagged subnets are found in an Availability Zone, the controller chooses the first subnet whose subnet ID comes first lexicographically. The subnet must have at least eight available IP addresses. -* If you're using the [.noloc]`{aws} Load Balancer Controller` version `2.1.1` or earlier, subnets must be tagged as follows. If using version `2.1.2` or later, this tag is optional. You might want to tag a subnet if you have multiple clusters running in the same VPC, or multiple {aws} services sharing subnets in a VPC, and want more control over where load balancers are provisioned for each cluster. If you explicitly specify subnet IDs as an annotation on a service object, then [.noloc]`Kubernetes` and the [.noloc]`{aws} Load Balancer Controller` use those subnets directly to create the load balancer. Subnet tagging isn't required if you choose to use this method for provisioning load balancers and you can skip the following private and public subnet tagging requirements. Replace [.replaceable]`my-cluster` with your cluster name. +* If you're using the {aws} Load Balancer Controller version `2.1.1` or earlier, subnets must be tagged as follows. If using version `2.1.2` or later, this tag is optional. You might want to tag a subnet if you have multiple clusters running in the same VPC, or multiple {aws} services sharing subnets in a VPC, and want more control over where load balancers are provisioned for each cluster. If you explicitly specify subnet IDs as an annotation on a service object, then Kubernetes and the {aws} Load Balancer Controller use those subnets directly to create the load balancer. Subnet tagging isn't required if you choose to use this method for provisioning load balancers and you can skip the following private and public subnet tagging requirements. Replace [.replaceable]`my-cluster` with your cluster name. + ** *Key* – `kubernetes.io/cluster/` ** *Value* – `shared` or `owned` -* Your public and private subnets must meet the following requirements, unless you explicitly specify subnet IDs as an annotation on a service or ingress object. If you provision load balancers by explicitly specifying subnet IDs as an annotation on a service or ingress object, then [.noloc]`Kubernetes` and the [.noloc]`{aws} Load Balancer Controller` use those subnets directly to create the load balancer and the following tags aren't required. +* Your public and private subnets must meet the following requirements, unless you explicitly specify subnet IDs as an annotation on a service or ingress object. If you provision load balancers by explicitly specifying subnet IDs as an annotation on a service or ingress object, then Kubernetes and the {aws} Load Balancer Controller use those subnets directly to create the load balancer and the following tags aren't required. + ** *Private subnets* - – Must be tagged in the following format. This is so that [.noloc]`Kubernetes` and the {aws} Load Balancer Controller know that the subnets can be used for internal load balancers. If you use `eksctl` or an Amazon EKS {aws} {aws} CloudFormation template to create your VPC after March 26, 2020, then the subnets are tagged appropriately when they're created. For more information about the Amazon EKS {aws} {aws} CloudFormation VPC templates, see <>. + – Must be tagged in the following format. This is so that Kubernetes and the {aws} Load Balancer Controller know that the subnets can be used for internal load balancers. If you use `eksctl` or an Amazon EKS {aws} {aws} CloudFormation template to create your VPC after March 26, 2020, then the subnets are tagged appropriately when they're created. For more information about the Amazon EKS {aws} {aws} CloudFormation VPC templates, see <>. + *** *Key* – `kubernetes.io/role/internal-elb` *** *Value* – `1` ** *Public subnets* - – Must be tagged in the following format. This is so that [.noloc]`Kubernetes` knows to use only those subnets for external load balancers instead of choosing a public subnet in each Availability Zone (based on the lexicographical order of the subnet IDs). If you use `eksctl` or an Amazon EKS {aws} CloudFormation template to create your VPC after March 26, 2020, then the subnets are tagged appropriately when they're created. For more information about the Amazon EKS {aws} CloudFormation VPC templates, see <>. + – Must be tagged in the following format. This is so that Kubernetes knows to use only those subnets for external load balancers instead of choosing a public subnet in each Availability Zone (based on the lexicographical order of the subnet IDs). If you use `eksctl` or an Amazon EKS {aws} CloudFormation template to create your VPC after March 26, 2020, then the subnets are tagged appropriately when they're created. For more information about the Amazon EKS {aws} CloudFormation VPC templates, see <>. + *** *Key* – `kubernetes.io/role/elb` *** *Value* – `1` + -If the subnet role tags aren't explicitly added, the [.noloc]`Kubernetes` service controller examines the route table of your cluster VPC subnets to determine if the subnet is private or public. We recommend that you don't rely on this behavior, and instead explicitly add the private or public role tags. The [.noloc]`{aws} Load Balancer Controller` doesn't examine route tables, and requires the private and public tags to be present for successful auto discovery. +If the subnet role tags aren't explicitly added, the Kubernetes service controller examines the route table of your cluster VPC subnets to determine if the subnet is private or public. We recommend that you don't rely on this behavior, and instead explicitly add the private or public role tags. The {aws} Load Balancer Controller doesn't examine route tables, and requires the private and public tags to be present for successful auto discovery. == Considerations -* The configuration of your load balancer is controlled by annotations that are added to the manifest for your service. Service annotations are different when using the [.noloc]`{aws} Load Balancer Controller` than they are when using the {aws} cloud provider load balancer controller. Make sure to review the https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/service/annotations/[annotations] for the [.noloc]`{aws} Load Balancer Controller` before deploying services. -* When using the <>, the [.noloc]`{aws} Load Balancer Controller` can load balance to Amazon EC2 IP or instance targets and Fargate IP targets. When using <>, the controller can only load balance to instance targets, unless you are load balancing to Amazon EKS Hybrid Nodes. For hybrid nodes, the controller can load balance IP targets. For more information about Network Load Balancer target types, see link:elasticloadbalancing/latest/network/load-balancer-target-groups.html#target-type[Target type,type="documentation"] in the User Guide for Network Load Balancers -* If you want to add tags to the load balancer when or after it's created, add the following annotation in your service specification. For more information, see https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/service/annotations/#aws-resource-tags[{aws} Resource Tags] in the [.noloc]`{aws} Load Balancer Controller` documentation. +* The configuration of your load balancer is controlled by annotations that are added to the manifest for your service. Service annotations are different when using the {aws} Load Balancer Controller than they are when using the {aws} cloud provider load balancer controller. Make sure to review the https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/guide/service/annotations/[annotations] for the {aws} Load Balancer Controller before deploying services. +* When using the <>, the {aws} Load Balancer Controller can load balance to Amazon EC2 IP or instance targets and Fargate IP targets. When using <>, the controller can only load balance to instance targets, unless you are load balancing to Amazon EKS Hybrid Nodes. For hybrid nodes, the controller can load balance IP targets. For more information about Network Load Balancer target types, see link:elasticloadbalancing/latest/network/load-balancer-target-groups.html#target-type[Target type,type="documentation"] in the User Guide for Network Load Balancers +* If you want to add tags to the load balancer when or after it's created, add the following annotation in your service specification. For more information, see https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/guide/service/annotations/#aws-resource-tags[{aws} Resource Tags] in the {aws} Load Balancer Controller documentation. + [source,bash,subs="verbatim,attributes"] ---- service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags ---- -* You can assign link:AWSEC2/latest/UserGuide/elastic-ip-addresses-eip.html[Elastic IP addresses,type="documentation"] to the Network Load Balancer by adding the following annotation. Replace the [.replaceable]`example values` with the `Allocation IDs` of your Elastic IP addresses. The number of `Allocation IDs` must match the number of subnets that are used for the load balancer. For more information, see the https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/service/annotations/#eip-allocations[{aws} Load Balancer Controller] documentation. +* You can assign link:AWSEC2/latest/UserGuide/elastic-ip-addresses-eip.html[Elastic IP addresses,type="documentation"] to the Network Load Balancer by adding the following annotation. Replace the example values with the `Allocation IDs` of your Elastic IP addresses. The number of `Allocation IDs` must match the number of subnets that are used for the load balancer. For more information, see the https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/guide/service/annotations/#eip-allocations[{aws} Load Balancer Controller] documentation. + [source,bash,subs="verbatim,attributes"] ---- service.beta.kubernetes.io/aws-load-balancer-eip-allocations: eipalloc-xxxxxxxxxxxxxxxxx,eipalloc-yyyyyyyyyyyyyyyyy ---- -* Amazon EKS adds one inbound rule to the node's security group for client traffic and one rule for each load balancer subnet in the VPC for health checks for each Network Load Balancer that you create. Deployment of a service of type `LoadBalancer` can fail if Amazon EKS attempts to create rules that exceed the quota for the maximum number of rules allowed for a security group. For more information, see link:vpc/latest/userguide/amazon-vpc-limits.html#vpc-limits-security-groups[Security groups,type="documentation"] in Amazon VPC quotas in the Amazon VPC User Guide. Consider the following options to minimize the chances of exceeding the maximum number of rules for a security group: +* Amazon EKS adds one inbound rule to the node's security group for client traffic and one rule for each load balancer subnet in the VPC for health checks for each Network Load Balancer that you create. Deployment of a service of type `LoadBalancer` can fail if Amazon EKS attempts to create rules that exceed the quota for the maximum number of rules allowed for a security group. For more information, see link:vpc/latest/userguide/amazon-vpc-limits.html#vpc-limits-security-groups[Security groups,type="documentation"] in Amazon VPC quotas in the Amazon VPC User Guide. Consider the following options to minimize the chances of exceeding the maximum number of rules for a security group: + ** Request an increase in your rules per security group quota. For more information, see link:servicequotas/latest/userguide/request-quota-increase.html[Requesting a quota increase,type="documentation"] in the Service Quotas User Guide. -** Use IP targets, rather than instance targets. With IP targets, you can share rules for the same target ports. You can manually specify load balancer subnets with an annotation. For more information, see https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/service/annotations/[Annotations] on [.noloc]`GitHub`. +** Use IP targets, rather than instance targets. With IP targets, you can share rules for the same target ports. You can manually specify load balancer subnets with an annotation. For more information, see https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/guide/service/annotations/[Annotations] on GitHub. ** Use an ingress, instead of a service of type `LoadBalancer`, to send traffic to your service. The {aws} Application Load Balancer requires fewer rules than Network Load Balancers. You can share an ALB across multiple ingresses. For more information, see <>. You can't share a Network Load Balancer across multiple services. ** Deploy your clusters to multiple accounts. -* If your [.noloc]`Pods` run on [.noloc]`Windows` in an Amazon EKS cluster, a single service with a load balancer can support up to 1024 back-end [.noloc]`Pods`. Each [.noloc]`Pod` has its own unique IP address. -* We recommend only creating new Network Load Balancers with the [.noloc]`{aws} Load Balancer Controller`. Attempting to replace existing Network Load Balancers created with the {aws} cloud provider load balancer controller can result in multiple Network Load Balancers that might cause application downtime. +* If your Pods run on Windows in an Amazon EKS cluster, a single service with a load balancer can support up to 1024 back-end Pods. Each Pod has its own unique IP address. +* We recommend only creating new Network Load Balancers with the {aws} Load Balancer Controller. Attempting to replace existing Network Load Balancers created with the {aws} cloud provider load balancer controller can result in multiple Network Load Balancers that might cause application downtime. -[[network-load-balancer,network-load-balancer.title]] +[#network-load-balancer] == Create a network load balancer You can create a network load balancer with IP or instance targets. === Create network load balancer -- IP Targets -* You can use IP targets with [.noloc]`Pods` deployed to Amazon EC2 nodes, Fargate, or Amazon EKS Hybrid Nodes. Your [.noloc]`Kubernetes` service must be created as type `LoadBalancer`. For more information, see https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer[Type LoadBalancer] in the [.noloc]`Kubernetes` documentation. +* You can use IP targets with Pods deployed to Amazon EC2 nodes, Fargate, or Amazon EKS Hybrid Nodes. Your Kubernetes service must be created as type `LoadBalancer`. For more information, see https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer[Type LoadBalancer] in the Kubernetes documentation. + -To create a load balancer that uses IP targets, add the following annotations to a service manifest and deploy your service. The `external` value for `aws-load-balancer-type` is what causes the [.noloc]`{aws} Load Balancer Controller`, rather than the {aws} cloud provider load balancer controller, to create the Network Load Balancer. You can view a <> with the annotations. +To create a load balancer that uses IP targets, add the following annotations to a service manifest and deploy your service. The `external` value for `aws-load-balancer-type` is what causes the {aws} Load Balancer Controller, rather than the {aws} cloud provider load balancer controller, to create the Network Load Balancer. You can view a <> with the annotations. + [source,bash,subs="verbatim,attributes"] ---- @@ -107,7 +103,7 @@ service.beta.kubernetes.io/aws-load-balancer-type: "external" service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip" ---- + -NOTE: If you're load balancing to `IPv6` [.noloc]`Pods`, add the following annotation. You can only load balance over `IPv6` to IP targets, not instance targets. Without this annotation, load balancing is over `IPv4`. +NOTE: If you're load balancing to `IPv6` Pods, add the following annotation. You can only load balance over `IPv6` to IP targets, not instance targets. Without this annotation, load balancing is over `IPv4`. + [source,bash,subs="verbatim,attributes"] ---- @@ -116,7 +112,7 @@ service.beta.kubernetes.io/aws-load-balancer-ip-address-type: dualstack + Network Load Balancers are created with the `internal` `aws-load-balancer-scheme`, by default. You can launch Network Load Balancers in any subnet in your cluster's VPC, including subnets that weren't specified when you created your cluster. + -[.noloc]`Kubernetes` examines the route table for your subnets to identify whether they are public or private. Public subnets have a route directly to the internet using an internet gateway, but private subnets do not. +Kubernetes examines the route table for your subnets to identify whether they are public or private. Public subnets have a route directly to the internet using an internet gateway, but private subnets do not. + If you want to create a Network Load Balancer in a public subnet to load balance to Amazon EC2 nodes (Fargate can only be private), specify `internet-facing` with the following annotation: + @@ -132,7 +128,7 @@ IMPORTANT: Do not edit the annotations after creating your service. If you need === Create network load balancer -- Instance Targets -* The {aws} cloud provider load balancer controller creates Network Load Balancers with instance targets only. Version `2.2.0` and later of the {aws} Load Balancer Controller also creates Network Load Balancers with instance targets. We recommend using it, rather than the {aws} cloud provider load balancer controller, to create new Network Load Balancers. You can use Network Load Balancer instance targets with [.noloc]`Pods` deployed to Amazon EC2 nodes, but not to Fargate. To load balance network traffic across [.noloc]`Pods` deployed to Fargate, you must use IP targets. +* The {aws} cloud provider load balancer controller creates Network Load Balancers with instance targets only. Version `2.2.0` and later of the {aws} Load Balancer Controller also creates Network Load Balancers with instance targets. We recommend using it, rather than the {aws} cloud provider load balancer controller, to create new Network Load Balancers. You can use Network Load Balancer instance targets with Pods deployed to Amazon EC2 nodes, but not to Fargate. To load balance network traffic across Pods deployed to Fargate, you must use IP targets. + To deploy a Network Load Balancer to a private subnet, your service specification must have the following annotations. You can view a <> with the annotations. The `external` value for `aws-load-balancer-type` is what causes the {aws} Load Balancer Controller, rather than the {aws} cloud provider load balancer controller, to create the Network Load Balancer. + @@ -142,7 +138,7 @@ service.beta.kubernetes.io/aws-load-balancer-type: "external" service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "instance" ---- + -Network Load Balancers are created with the `internal` `aws-load-balancer-scheme`, by default. For internal Network Load Balancers, your Amazon EKS cluster must be configured to use at least one private subnet in your VPC. [.noloc]`Kubernetes` examines the route table for your subnets to identify whether they are public or private. Public subnets have a route directly to the internet using an internet gateway, but private subnets do not. +Network Load Balancers are created with the `internal` `aws-load-balancer-scheme`, by default. For internal Network Load Balancers, your Amazon EKS cluster must be configured to use at least one private subnet in your VPC. Kubernetes examines the route table for your subnets to identify whether they are public or private. Public subnets have a route directly to the internet using an internet gateway, but private subnets do not. + If you want to create an Network Load Balancer in a public subnet to load balance to Amazon EC2 nodes, specify `internet-facing` with the following annotation: + @@ -154,12 +150,12 @@ service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing" IMPORTANT: Do not edit the annotations after creating your service. If you need to modify it, delete the service object and create it again with the desired value for this annotation. -[[load-balancer-sample-application,load-balancer-sample-application.title]] +[#load-balancer-sample-application] == (Optional) Deploy a sample application * At least one public or private subnet in your cluster VPC. -* Have the [.noloc]`{aws} Load Balancer Controller` deployed on your cluster. For more information, see <>. We recommend version `2.7.2` or later. -. If you're deploying to Fargate, make sure you have an available private subnet in your VPC and create a Fargate profile. If you're not deploying to Fargate, skip this step. You can create the profile by running the following command or in the <> using the same values for `name` and `namespace` that are in the command. Replace the [.replaceable]`example values` with your own. +* Have the {aws} Load Balancer Controller deployed on your cluster. For more information, see <>. We recommend version `2.7.2` or later. +. If you're deploying to Fargate, make sure you have an available private subnet in your VPC and create a Fargate profile. If you're not deploying to Fargate, skip this step. You can create the profile by running the following command or in the <> using the same values for `name` and `namespace` that are in the command. Replace the example values with your own. + [source,bash,subs="verbatim,attributes"] ---- @@ -177,7 +173,7 @@ eksctl create fargateprofile \ ---- kubectl create namespace nlb-sample-app ---- -.. Save the following contents to a file named [.replaceable]`sample-deployment`.yaml` file on your computer. +.. Save the following contents to a file named `sample-deployment.yaml` file on your computer. + [source,yaml,subs="verbatim,attributes"] ---- @@ -211,7 +207,7 @@ kubectl apply -f sample-deployment.yaml ---- . Create a service with an internet-facing Network Load Balancer that load balances to IP targets. + -.. [[network-load-balancing-service-sample-manifest]]Save the following contents to a file named [.replaceable]`sample-service`.yaml` file on your computer. If you're deploying to Fargate nodes, remove the `service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing` line. +.. [[network-load-balancing-service-sample-manifest]]Save the following contents to a file named `sample-service.yaml` file on your computer. If you're deploying to Fargate nodes, remove the `service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing` line. + [source,yaml,subs="verbatim,attributes"] ---- @@ -255,8 +251,8 @@ sample-service LoadBalancer 10.100.240.137 k8s-nlbsampl-nlbsampl-xxxxxxxxxx ---- + NOTE: The values for [.replaceable]`10.100.240.137` and [.replaceable]`xxxxxxxxxx`-[.replaceable]`xxxxxxxxxxxxxxxx` will be different than the example output (they will be unique to your load balancer) and [.replaceable]`us-west-2` may be different for you, depending on which {aws} Region that your cluster is in. -. Open the link:ec2[Amazon EC2 {aws-management-console},type="console"]. Select *Target Groups* (under *Load Balancing*) in the left navigation pane. In the *Name* column, select the target group's name where the value in the *Load balancer* column matches a portion of the name in the `EXTERNAL-IP` column of the output in the previous step. For example, you'd select the target group named `k8s-default-samplese-[.replaceable]``xxxxxxxxxx``` if your output were the same as the previous output. The *Target type* is `IP` because that was specified in the sample service manifest. -. Select the *Target group* and then select the *Targets* tab. Under *Registered targets*, you should see three IP addresses of the three replicas deployed in a previous step. Wait until the status of all targets is *healthy* before continuing. It might take several minutes before all targets are `healthy`. The targets might be in an `unhealthy` state before changing to a `healthy` state. +. Open the link:ec2[Amazon EC2 {aws-management-console},type="console"]. Select *Target Groups* (under *Load Balancing*) in the left navigation pane. In the *Name* column, select the target group's name where the value in the *Load balancer* column matches a portion of the name in the `EXTERNAL-IP` column of the output in the previous step. For example, you'd select the target group named `k8s-default-samplese-[.replaceable]``xxxxxxxxxx``` if your output were the same as the previous output. The *Target type* is `IP` because that was specified in the sample service manifest. +. Select the *Target group* and then select the *Targets* tab. Under *Registered targets*, you should see three IP addresses of the three replicas deployed in a previous step. Wait until the status of all targets is *healthy* before continuing. It might take several minutes before all targets are `healthy`. The targets might be in an `unhealthy` state before changing to a `healthy` state. . Send traffic to the service replacing [.replaceable]`xxxxxxxxxx-xxxxxxxxxxxxxxxx` and [.replaceable]`us-west-2` with the values returned in the output for a <> for `EXTERNAL-IP`. If you deployed to a private subnet, then you'll need to view the page from a device within your VPC, such as a bastion host. For more information, see link:quickstart/architecture/linux-bastion/[Linux Bastion Hosts on {aws},type="marketing"]. + [source,bash,subs="verbatim,attributes"] @@ -279,4 +275,4 @@ An example output is as follows. [source,bash,subs="verbatim,attributes"] ---- kubectl delete namespace nlb-sample-app ----- +---- \ No newline at end of file diff --git a/latest/ug/workloads/remove-addon-role.adoc b/latest/ug/workloads/remove-addon-role.adoc index c2ec0d41b..3ab469344 100644 --- a/latest/ug/workloads/remove-addon-role.adoc +++ b/latest/ug/workloads/remove-addon-role.adoc @@ -1,10 +1,9 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[remove-addon-role,remove-addon-role.title]] +[#remove-addon-role] = Remove Pod Identity associations from an Amazon EKS add-on -:info_titleabbrev: Remove Pod Identity associations - -include::../attributes.txt[] +:info_titleabbrev: Remove Pod Identity [abstract] -- @@ -24,4 +23,4 @@ Remove the Pod Identity associations from an Amazon EKS add-on. aws eks update-addon --cluster-name \ --addon-name \ --pod-identity-associations "[]" ----- +---- \ No newline at end of file diff --git a/latest/ug/workloads/removing-an-addon.adoc b/latest/ug/workloads/removing-an-addon.adoc index e8026954d..bfff11ea3 100644 --- a/latest/ug/workloads/removing-an-addon.adoc +++ b/latest/ug/workloads/removing-an-addon.adoc @@ -1,13 +1,10 @@ -//!!NODE_ROOT
+include::../attributes.txt[] [.topic] -[[removing-an-add-on,removing-an-add-on.title]] +[#removing-an-add-on] = Remove an Amazon EKS add-on from a cluster :info_titleabbrev: Remove an add-on -include::../attributes.txt[] - - [abstract] -- Learn how to remove an Amazon EKS add-on. @@ -28,7 +25,7 @@ When you remove an Amazon EKS add-on from a cluster: * You can choose to leave the add-on software on your cluster so that you can self-manage it, or you can remove the add-on software from your cluster. You should only remove the add-on software from your cluster if there are no resources on your cluster are dependent on the functionality that the add-on provides. -[[removing-an-add-on-prereq,removing-an-add-on-prereq.title]] +[#removing-an-add-on-prereq] == Prerequisites Complete the following before you create an add-on: @@ -40,7 +37,7 @@ Complete the following before you create an add-on: * Version `{eksctl-min-version}` or later of the `eksctl` command line tool installed on your device or {aws} CloudShell. To install or update `eksctl`, see https://eksctl.io/installation[Installation] in the `eksctl` documentation.. -[[removing-an-add-on-procedure,removing-an-add-on-procedure.title]] +[#removing-an-add-on-procedure] == Procedure You have two options when removing an Amazon EKS add-on. @@ -110,7 +107,7 @@ aws eks list-addons --cluster-name my-cluster + An example output is as follows. + -[source,json,subs="verbatim,attributes"] +[source,json,subs="verbatim,attributes",role="nocopy"] ---- { "addons": [ @@ -130,7 +127,7 @@ aws eks delete-addon --cluster-name my-cluster --addon-name name-of-addon --pres + The abbreviated example output is as follows. + -[source,json,subs="verbatim,attributes"] +[source,json,subs="verbatim,attributes",role="nocopy"] ---- { "addon": { @@ -152,4 +149,4 @@ After the add-on is removed, the example output is as follows. [source,bash,subs="verbatim,attributes"] ---- An error occurred (ResourceNotFoundException) when calling the DescribeAddon operation: No addon: name-of-addon found in cluster: my-cluster ----- +---- \ No newline at end of file diff --git a/latest/ug/workloads/restrict-service-external-ip.adoc b/latest/ug/workloads/restrict-service-external-ip.adoc index 6e4e860d5..50cc8ea64 100644 --- a/latest/ug/workloads/restrict-service-external-ip.adoc +++ b/latest/ug/workloads/restrict-service-external-ip.adoc @@ -1,17 +1,16 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[restrict-service-external-ip,restrict-service-external-ip.title]] +[#restrict-service-external-ip] = Restrict external IP addresses that can be assigned to services -:info_titleabbrev: Restrict service external IP address assignment - -include::../attributes.txt[] +:info_titleabbrev: Restrict service external IPs -[.noloc]`Kubernetes` services can be reached from inside of a cluster through: +Kubernetes services can be reached from inside of a cluster through: -* A cluster IP address that is assigned automatically by [.noloc]`Kubernetes` -* Any IP address that you specify for the `externalIPs` property in a service spec. External IP addresses are not managed by [.noloc]`Kubernetes` and are the responsibility of the cluster administrator. External IP addresses specified with `externalIPs` are different than the external IP address assigned to a service of type `LoadBalancer` by a cloud provider. +* A cluster IP address that is assigned automatically by Kubernetes +* Any IP address that you specify for the `externalIPs` property in a service spec. External IP addresses are not managed by Kubernetes and are the responsibility of the cluster administrator. External IP addresses specified with `externalIPs` are different than the external IP address assigned to a service of type `LoadBalancer` by a cloud provider. -To learn more about [.noloc]`Kubernetes` services, see https://kubernetes.io/docs/concepts/services-networking/service/[Service] in the [.noloc]`Kubernetes` documentation. You can restrict the IP addresses that can be specified for `externalIPs` in a service spec. +To learn more about Kubernetes services, see https://kubernetes.io/docs/concepts/services-networking/service/[Service] in the Kubernetes documentation. You can restrict the IP addresses that can be specified for `externalIPs` in a service spec. . Deploy `cert-manager` to manage webhook certificates. For more information, see the https://cert-manager.io/docs/[cert-manager] documentation. + @@ -19,7 +18,7 @@ To learn more about [.noloc]`Kubernetes` services, see https://kubernetes.io/doc ---- kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.5.4/cert-manager.yaml ---- -. Verify that the `cert-manager` [.noloc]`Pods` are running. +. Verify that the `cert-manager` Pods are running. + [source,bash,subs="verbatim,attributes"] ---- @@ -56,7 +55,7 @@ my-namespace my-service ClusterIP ---- + If any of the values are IP addresses that are not within the block you want to restrict access to, you'll need to change the addresses to be within the block, and redeploy the services. For example, the `my-service` service in the previous output has an external IP address assigned to it that isn't within the CIDR block example in step 5. -. Download the external IP webhook manifest. You can also view the https://github.com/kubernetes-sigs/externalip-webhook[source code for the webhook] on [.noloc]`GitHub`. +. Download the external IP webhook manifest. You can also view the https://github.com/kubernetes-sigs/externalip-webhook[source code for the webhook] on GitHub. + [source,bash,subs="verbatim,attributes"] ---- @@ -86,4 +85,4 @@ sed -i.bak -e 's|amazonaws.com||' externalip-webhook.yaml kubectl apply -f externalip-webhook.yaml ---- + -An attempt to deploy a service to your cluster with an IP address specified for `externalIPs` that is not contained in the blocks that you specified in the Specify CIDR blocks step will fail. +An attempt to deploy a service to your cluster with an IP address specified for `externalIPs` that is not contained in the blocks that you specified in the Specify CIDR blocks step will fail. \ No newline at end of file diff --git a/latest/ug/workloads/retreive-iam-info.adoc b/latest/ug/workloads/retreive-iam-info.adoc index 4ec80c3e0..4d5cbca41 100644 --- a/latest/ug/workloads/retreive-iam-info.adoc +++ b/latest/ug/workloads/retreive-iam-info.adoc @@ -1,11 +1,10 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[retreive-iam-info,retreive-iam-info.title]] +[#retreive-iam-info] = Retrieve IAM information about an Amazon EKS add-on :info_titleabbrev: Retrieve IAM information -include::../attributes.txt[] - [abstract] -- Learn how to determine the role and policy to use for an Amazon EKS add-on. @@ -18,7 +17,7 @@ Before you create an add-on, use the {aws} CLI to determine: == Procedure -. Determine the name of the add-on you want to install, and the [.noloc]`Kubernetes` version of your cluster. For more information about add-ons, see <>. +. Determine the name of the add-on you want to install, and the Kubernetes version of your cluster. For more information about add-ons, see <>. . Use the {aws} CLI to determine if the add-on requires IAM permissions. + [source,shell,subs="verbatim,attributes"] @@ -102,39 +101,42 @@ Review the following output. Note the `recommendedManagedPolicies`. ---- . Create an IAM role and attach the recommended Managed Policy. Alternatively, review the managed policy and scope down the permissions as appropriate. For more information see <>. -[[pod-id-add-on-versions,pod-id-add-on-versions.title]] +[#pod-id-add-on-versions] == Pod Identity Support Reference The following table indicates if certain Amazon EKS add-ons support EKS Pod Identity. -[cols="2,1,2"] -|=== -| Add-on Name | Pod Identity Support | Minimum Version Required +[%header,cols="2,1,2"] +|=== +|Add-on name +|Pod identity support +|Minimum version required + -| xref:add-ons-aws-ebs-csi-driver[Amazon EBS CSI Driver] -| Yes -| v1.26.0-eksbuild.1 +|<> +|Yes +|v1.26.0-eksbuild.1 -| xref:add-ons-vpc-cni[Amazon VPC CNI] -| Yes -| v1.15.5-eksbuild.1 +|<> +|Yes +|v1.15.5-eksbuild.1 -| xref:add-ons-aws-efs-csi-driver[Amazon EFS CSI Driver] -| Yes -| v2.0.5-eksbuild.1 +|<> +|Yes +|v2.0.5-eksbuild.1 -| xref:add-ons-adot[{aws} Distro for OpenTelemetry] -| Yes -| v0.94.1-eksbuild.1 +|<> +|Yes +|v0.94.1-eksbuild.1 -| xref:mountpoint-for-s3-add-on[Mountpoint for Amazon S3 CSI Driver] -| No -| N/A +|<> +|No +|N/A -| xref:amazon-cloudwatch-observability[Amazon CloudWatch Observability agent] -| No -| N/A +|<> +|Yes +|v3.1.0-eksbuild.1 |=== -This table was last updated on October 28, 2024. +This table was last updated on October 28, 2024. \ No newline at end of file diff --git a/latest/ug/workloads/sample-deployment-windows.adoc b/latest/ug/workloads/sample-deployment-windows.adoc index 9d595b197..bca075c0b 100644 --- a/latest/ug/workloads/sample-deployment-windows.adoc +++ b/latest/ug/workloads/sample-deployment-windows.adoc @@ -1,28 +1,24 @@ -//!!NODE_ROOT
+include::../attributes.txt[] [.topic] -[[sample-deployment-win,sample-deployment-win.title]] +[#sample-deployment-win] = Deploy a sample application on Windows -:info_doctype: section -:info_title: Deploy a sample application on Windows -:info_titleabbrev: Sample application deployment (Windows) - -include::../attributes.txt[] +:info_titleabbrev: Sample deployment (Windows) In this topic, you deploy a sample application to your cluster on Windows nodes. == Prerequisites -* An existing [.noloc]`Kubernetes` cluster with at least one node. If you don't have an existing Amazon EKS cluster, you can deploy one using one of the guides in <>. You must have <> enabled for your cluster and at least one Amazon EC2 [.noloc]`Windows` node. +* An existing Kubernetes cluster with at least one node. If you don't have an existing Amazon EKS cluster, you can deploy one using one of the guides in <>. You must have <> enabled for your cluster and at least one Amazon EC2 Windows node. * `Kubectl` installed on your computer. For more information, see <>. * `Kubectl` configured to communicate with your cluster. For more information, see <>. * If you plan to deploy your sample workload to Fargate, then you must have an existing <> that includes the same namespace created in this tutorial, which is `eks-sample-app`, unless you change the name. If you created a cluster with one of the gudes in <>, then you'll have to create a new profile, or add the namespace to your existing profile, because the profile created in the getting started guides doesn't specify the namespace used in this tutorial. Your VPC must also have at least one private subnet. -Though many variables are changeable in the following steps, we recommend only changing variable values where specified. Once you have a better understanding of [.noloc]`Kubernetes` [.noloc]`Pods`, deployments, and services, you can experiment with changing other values. +Though many variables are changeable in the following steps, we recommend only changing variable values where specified. Once you have a better understanding of Kubernetes Pods, deployments, and services, you can experiment with changing other values. == Create a namespace -A namespace allows you to group resources in [.noloc]`Kubernetes`. For more information, see https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/[Namespaces] in the [.noloc]`Kubernetes` documentation. If you plan to deploy your sample application to <>, make sure that the value for `namespace` in your <> is `eks-sample-app`. +A namespace allows you to group resources in Kubernetes. For more information, see https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/[Namespaces] in the Kubernetes documentation. If you plan to deploy your sample application to <>, make sure that the value for `namespace` in your <> is `eks-sample-app`. [source,bash,subs="verbatim,attributes"] ---- @@ -30,13 +26,13 @@ kubectl create namespace eks-sample-app ---- -== Create a [.noloc]`Kubernetes` deployment +== Create a Kubernetes deployment -This sample deployment pulls a container image from a public repository and deploys three replicas (individual [.noloc]`Pods`) of it to your cluster. To learn more, see https://kubernetes.io/docs/concepts/workloads/controllers/deployment/[Deployments] in the [.noloc]`Kubernetes` documentation. +This sample deployment pulls a container image from a public repository and deploys three replicas (individual Pods) of it to your cluster. To learn more, see https://kubernetes.io/docs/concepts/workloads/controllers/deployment/[Deployments] in the Kubernetes documentation. . Save the following contents to a file named `eks-sample-deployment.yaml`. The containers in the sample application don't use network storage, but you might have applications that need to. For more information, see <>. + -** The `kubernetes.io/os: windows` `nodeSelector` means that if you had [.noloc]`Windows` and [.noloc]`Linux` nodes (for example) in your cluster, the image would only be deployed to [.noloc]`Windows` nodes. For more information, see https://kubernetes.io/docs/reference/labels-annotations-taints/[Well-Known Labels, Annotations and Taints] in the [.noloc]`Kubernetes` documentation. +** The `kubernetes.io/os: windows` `nodeSelector` means that if you had Windows and Linux nodes (for example) in your cluster, the image would only be deployed to Windows nodes. For more information, see https://kubernetes.io/docs/reference/labels-annotations-taints/[Well-Known Labels, Annotations and Taints] in the Kubernetes documentation. + [source,yaml,subs="verbatim,attributes"] ---- @@ -62,7 +58,7 @@ spec: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - - key: beta.kubernetes.io/arch + - key: kubernetes.io/arch operator: In values: - amd64 @@ -89,9 +85,9 @@ kubectl apply -f eks-sample-deployment.yaml == Create a service -A service allows you to access all replicas through a single IP address or name. For more information, see https://kubernetes.io/docs/concepts/services-networking/service/[Service] in the [.noloc]`Kubernetes` documentation. Though not implemented in the sample application, if you have applications that need to interact with other {aws} services, we recommend that you create [.noloc]`Kubernetes` service accounts for your [.noloc]`Pods`, and associate them to {aws} IAM accounts. By specifying service accounts, your [.noloc]`Pods` have only the minimum permissions that you specify for them to interact with other services. For more information, see <>. +A service allows you to access all replicas through a single IP address or name. For more information, see https://kubernetes.io/docs/concepts/services-networking/service/[Service] in the Kubernetes documentation. Though not implemented in the sample application, if you have applications that need to interact with other {aws} services, we recommend that you create Kubernetes service accounts for your Pods, and associate them to {aws} IAM accounts. By specifying service accounts, your Pods have only the minimum permissions that you specify for them to interact with other services. For more information, see <>. -. Save the following contents to a file named `eks-sample-service.yaml`. [.noloc]`Kubernetes` assigns the service its own IP address that is accessible only from within the cluster. To access the service from outside of your cluster, deploy the <> to load balance <> or <> traffic to the service. +. Save the following contents to a file named `eks-sample-service.yaml`. Kubernetes assigns the service its own IP address that is accessible only from within the cluster. To access the service from outside of your cluster, deploy the <> to load balance <> or <> traffic to the service. + [source,yaml,subs="verbatim,attributes"] ---- @@ -117,7 +113,7 @@ spec: kubectl apply -f eks-sample-service.yaml ---- -[[sample-app-view-namespace-win,sample-app-view-namespace-win.title]] +[#sample-app-view-namespace-win] == Review resources created . View all resources that exist in the `eks-sample-app` namespace. @@ -146,9 +142,9 @@ NAME DESIRED CURRENT RE replicaset.apps/eks-sample-windows-deployment-776d8f8fd8 3 3 3 27m ---- + -In the output, you see the service and deployment that were specified in the sample manifests deployed in previous steps. You also see three [.noloc]`Pods`. This is because `3` `replicas` were specified in the sample manifest. For more information about [.noloc]`Pods`, see https://kubernetes.io/docs/concepts/workloads/pods/pod/[Pods] in the [.noloc]`Kubernetes` documentation. [.noloc]`Kubernetes` automatically creates the `replicaset` resource, even though it isn't specified in the sample manifests. For more information about `ReplicaSets`, see https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/[ReplicaSet] in the [.noloc]`Kubernetes` documentation. +In the output, you see the service and deployment that were specified in the sample manifests deployed in previous steps. You also see three Pods. This is because `3` `replicas` were specified in the sample manifest. For more information about Pods, see https://kubernetes.io/docs/concepts/workloads/pods/pod/[Pods] in the Kubernetes documentation. Kubernetes automatically creates the `replicaset` resource, even though it isn't specified in the sample manifests. For more information about `ReplicaSets`, see https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/[ReplicaSet] in the Kubernetes documentation. + -NOTE: [.noloc]`Kubernetes` maintains the number of replicas that are specified in the manifest. If this were a production deployment and you wanted [.noloc]`Kubernetes` to horizontally scale the number of replicas or vertically scale the compute resources for the [.noloc]`Pods`, use the <> and the <> to do so. +NOTE: Kubernetes maintains the number of replicas that are specified in the manifest. If this were a production deployment and you wanted Kubernetes to horizontally scale the number of replicas or vertically scale the compute resources for the Pods, use the <> and the <> to do so. . View the details of the deployed service. + [source,bash,subs="verbatim,attributes"] @@ -176,8 +172,8 @@ Session Affinity: None Events: ---- + -In the previous output, the value for `IP:` is a unique IP address that can be reached from any node or [.noloc]`Pod` within the cluster, but it can't be reached from outside of the cluster. The values for `Endpoints` are IP addresses assigned from within your VPC to the [.noloc]`Pods` that are part of the service. -. View the details of one of the [.noloc]`Pods` listed in the output when you <> in a previous step. Replace [.replaceable]`776d8f8fd8-78w66` with the value returned for one of your [.noloc]`Pods`. +In the previous output, the value for `IP:` is a unique IP address that can be reached from any node or Pod within the cluster, but it can't be reached from outside of the cluster. The values for `Endpoints` are IP addresses assigned from within your VPC to the Pods that are part of the service. +. View the details of one of the Pods listed in the output when you <> in a previous step. Replace [.replaceable]`776d8f8fd8-78w66` with the value returned for one of your Pods. + [source,bash,subs="verbatim,attributes"] ---- @@ -212,19 +208,19 @@ Events: [...] ---- + -In the previous output, the value for `IP:` is a unique IP that's assigned to the [.noloc]`Pod` from the CIDR block assigned to the subnet that the node is in. If you prefer to assign [.noloc]`Pods` IP addresses from different CIDR blocks, you can change the default behavior. For more information, see <>. You can also see that the [.noloc]`Kubernetes` scheduler scheduled the [.noloc]`Pod` on the `Node` with the IP address [.replaceable]`192.168.45.132`. +In the previous output, the value for `IP:` is a unique IP that's assigned to the Pod from the CIDR block assigned to the subnet that the node is in. If you prefer to assign Pods IP addresses from different CIDR blocks, you can change the default behavior. For more information, see <>. You can also see that the Kubernetes scheduler scheduled the Pod on the `Node` with the IP address [.replaceable]`192.168.45.132`. + -TIP: Rather than using the command line, you can view many details about [.noloc]`Pods`, services, deployments, and other [.noloc]`Kubernetes` resources in the {aws-management-console}. For more information, see <>. +TIP: Rather than using the command line, you can view many details about Pods, services, deployments, and other Kubernetes resources in the {aws-management-console}. For more information, see <>. == Run a shell on a Pod -. Run a shell on the [.noloc]`Pod` that you described in the previous step, replacing [.replaceable]`65b7669776-m6qxz` with the ID of one of your [.noloc]`Pods`. +. Run a shell on the Pod that you described in the previous step, replacing [.replaceable]`65b7669776-m6qxz` with the ID of one of your Pods. + [source,bash,subs="verbatim,attributes"] ---- kubectl exec -it eks-sample-windows-deployment-65b7669776-m6qxz -n eks-sample-app -- powershell.exe ---- -. From the [.noloc]`Pod` shell, view the output from the web server that was installed with your deployment in a previous step. You only need to specify the service name. It is resolved to the service's IP address by [.noloc]`CoreDNS`, which is deployed with an Amazon EKS cluster, by default. +. From the Pod shell, view the output from the web server that was installed with your deployment in a previous step. You only need to specify the service name. It is resolved to the service's IP address by CoreDNS, which is deployed with an Amazon EKS cluster, by default. + [source,bash,subs="verbatim,attributes"] ---- @@ -240,7 +236,7 @@ StatusDescription : OK Content : < h t m l > < b o d y > < b r / > < b r / > < m a r q u e e > < H 1 > H e l l o E K S ! ! ! < H 1 > < m a r q u e e > < / b o d y > < h t m l > ---- -. From the [.noloc]`Pod` shell, view the DNS server for the [.noloc]`Pod`. +. From the Pod shell, view the DNS server for the Pod. + [source,bash,subs="verbatim,attributes"] ---- @@ -258,8 +254,8 @@ IPv4Address : 192.168.63.14 DNSServer : 10.100.0.10 ---- + -In the previous output, `10.100.0.10` is automatically assigned as the DNS server for all [.noloc]`Pods` deployed to the cluster. -. Disconnect from the [.noloc]`Pod` by typing `exit`. +In the previous output, `10.100.0.10` is automatically assigned as the DNS server for all Pods deployed to the cluster. +. Disconnect from the Pod by typing `exit`. . Once you're finished with the sample application, you can remove the sample namespace, service, and deployment with the following command. + [source,bash,subs="verbatim,attributes"] @@ -268,7 +264,7 @@ kubectl delete namespace eks-sample-app ---- -[[sample-deployment-win-next-steps,sample-deployment-win-next-steps.title]] +[#sample-deployment-win-next-steps] == Next Steps After you deploy the sample application, you might want to try some of the following exercises: diff --git a/latest/ug/workloads/sample-deployment.adoc b/latest/ug/workloads/sample-deployment.adoc index d1c011335..f80c6c11a 100644 --- a/latest/ug/workloads/sample-deployment.adoc +++ b/latest/ug/workloads/sample-deployment.adoc @@ -1,29 +1,24 @@ -//!!NODE_ROOT
- +include::../attributes.txt[] [.topic] -[[sample-deployment,sample-deployment.title]] +[#sample-deployment] = Deploy a sample application on Linux -:info_doctype: section -:info_title: Deploy a sample application on Linux -:info_titleabbrev: Sample application deployment (Linux) - -include::../attributes.txt[] +:info_titleabbrev: Sample deployment (Linux) In this topic, you deploy a sample application to your cluster on linux nodes. == Prerequisites -* An existing [.noloc]`Kubernetes` cluster with at least one node. If you don't have an existing Amazon EKS cluster, you can deploy one using one of the guides in <>. +* An existing Kubernetes cluster with at least one node. If you don't have an existing Amazon EKS cluster, you can deploy one using one of the guides in <>. * `Kubectl` installed on your computer. For more information, see <>. * `Kubectl` configured to communicate with your cluster. For more information, see <>. * If you plan to deploy your sample workload to Fargate, then you must have an existing <> that includes the same namespace created in this tutorial, which is `eks-sample-app`, unless you change the name. If you created a cluster with one of the gudes in <>, then you'll have to create a new profile, or add the namespace to your existing profile, because the profile created in the getting started guides doesn't specify the namespace used in this tutorial. Your VPC must also have at least one private subnet. -Though many variables are changeable in the following steps, we recommend only changing variable values where specified. Once you have a better understanding of [.noloc]`Kubernetes` [.noloc]`Pods`, deployments, and services, you can experiment with changing other values. +Though many variables are changeable in the following steps, we recommend only changing variable values where specified. Once you have a better understanding of Kubernetes Pods, deployments, and services, you can experiment with changing other values. == Create a namespace -A namespace allows you to group resources in [.noloc]`Kubernetes`. For more information, see https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/[Namespaces] in the [.noloc]`Kubernetes` documentation. If you plan to deploy your sample application to <>, make sure that the value for `namespace` in your <> is `eks-sample-app`. +A namespace allows you to group resources in Kubernetes. For more information, see https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/[Namespaces] in the Kubernetes documentation. If you plan to deploy your sample application to <>, make sure that the value for `namespace` in your <> is `eks-sample-app`. [source,bash,subs="verbatim,attributes"] ---- @@ -32,12 +27,12 @@ kubectl create namespace eks-sample-app == Create a Kubernetes deployment -Create a [.noloc]`Kubernetes` deployment. This sample deployment pulls a container image from a public repository and deploys three replicas (individual [.noloc]`Pods`) of it to your cluster. To learn more, see https://kubernetes.io/docs/concepts/workloads/controllers/deployment/[Deployments] in the [.noloc]`Kubernetes` documentation. +Create a Kubernetes deployment. This sample deployment pulls a container image from a public repository and deploys three replicas (individual Pods) of it to your cluster. To learn more, see https://kubernetes.io/docs/concepts/workloads/controllers/deployment/[Deployments] in the Kubernetes documentation. . Save the following contents to a file named `eks-sample-deployment.yaml`. The containers in the sample application don't use network storage, but you might have applications that need to. For more information, see <>. + -** The `amd64` or `arm64` `values` under the `kubernetes.io/arch` key mean that the application can be deployed to either hardware architecture (if you have both in your cluster). This is possible because this image is a multi-architecture image, but not all are. You can determine the hardware architecture that the image is supported on by viewing the https://gallery.ecr.aws/nginx/nginx[image details] in the repository that you're pulling it from. When deploying images that don't support a hardware architecture type, or that you don't want the image deployed to, remove that type from the manifest. For more information, see https://kubernetes.io/docs/reference/labels-annotations-taints/[Well-Known Labels, Annotations and Taints] in the [.noloc]`Kubernetes` documentation. -** The `kubernetes.io/os: linux` `nodeSelector` means that if you had [.noloc]`Linux` and [.noloc]`Windows` nodes (for example) in your cluster, the image would only be deployed to [.noloc]`Linux` nodes. For more information, see https://kubernetes.io/docs/reference/labels-annotations-taints/[Well-Known Labels, Annotations and Taints] in the [.noloc]`Kubernetes` documentation. +** The `amd64` or `arm64` `values` under the `kubernetes.io/arch` key mean that the application can be deployed to either hardware architecture (if you have both in your cluster). This is possible because this image is a multi-architecture image, but not all are. You can determine the hardware architecture that the image is supported on by viewing the https://gallery.ecr.aws/nginx/nginx[image details] in the repository that you're pulling it from. When deploying images that don't support a hardware architecture type, or that you don't want the image deployed to, remove that type from the manifest. For more information, see https://kubernetes.io/docs/reference/labels-annotations-taints/[Well-Known Labels, Annotations and Taints] in the Kubernetes documentation. +** The `kubernetes.io/os: linux` `nodeSelector` means that if you had Linux and Windows nodes (for example) in your cluster, the image would only be deployed to Linux nodes. For more information, see https://kubernetes.io/docs/reference/labels-annotations-taints/[Well-Known Labels, Annotations and Taints] in the Kubernetes documentation. + [source,yaml,subs="verbatim,attributes"] ---- @@ -88,9 +83,9 @@ kubectl apply -f eks-sample-deployment.yaml == Create a service -A service allows you to access all replicas through a single IP address or name. For more information, see https://kubernetes.io/docs/concepts/services-networking/service/[Service] in the [.noloc]`Kubernetes` documentation. Though not implemented in the sample application, if you have applications that need to interact with other {aws} services, we recommend that you create [.noloc]`Kubernetes` service accounts for your [.noloc]`Pods`, and associate them to {aws} IAM accounts. By specifying service accounts, your [.noloc]`Pods` have only the minimum permissions that you specify for them to interact with other services. For more information, see <>. +A service allows you to access all replicas through a single IP address or name. For more information, see https://kubernetes.io/docs/concepts/services-networking/service/[Service] in the Kubernetes documentation. Though not implemented in the sample application, if you have applications that need to interact with other {aws} services, we recommend that you create Kubernetes service accounts for your Pods, and associate them to {aws} IAM accounts. By specifying service accounts, your Pods have only the minimum permissions that you specify for them to interact with other services. For more information, see <>. -. Save the following contents to a file named `eks-sample-service.yaml`. [.noloc]`Kubernetes` assigns the service its own IP address that is accessible only from within the cluster. To access the service from outside of your cluster, deploy the <> to load balance <> or <> traffic to the service. +. Save the following contents to a file named `eks-sample-service.yaml`. Kubernetes assigns the service its own IP address that is accessible only from within the cluster. To access the service from outside of your cluster, deploy the <> to load balance <> or <> traffic to the service. + [source,yaml,subs="verbatim,attributes"] ---- @@ -116,7 +111,7 @@ spec: kubectl apply -f eks-sample-service.yaml ---- -[[sample-app-view-namespace,sample-app-view-namespace.title]] +[#sample-app-view-namespace] == Review resources created . View all resources that exist in the `eks-sample-app` namespace. @@ -145,9 +140,9 @@ NAME DESIRED CURRENT RE replicaset.apps/eks-sample-linux-deployment-776d8f8fd8 3 3 3 27m ---- + -In the output, you see the service and deployment that were specified in the sample manifests deployed in previous steps. You also see three [.noloc]`Pods`. This is because `3` `replicas` were specified in the sample manifest. For more information about [.noloc]`Pods`, see https://kubernetes.io/docs/concepts/workloads/pods/pod/[Pods] in the [.noloc]`Kubernetes` documentation. [.noloc]`Kubernetes` automatically creates the `replicaset` resource, even though it isn't specified in the sample manifests. For more information about `ReplicaSets`, see https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/[ReplicaSet] in the [.noloc]`Kubernetes` documentation. +In the output, you see the service and deployment that were specified in the sample manifests deployed in previous steps. You also see three Pods. This is because `3` `replicas` were specified in the sample manifest. For more information about Pods, see https://kubernetes.io/docs/concepts/workloads/pods/pod/[Pods] in the Kubernetes documentation. Kubernetes automatically creates the `replicaset` resource, even though it isn't specified in the sample manifests. For more information about `ReplicaSets`, see https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/[ReplicaSet] in the Kubernetes documentation. + -NOTE: [.noloc]`Kubernetes` maintains the number of replicas that are specified in the manifest. If this were a production deployment and you wanted [.noloc]`Kubernetes` to horizontally scale the number of replicas or vertically scale the compute resources for the [.noloc]`Pods`, use the <> and the <> to do so. +NOTE: Kubernetes maintains the number of replicas that are specified in the manifest. If this were a production deployment and you wanted Kubernetes to horizontally scale the number of replicas or vertically scale the compute resources for the Pods, use the <> and the <> to do so. . View the details of the deployed service. + [source,bash,subs="verbatim,attributes"] @@ -175,8 +170,8 @@ Session Affinity: None Events: ---- + -In the previous output, the value for `IP:` is a unique IP address that can be reached from any node or [.noloc]`Pod` within the cluster, but it can't be reached from outside of the cluster. The values for `Endpoints` are IP addresses assigned from within your VPC to the [.noloc]`Pods` that are part of the service. -. View the details of one of the [.noloc]`Pods` listed in the output when you <> in a previous step. Replace [.replaceable]`776d8f8fd8-78w66` with the value returned for one of your [.noloc]`Pods`. +In the previous output, the value for `IP:` is a unique IP address that can be reached from any node or Pod within the cluster, but it can't be reached from outside of the cluster. The values for `Endpoints` are IP addresses assigned from within your VPC to the Pods that are part of the service. +. View the details of one of the Pods listed in the output when you <> in a previous step. Replace [.replaceable]`776d8f8fd8-78w66` with the value returned for one of your Pods. + [source,bash,subs="verbatim,attributes"] ---- @@ -211,19 +206,19 @@ Events: [...] ---- + -In the previous output, the value for `IP:` is a unique IP that's assigned to the [.noloc]`Pod` from the CIDR block assigned to the subnet that the node is in. If you prefer to assign [.noloc]`Pods` IP addresses from different CIDR blocks, you can change the default behavior. For more information, see <>. You can also see that the [.noloc]`Kubernetes` scheduler scheduled the [.noloc]`Pod` on the `Node` with the IP address [.replaceable]`192.168.45.132`. +In the previous output, the value for `IP:` is a unique IP that's assigned to the Pod from the CIDR block assigned to the subnet that the node is in. If you prefer to assign Pods IP addresses from different CIDR blocks, you can change the default behavior. For more information, see <>. You can also see that the Kubernetes scheduler scheduled the Pod on the `Node` with the IP address [.replaceable]`192.168.45.132`. + -TIP: Rather than using the command line, you can view many details about [.noloc]`Pods`, services, deployments, and other [.noloc]`Kubernetes` resources in the {aws-management-console}. For more information, see <>. +TIP: Rather than using the command line, you can view many details about Pods, services, deployments, and other Kubernetes resources in the {aws-management-console}. For more information, see <>. == Run a shell on a Pod -. Run a shell on the [.noloc]`Pod` that you described in the previous step, replacing [.replaceable]`65b7669776-m6qxz` with the ID of one of your [.noloc]`Pods`. +. Run a shell on the Pod that you described in the previous step, replacing [.replaceable]`65b7669776-m6qxz` with the ID of one of your Pods. + [source,bash,subs="verbatim,attributes"] ---- kubectl exec -it eks-sample-linux-deployment-65b7669776-m6qxz -n eks-sample-app -- /bin/bash ---- -. From the [.noloc]`Pod` shell, view the output from the web server that was installed with your deployment in a previous step. You only need to specify the service name. It is resolved to the service's IP address by [.noloc]`CoreDNS`, which is deployed with an Amazon EKS cluster, by default. +. From the Pod shell, view the output from the web server that was installed with your deployment in a previous step. You only need to specify the service name. It is resolved to the service's IP address by CoreDNS, which is deployed with an Amazon EKS cluster, by default. + [source,bash,subs="verbatim,attributes"] ---- @@ -240,7 +235,7 @@ An example output is as follows. Welcome to nginx! [...] ---- -. From the [.noloc]`Pod` shell, view the DNS server for the [.noloc]`Pod`. +. From the Pod shell, view the DNS server for the Pod. + [source,bash,subs="verbatim,attributes"] ---- @@ -256,8 +251,8 @@ search eks-sample-app.svc.cluster.local svc.cluster.local cluster.local us-west- options ndots:5 ---- + -In the previous output, `10.100.0.10` is automatically assigned as the `nameserver` for all [.noloc]`Pods` deployed to the cluster. -. Disconnect from the [.noloc]`Pod` by typing `exit`. +In the previous output, `10.100.0.10` is automatically assigned as the `nameserver` for all Pods deployed to the cluster. +. Disconnect from the Pod by typing `exit`. . Once you're finished with the sample application, you can remove the sample namespace, service, and deployment with the following command. + [source,bash,subs="verbatim,attributes"] @@ -266,7 +261,7 @@ kubectl delete namespace eks-sample-app ---- -[[sample-deployment-next-steps,sample-deployment-next-steps.title]] +[#sample-deployment-next-steps] == Next Steps After you deploy the sample application, you might want to try some of the following exercises: @@ -274,4 +269,4 @@ After you deploy the sample application, you might want to try some of the follo * <> -* <> +* <> \ No newline at end of file diff --git a/latest/ug/workloads/update-addon-role.adoc b/latest/ug/workloads/update-addon-role.adoc index 2210c3f46..c36134aad 100644 --- a/latest/ug/workloads/update-addon-role.adoc +++ b/latest/ug/workloads/update-addon-role.adoc @@ -1,10 +1,9 @@ -//!!NODE_ROOT
+include::../attributes.txt[] + [.topic] -[[update-addon-role,update-addon-role.title]] +[#update-addon-role] = Use Pod Identities to assign an IAM role to an Amazon EKS add-on -:info_titleabbrev: Use Pod Identities to assign an IAM role - -include::../attributes.txt[] +:info_titleabbrev: Use Pod Identities [abstract] -- @@ -59,6 +58,4 @@ If successful, you should see output similar to the following. Note the OwnerARN } ] } ----- - - +---- \ No newline at end of file diff --git a/latest/ug/workloads/updating-an-add-on.adoc b/latest/ug/workloads/updating-an-add-on.adoc index 56f8714e1..af3586a85 100644 --- a/latest/ug/workloads/updating-an-add-on.adoc +++ b/latest/ug/workloads/updating-an-add-on.adoc @@ -1,21 +1,18 @@ -//!!NODE_ROOT
+include::../attributes.txt[] [.topic] -[[updating-an-add-on,updating-an-add-on.title]] +[#updating-an-add-on] = Update an Amazon EKS add-on :info_titleabbrev: Update an add-on -include::../attributes.txt[] - - [abstract] -- Learn how to update your Amazon EKS add-on to a new version. -- -Amazon EKS doesn't automatically update an add-on when new versions are released or after you update your cluster to a new [.noloc]`Kubernetes` minor version. To update an add-on for an existing cluster, you must initiate the update. After you initiate the update, Amazon EKS updates the add-on for you. Before updating an add-on, review the current documentation for the add-on. For a list of available add-ons, see <>. If the add-on requires an IAM role, see the details for the specific add-on in <> for details about creating the role. +Amazon EKS doesn't automatically update an add-on when new versions are released or after you update your cluster to a new Kubernetes minor version. To update an add-on for an existing cluster, you must initiate the update. After you initiate the update, Amazon EKS updates the add-on for you. Before updating an add-on, review the current documentation for the add-on. For a list of available add-ons, see <>. If the add-on requires an IAM role, see the details for the specific add-on in <> for details about creating the role. -[[updating-an-add-on-prereq,updating-an-add-on-prereq.title]] +[#updating-an-add-on-prereq] == Prerequisites Complete the following before you create an add-on: @@ -26,7 +23,7 @@ Complete the following before you create an add-on: * Verify that the Amazon EKS add-on version is compatible with your cluster. For more information, see <>. -[[updating-an-add-on-procedure,updating-an-add-on-procedure.title]] +[#updating-an-add-on-procedure] == Procedure You can update an Amazon EKS add-on using `eksctl`, the {aws-management-console}, or the {aws} CLI. @@ -58,9 +55,9 @@ Your output might look different, depending on which add-ons and versions that y *** Replace [.replaceable]`my-cluster` with the name of your cluster. *** Replace [.replaceable]`region-code` with the {aws} Region that your cluster is in. *** Replace [.replaceable]`vpc-cni` with the name of an add-on returned in the output of the previous step that you want to update. -*** If you want to update to a version earlier than the latest available version, then replace [.replaceable]`latest` with the version number returned in the output of the previous step that you want to use. Some add-ons have recommended versions. For more information, see the documentation for the add-on that you're updating. For a list of add-ons, see <>.*** If the add-on uses a [.noloc]`Kubernetes` service account and IAM role, replace [.replaceable]`111122223333` with your account ID and [.replaceable]`role-name` with the name of an existing IAM role that you've created. For instructions on creating the role, see the documentation for the add-on that you're creating. For a list of add-ons, see <>. Specifying a service account role requires that you have an IAM [.noloc]`OpenID Connect` ([.noloc]`OIDC`) provider for your cluster. To determine whether you have one for your cluster, or to create one, see <>. +*** If you want to update to a version earlier than the latest available version, then replace [.replaceable]`latest` with the version number returned in the output of the previous step that you want to use. Some add-ons have recommended versions. For more information, see the documentation for the add-on that you're updating. For a list of add-ons, see <>.*** If the add-on uses a Kubernetes service account and IAM role, replace [.replaceable]`111122223333` with your account ID and [.replaceable]`role-name` with the name of an existing IAM role that you've created. For instructions on creating the role, see the documentation for the add-on that you're creating. For a list of add-ons, see <>. Specifying a service account role requires that you have an IAM OpenID Connect (OIDC) provider for your cluster. To determine whether you have one for your cluster, or to create one, see <>. + -If the add-on doesn't use a [.noloc]`Kubernetes` service account and IAM role, delete the `serviceAccountRoleARN: {arn-aws}iam::[.replaceable]``111122223333``:role/[.replaceable]``role-name``` line. +If the add-on doesn't use a Kubernetes service account and IAM role, delete the `serviceAccountRoleARN: {arn-aws}iam::[.replaceable]``111122223333``:role/[.replaceable]``role-name``` line. *** The [.replaceable]`preserve` option preserves existing values for the add-on. If you have set custom values for add-on settings, and you don't use this option, Amazon EKS overwrites your values with its default values. If you use this option, then we recommend that you test any field and value changes on a non-production cluster before updating the add-on on your production cluster. If you change this value to `overwrite`, all settings are changed to Amazon EKS default values. If you've set custom values for any settings, they might be overwritten with Amazon EKS default values. If you change this value to `none`, Amazon EKS doesn't change the value of any settings, but the update might fail. If the update fails, you receive an error message to help you resolve the conflict. + [source,yaml,subs="verbatim,attributes"] @@ -102,23 +99,23 @@ For more information about updating add-ons, see https://eksctl.io/usage/addons/ . On the *Configure [.replaceable]`name of addon`* page, do the following: + .. Choose the *Version* that you'd like to use. The add-on might have a recommended version. For more information, see the documentation for the add-on that you're updating. For a list of add-ons, see <>. -.. You have two options for configuring roles for add-ons: EKS Pod Identities IAM role and IAM roles for service accounts (IRSA). Follow the appropriate step below for your preferred option. If all of the add-ons that you selected have *Requires subscription* under *Status*, choose *Next*. For the add-ons that don't have *Requires subscription* under *Status*, do the following: +.. You have two options for configuring roles for add-ons: EKS Pod Identities IAM role and IAM roles for service accounts (IRSA). Follow the appropriate step below for your preferred option. If all of the add-ons that you selected have *Requires subscription* under *Status*, choose *Next*. For the add-ons that don't have *Requires subscription* under *Status*, do the following: + ... For *Pod Identity IAM role for service account*, you can either use an existing EKS Pod Identity IAM role or create one using the *Create Recommended Role* button. This field will only provide options with the appropriate trust policy. If there's no role to select, then you don't have an existing role with a matching trust policy. To configure an EKS Pod Identity IAM role for service accounts of the selected add-on, choose *Create recommended role*. The role creation wizard opens in a separate window. The wizard will automatically populate the role information as follows. For each add-on where you want to create the EKS Pod Identity IAM role, complete the steps in the IAM wizard as follows. * On the *Select trusted entity* step, the {aws} service option for *EKS* and the use case for *EKS - Pod Identity* are preselected, and the appropriate trust policy will be automatically populated for the add-on. For example, the role will be created with the appropriate trust policy containing the pods.eks.amazonaws.com IAM Principal as detailed in <>. Choose *Next*. -* On the *Add permissions* step, the appropriate managed policy for the role policy is preselected for the add-on. For example, for the Amazon VPC CNI add-on, the role will be created with the managed policy ` AmazonEKS_CNI_Policy` as detailed in <>. Choose *Next*. +* On the *Add permissions* step, the appropriate managed policy for the role policy is preselected for the add-on. For example, for the Amazon VPC CNI add-on, the role will be created with the managed policy `AmazonEKS_CNI_Policy` as detailed in <>. Choose *Next*. * On the *Name, review, and create* step, in *Role name*, the default role name is automatically populated for the add-on. For example, for the *Amazon VPC CNI* add-on, the role will be created with the name *AmazonEKSPodIdentityAmazonVPCCNIRole*. In *Description*, the default description is automatically populated with the appropriate description for the add-on. For example, for the Amazon VPC CNI add-on, the role will be created with the description *Allows pods running in Amazon EKS cluster* to access {aws} resources. In *Trust policy*, view the populated trust policy for the add-on. Choose *Create role*. + NOTE: Retaining the default role name enables EKS to pre-select the role for add-ons in new clusters or when adding add-ons to existing clusters. You can still override this name and the role will be available for the add-on across your clusters, but the role will need to be manually selected from the drop down. -... For add-ons that do not have *Requires subscription* under *Status* and where you want to configure roles using IRSA, see the documentation for the add-on that you're creating to create an IAM policy and attach it to a role. For a list of add-ons, see <>. Selecting an IAM role requires that you have an IAM [.noloc]`OpenID Connect` ([.noloc]`OIDC`) provider for your cluster. To determine whether you have one for your cluster, or to create one, see <>. +... For add-ons that do not have *Requires subscription* under *Status* and where you want to configure roles using IRSA, see the documentation for the add-on that you're creating to create an IAM policy and attach it to a role. For a list of add-ons, see <>. Selecting an IAM role requires that you have an IAM OpenID Connect (OIDC) provider for your cluster. To determine whether you have one for your cluster, or to create one, see <>. .. Expand the *Optional configuration settings*. -.. In *Configuration values*, enter any add-on specific configuration information. For more information, see the documentation for the add-on that you're updating. For a list of add-ons, see <>... For *Conflict resolution method*, select one of the options. If you have set custom values for add-on settings, we recommend the *Preserve* option. If you don't choose this option, Amazon EKS overwrites your values with its default values. If you use this option, then we recommend that you test any field and value changes on a non-production cluster before updating the add-on on your production cluster. +.. In *Configuration values*, enter any add-on specific configuration information. For more information, see the documentation for the add-on that you're updating. For a list of add-ons, see <>... For *Conflict resolution method*, select one of the options. If you have set custom values for add-on settings, we recommend the *Preserve* option. If you don't choose this option, Amazon EKS overwrites your values with its default values. If you use this option, then we recommend that you test any field and value changes on a non-production cluster before updating the add-on on your production cluster. If you change this value to overwrite, all settings are changed to Amazon EKS default values. If you've set custom values for any settings, they might be overwritten with Amazon EKS default values. If you change this value to none, Amazon EKS doesn't change the value of any settings, but the update might fail. If the update fails, you receive an error message to help you resolve the conflict. . Choose *Save changes*. == Update add-on ({aws} CLI) -. You need version `2.12.3` or later or version `1.27.160` or later of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device or {aws} CloudShell. To check your current version, use `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or [.noloc]`Homebrew` for [.noloc]`macOS` are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the _{aws} Command Line Interface User Guide_. The {aws} CLI version that is installed in {aws} CloudShell might also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the _{aws} CloudShell User Guide_. +. You need version `2.12.3` or later or version `1.27.160` or later of the {aws} Command Line Interface ({aws} CLI) installed and configured on your device or {aws} CloudShell. To check your current version, use `aws --version | cut -d / -f2 | cut -d ' ' -f1`. Package managers such `yum`, `apt-get`, or Homebrew for macOS are often several versions behind the latest version of the {aws} CLI. To install the latest version, see link:cli/latest/userguide/cli-chap-install.html[Installing, updating, and uninstalling the {aws} CLI,type="documentation"] and link:cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config[Quick configuration with aws configure,type="documentation"] in the _{aws} Command Line Interface User Guide_. The {aws} CLI version that is installed in {aws} CloudShell might also be several versions behind the latest version. To update it, see link:cloudshell/latest/userguide/vm-specs.html#install-cli-software[Installing {aws} CLI to your home directory,type="documentation"] in the _{aws} CloudShell User Guide_. + . See a list of installed add-ons. Replace [.replaceable]`my-cluster` with the name of your cluster. + @@ -129,7 +126,7 @@ aws eks list-addons --cluster-name my-cluster + An example output is as follows. + -[source,json,subs="verbatim,attributes"] +[source,json,subs="verbatim,attributes",role="nocopy"] ---- { "addons": [ @@ -152,11 +149,11 @@ An example output is as follows. ---- v1.10.4-eksbuild.1 ---- -. Determine which versions of the add-on are available for your cluster's version. Replace [.replaceable]`1.30` with your cluster's version and [.replaceable]`vpc-cni` with the name of the add-on that you want to update. +. Determine which versions of the add-on are available for your cluster's version. Replace [.replaceable]`{k8s-n}` with your cluster's version and [.replaceable]`vpc-cni` with the name of the add-on that you want to update. + [source,bash,subs="verbatim,attributes"] ---- -aws eks describe-addon-versions --kubernetes-version 1.30 --addon-name vpc-cni \ +aws eks describe-addon-versions --kubernetes-version {k8s-n} --addon-name vpc-cni \ --query 'addons[].addonVersions[].{Version: addonVersion, Defaultversion: compatibilities[0].defaultVersion}' --output table ---- + @@ -181,9 +178,9 @@ The version with `True` in the `Defaultversion` column is the version that the a + *** Replace [.replaceable]`my-cluster` with the name of your cluster. *** Replace [.replaceable]`vpc-cni` with the name of the add-on that you want to update that was returned in the output of a previous step. -*** Replace [.replaceable]`version-number` with the version returned in the output of the previous step that you want to update to. Some add-ons have recommended versions. For more information, see the documentation for the add-on that you're updating. For a list of add-ons, see <>.*** If the add-on uses a [.noloc]`Kubernetes` service account and IAM role, replace [.replaceable]`111122223333` with your account ID and [.replaceable]`role-name` with the name of an existing IAM role that you've created. For instructions on creating the role, see the documentation for the add-on that you're creating. For a list of add-ons, see <>. Specifying a service account role requires that you have an IAM [.noloc]`OpenID Connect` ([.noloc]`OIDC`) provider for your cluster. To determine whether you have one for your cluster, or to create one, see <>. +*** Replace [.replaceable]`version-number` with the version returned in the output of the previous step that you want to update to. Some add-ons have recommended versions. For more information, see the documentation for the add-on that you're updating. For a list of add-ons, see <>.*** If the add-on uses a Kubernetes service account and IAM role, replace [.replaceable]`111122223333` with your account ID and [.replaceable]`role-name` with the name of an existing IAM role that you've created. For instructions on creating the role, see the documentation for the add-on that you're creating. For a list of add-ons, see <>. Specifying a service account role requires that you have an IAM OpenID Connect (OIDC) provider for your cluster. To determine whether you have one for your cluster, or to create one, see <>. + -If the add-on doesn't use a [.noloc]`Kubernetes` service account and IAM role, delete the `serviceAccountRoleARN: {arn-aws}iam::[.replaceable]``111122223333``:role/[.replaceable]``role-name``` line. +If the add-on doesn't use a Kubernetes service account and IAM role, delete the `serviceAccountRoleARN: {arn-aws}iam::[.replaceable]``111122223333``:role/[.replaceable]``role-name``` line. *** The `--resolve-conflicts PRESERVE` option preserves existing values for the add-on. If you have set custom values for add-on settings, and you don't use this option, Amazon EKS overwrites your values with its default values. If you use this option, then we recommend that you test any field and value changes on a non-production cluster before updating the add-on on your production cluster. If you change this value to `OVERWRITE`, all settings are changed to Amazon EKS default values. If you've set custom values for any settings, they might be overwritten with Amazon EKS default values. If you change this value to `NONE`, Amazon EKS doesn't change the value of any settings, but the update might fail. If the update fails, you receive an error message to help you resolve the conflict. *** If you want to remove all custom configuration then perform the update using the [.replaceable]`--configuration-values '{}'` option. This sets all custom configuration back to the default values. If you don't want to change your custom configuration, don't provide the [.replaceable]`--configuration-values` flag. If you want to adjust a custom configuration then replace [.replaceable]`{}` with the new parameters. + @@ -201,7 +198,7 @@ aws eks describe-addon --cluster-name my-cluster --addon-name vpc-cni + An example output is as follows. + -[source,json,subs="verbatim,attributes"] +[source,json,subs="verbatim,attributes",role="nocopy"] ---- { "addon": { @@ -212,5 +209,4 @@ An example output is as follows. } ---- + -The update is complete when the status is `ACTIVE`. - +The update is complete when the status is `ACTIVE`. \ No newline at end of file diff --git a/latest/ug/workloads/vertical-pod-autoscaler.adoc b/latest/ug/workloads/vertical-pod-autoscaler.adoc index 77a0ce920..721a11db2 100644 --- a/latest/ug/workloads/vertical-pod-autoscaler.adoc +++ b/latest/ug/workloads/vertical-pod-autoscaler.adoc @@ -1,36 +1,32 @@ -//!!NODE_ROOT
include::../attributes.txt[] [.topic] -[[vertical-pod-autoscaler,vertical-pod-autoscaler.title]] -= Adjust pod resources with [.noloc]`Vertical Pod Autoscaler` -:info_doctype: section -:info_title: Adjust pod resources with Vertical Pod Autoscaler +[#vertical-pod-autoscaler] += Adjust pod resources with Vertical Pod Autoscaler :info_titleabbrev: Vertical Pod Autoscaler -:info_abstract: Discover how the Kubernetes Vertical Pod Autoscaler automatically adjusts CPU and memory reservations for your Pods to optimize resource utilization and right-size applications on Amazon EKS. [abstract] -- Discover how the Kubernetes Vertical Pod Autoscaler automatically adjusts CPU and memory reservations for your Pods to optimize resource utilization and right-size applications on Amazon EKS. -- -The [.noloc]`Kubernetes` https://github.com/kubernetes/autoscaler/tree/master/vertical-pod-autoscaler[Vertical Pod Autoscaler] automatically adjusts the CPU and memory reservations for your [.noloc]`Pods` to help "right size" your applications. This adjustment can improve cluster resource utilization and free up CPU and memory for other [.noloc]`Pods`. This topic helps you to deploy the Vertical Pod Autoscaler to your cluster and verify that it is working. +The Kubernetes https://github.com/kubernetes/autoscaler/tree/master/vertical-pod-autoscaler[Vertical Pod Autoscaler] automatically adjusts the CPU and memory reservations for your Pods to help "right size" your applications. This adjustment can improve cluster resource utilization and free up CPU and memory for other Pods. This topic helps you to deploy the Vertical Pod Autoscaler to your cluster and verify that it is working. * You have an existing Amazon EKS cluster. If you don't, see <>. -* You have the [.noloc]`Kubernetes` Metrics Server installed. For more information, see <>. +* You have the Kubernetes Metrics Server installed. For more information, see <>. * You are using a `kubectl` client that is <>. * OpenSSL `1.1.1` or later installed on your device. -[[vpa-deploy,vpa-deploy.title]] +[#vpa-deploy] == Deploy the Vertical Pod Autoscaler In this section, you deploy the Vertical Pod Autoscaler to your cluster. . Open a terminal window and navigate to a directory where you would like to download the Vertical Pod Autoscaler source code. -. Clone the https://github.com/kubernetes/autoscaler[kubernetes/autoscaler][.noloc]`GitHub` repository. +. Clone the https://github.com/kubernetes/autoscaler[kubernetes/autoscaler]GitHub repository. + [source,bash,subs="verbatim,attributes"] ---- @@ -71,7 +67,7 @@ sed -i.bak -e 's/registry.k8s.io/111122223333.dkr.ecr.region-code.amazonaws.com/ ---- ./hack/vpa-up.sh ---- -. Verify that the Vertical Pod Autoscaler [.noloc]`Pods` have been created successfully. +. Verify that the Vertical Pod Autoscaler Pods have been created successfully. + [source,bash,subs="verbatim,attributes"] ---- @@ -91,7 +87,7 @@ vpa-updater-786b96955c-bgp9d 1/1 Running 0 8s ---- -[[vpa-sample-app,vpa-sample-app.title]] +[#vpa-sample-app] == Test your Vertical Pod Autoscaler installation In this section, you deploy a sample application to verify that the Vertical Pod Autoscaler is working. @@ -102,7 +98,7 @@ In this section, you deploy a sample application to verify that the Vertical Pod ---- kubectl apply -f examples/hamster.yaml ---- -. Get the [.noloc]`Pods` from the `hamster` example application. +. Get the Pods from the `hamster` example application. + [source,bash,subs="verbatim,attributes"] ---- @@ -116,7 +112,7 @@ An example output is as follows. hamster-c7d89d6db-rglf5 1/1 Running 0 48s hamster-c7d89d6db-znvz5 1/1 Running 0 48s ---- -. Describe one of the [.noloc]`Pods` to view its `cpu` and `memory` reservation. Replace [.replaceable]`c7d89d6db-rglf5` with one of the IDs returned in your output from the previous step. +. Describe one of the Pods to view its `cpu` and `memory` reservation. Replace [.replaceable]`c7d89d6db-rglf5` with one of the IDs returned in your output from the previous step. + [source,bash,subs="verbatim,attributes"] ---- @@ -150,16 +146,16 @@ Containers: [...] ---- + -You can see that the original [.noloc]`Pod` reserves 100 millicpu of CPU and 50 mebibytes of memory. For this example application, 100 millicpu is less than the [.noloc]`Pod` needs to run, so it is CPU-constrained. It also reserves much less memory than it needs. The Vertical Pod Autoscaler `vpa-recommender` deployment analyzes the [.noloc]`hamster` [.noloc]`Pods` to see if the CPU and memory requirements are appropriate. If adjustments are needed, the `vpa-updater` relaunches the [.noloc]`Pods` with updated values. -. Wait for the `vpa-updater` to launch a new [.noloc]`hamster` [.noloc]`Pods`. This should take a minute or two. You can monitor the [.noloc]`Pods` with the following command. +You can see that the original Pod reserves 100 millicpu of CPU and 50 mebibytes of memory. For this example application, 100 millicpu is less than the Pod needs to run, so it is CPU-constrained. It also reserves much less memory than it needs. The Vertical Pod Autoscaler `vpa-recommender` deployment analyzes the hamster Pods to see if the CPU and memory requirements are appropriate. If adjustments are needed, the `vpa-updater` relaunches the Pods with updated values. +. Wait for the `vpa-updater` to launch a new hamster Pods. This should take a minute or two. You can monitor the Pods with the following command. + -NOTE: If you are not sure that a new [.noloc]`Pod` has launched, compare the [.noloc]`Pod` names with your previous list. When the new [.noloc]`Pod` launches, you will see a new [.noloc]`Pod` name. +NOTE: If you are not sure that a new Pod has launched, compare the Pod names with your previous list. When the new Pod launches, you will see a new Pod name. + [source,bash,subs="verbatim,attributes"] ---- kubectl get --watch Pods -l app=hamster ---- -. When a new [.noloc]`hamster` [.noloc]`Pods` is started, describe it and view the updated CPU and memory reservations. +. When a new hamster Pods is started, describe it and view the updated CPU and memory reservations. + [source,bash,subs="verbatim,attributes"] ---- @@ -193,7 +189,7 @@ Containers: [...] ---- + -In the previous output, you can see that the `cpu` reservation increased to 587 millicpu, which is over five times the original value. The `memory` increased to 262,144 Kilobytes, which is around 250 mebibytes, or five times the original value. This [.noloc]`Pod` was under-resourced, and the Vertical Pod Autoscaler corrected the estimate with a much more appropriate value. +In the previous output, you can see that the `cpu` reservation increased to 587 millicpu, which is over five times the original value. The `memory` increased to 262,144 Kilobytes, which is around 250 mebibytes, or five times the original value. This Pod was under-resourced, and the Vertical Pod Autoscaler corrected the estimate with a much more appropriate value. . Describe the `hamster-vpa` resource to view the new recommendation. + [source,bash,subs="verbatim,attributes"] @@ -250,4 +246,4 @@ Events: [source,bash,subs="verbatim,attributes"] ---- kubectl delete -f examples/hamster.yaml ----- +---- \ No newline at end of file diff --git a/latest/ug/workloads/workloads-add-ons-available-eks.adoc b/latest/ug/workloads/workloads-add-ons-available-eks.adoc index 4ff6f0c89..0b270fbb2 100644 --- a/latest/ug/workloads/workloads-add-ons-available-eks.adoc +++ b/latest/ug/workloads/workloads-add-ons-available-eks.adoc @@ -1,42 +1,38 @@ -//!!NODE_ROOT
- include::../attributes.txt[] [.topic] -[[workloads-add-ons-available-eks,workloads-add-ons-available-eks.title]] -= {aws} Add-ons -:info_titleabbrev: {aws} Add-ons - -include::../attributes.txt[] +[#workloads-add-ons-available-eks] += {aws} add-ons +:info_titleabbrev: {aws} add-ons [abstract] -- Learn about the availabe Amazon EKS add-ons from {aws}. -- -The following Amazon EKS add-ons are available to create on your cluster. You can view the most current list of available add-ons using `eksctl`, the {aws-management-console}, or the {aws} CLI. To see all available add-ons or to install an add-on, see <>. If an add-on requires IAM permissions, then you must have an IAM [.noloc]`OpenID Connect` ([.noloc]`OIDC`) provider for your cluster. To determine whether you have one, or to create one, see <>. You can an create or delete an add-on after you've installed it. For more information, see <> or <>. For more information about considerations specific to running EKS add-ons with Amazon EKS Hybrid Nodes, see <>. +The following Amazon EKS add-ons are available to create on your cluster. You can view the most current list of available add-ons using `eksctl`, the {aws-management-console}, or the {aws} CLI. To see all available add-ons or to install an add-on, see <>. If an add-on requires IAM permissions, then you must have an IAM OpenID Connect (OIDC) provider for your cluster. To determine whether you have one, or to create one, see <>. You can an create or delete an add-on after you've installed it. For more information, see <> or <>. For more information about considerations specific to running EKS add-ons with Amazon EKS Hybrid Nodes, see <>. You can use any of the following Amazon EKS add-ons. [role="no-scroll"] -[cols="1,1,1", options="header"] +[%header,cols="3"] |=== - |Description |Learn more |Compatible compute types + | Provide native VPC networking for your cluster |<> |EC2 -| A flexible, extensible DNS server that can serve as the [.noloc]`Kubernetes` cluster DNS +| A flexible, extensible DNS server that can serve as the Kubernetes cluster DNS |<> -|EC2, Fargate, EKS Auto Mode, Amazon EKS Hybrid Nodes +|EC2, Fargate, EKS Auto Mode, EKS Hybrid Nodes |Maintain network rules on each Amazon EC2 node |<> -|EC2, Amazon EKS Hybrid Nodes +|EC2, EKS Hybrid Nodes |Provide Amazon EBS storage for your cluster |<> @@ -46,45 +42,66 @@ You can use any of the following Amazon EKS add-ons. |<> |EC2, EKS Auto Mode +|Provide Amazon FSx for Lustre storage for your cluster +|<> +|EC2, EKS Auto Mode + |Provide Amazon S3 storage for your cluster |<> |EC2, EKS Auto Mode |Detect additional node health issues |<> -|EC2 +|EC2, EKS Hybrid Nodes |Enable the use of snapshot functionality in compatible CSI drivers, such as the Amazon EBS CSI driver |<> -|EC2, Fargate, EKS Auto Mode, Amazon EKS Hybrid Nodes +|EC2, Fargate, EKS Auto Mode, EKS Hybrid Nodes |SageMaker HyperPod task governance optimizes compute resource allocation and usage across teams in Amazon EKS clusters, addressing inefficiencies in task prioritization and resource sharing. |<> |EC2, EKS Auto Mode, +|The Amazon SageMaker HyperPod Observability AddOn provides comprehensive monitoring and observability capabilities for HyperPod clusters. +|<> +|EC2, EKS Auto Mode, + +|Amazon SageMaker HyperPod training operator enables efficient distributed training on Amazon EKS clusters with advanced scheduling and resource management capabilities. +|<> +|EC2, EKS Auto Mode + |A Kubernetes agent that collects and reports network flow data to Amazon CloudWatch, enabling comprehensive monitoring of TCP connections across cluster nodes. |<> |EC2, EKS Auto Mode |Secure, production-ready, {aws} supported distribution of the OpenTelemetry project |<> -|EC2, Fargate, EKS Auto Mode, Amazon EKS Hybrid Nodes +|EC2, Fargate, EKS Auto Mode, EKS Hybrid Nodes -|Security monitoring service that analyzes and processes foundational data sources including {aws} CloudTrail management events and Amazon VPC flow logs. Amazon GuardDuty also processes features, such as [.noloc]`Kubernetes` audit logs and runtime monitoring +|Security monitoring service that analyzes and processes foundational data sources including {aws} CloudTrail management events and Amazon VPC flow logs. Amazon GuardDuty also processes features, such as Kubernetes audit logs and runtime monitoring |<> |EC2, EKS Auto Mode |Monitoring and observability service provided by {aws}. This add-on installs the CloudWatch Agent and enables both CloudWatch Application Signals and CloudWatch Container Insights with enhanced observability for Amazon EKS |<> -|EC2, EKS Auto Mode, Amazon EKS Hybrid Nodes +|EC2, EKS Auto Mode, EKS Hybrid Nodes |Ability to manage credentials for your applications, similar to the way that EC2 instance profiles provide credentials to EC2 instances |<> -|EC2, Amazon EKS Hybrid Nodes +|EC2, EKS Hybrid Nodes + +|Enable cert-manager to issue X.509 certificates from {aws} Private CA. Requires cert-manager. +|<> +|EC2, Fargate, EKS Auto Mode, EKS Hybrid Nodes + +|Generate Prometheus metrics about SR-IOV network device performance +|<> +|EC2 + |=== -[[add-ons-vpc-cni,add-ons-vpc-cni.title]] +[#add-ons-vpc-cni] == Amazon VPC CNI plugin for Kubernetes [abstract] @@ -92,7 +109,7 @@ You can use any of the following Amazon EKS add-ons. Learn about the vpc-cni Amazon EKS add-on. -- -The [.noloc]`Amazon VPC CNI plugin for Kubernetes` Amazon EKS add-on is a [.noloc]`Kubernetes` container network interface (CNI) plugin that provides native VPC networking for your cluster. The self-managed or managed type of this add-on is installed on each Amazon EC2 node, by default. For more information, see https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/[Kubernetes container network interface (CNI) plugin]. +The Amazon VPC CNI plugin for Kubernetes Amazon EKS add-on is a Kubernetes container network interface (CNI) plugin that provides native VPC networking for your cluster. The self-managed or managed type of this add-on is installed on each Amazon EC2 node, by default. For more information, see https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/[Kubernetes container network interface (CNI) plugin]. [NOTE] ==== @@ -102,14 +119,14 @@ You do not need to install this add-on on Amazon EKS Auto Mode clusters. For mor The Amazon EKS add-on name is `vpc-cni`. -[[add-ons-vpc-cni-iam-permissions,add-ons-vpc-cni-iam-permissions.title]] +[#add-ons-vpc-cni-iam-permissions] === Required IAM permissions This add-on uses the IAM roles for service accounts capability of Amazon EKS. For more information, see <>. -If your cluster uses the `IPv4` family, the permissions in the link:aws-managed-policy/latest/reference/AmazonEKS_CNI_Policy.html[AmazonEKS_CNI_Policy,type="documentation"] are required. If your cluster uses the `IPv6` family, you must link:IAM/latest/UserGuide/access_policies_create.html[create an IAM policy,type="documentation"] with the permissions in https://github.com/aws/amazon-vpc-cni-k8s/blob/master/docs/iam-policy.md#ipv6-mode[IPv6 mode]. You can create an IAM role, attach one of the policies to it, and annotate the [.noloc]`Kubernetes` service account used by the add-on with the following command. +If your cluster uses the `IPv4` family, the permissions in the link:aws-managed-policy/latest/reference/AmazonEKS_CNI_Policy.html[AmazonEKS_CNI_Policy,type="documentation"] are required. If your cluster uses the `IPv6` family, you must link:IAM/latest/UserGuide/access_policies_create.html[create an IAM policy,type="documentation"] with the permissions in https://github.com/aws/amazon-vpc-cni-k8s/blob/master/docs/iam-policy.md#ipv6-mode[IPv6 mode]. You can create an IAM role, attach one of the policies to it, and annotate the Kubernetes service account used by the add-on with the following command. -Replace [.replaceable]`my-cluster` with the name of your cluster and [.replaceable]`AmazonEKSVPCCNIRole` with the name for your role. If your cluster uses the `IPv6` family, then replace [.replaceable]`AmazonEKS_CNI_Policy` with the name of the policy that you created. This command requires that you have https://eksctl.io[eksctl] installed on your device. If you need to use a different tool to create the role, attach the policy to it, and annotate the [.noloc]`Kubernetes` service account, see <>. +Replace [.replaceable]`my-cluster` with the name of your cluster and [.replaceable]`AmazonEKSVPCCNIRole` with the name for your role. If your cluster uses the `IPv6` family, then replace [.replaceable]`AmazonEKS_CNI_Policy` with the name of the policy that you created. This command requires that you have https://eksctl.io[eksctl] installed on your device. If you need to use a different tool to create the role, attach the policy to it, and annotate the Kubernetes service account, see <>. [source,bash,subs="verbatim,attributes"] ---- @@ -118,12 +135,12 @@ eksctl create iamserviceaccount --name aws-node --namespace kube-system --cluste ---- -[[add-ons-vpc-cni-update-information,add-ons-vpc-cni-update-information.title]] +[#add-ons-vpc-cni-update-information] === Update information You can only update one minor version at a time. For example, if your current version is `1.28.[.replaceable]``x``-eksbuild.[.replaceable]``y``` and you want to update to `1.30.[.replaceable]``x``-eksbuild.[.replaceable]``y```, then you must update your current version to `1.29.[.replaceable]``x``-eksbuild.[.replaceable]``y``` and then update it again to `1.30.[.replaceable]``x``-eksbuild.[.replaceable]``y```. For more information about updating the add-on, see <>. -[[add-ons-coredns,add-ons-coredns.title]] +[#add-ons-coredns] == CoreDNS [abstract] @@ -131,7 +148,7 @@ You can only update one minor version at a time. For example, if your current ve Learn about the CoreDNS Amazon EKS add-on. -- -The CoreDNS Amazon EKS add-on is a flexible, extensible DNS server that can serve as the [.noloc]`Kubernetes` cluster DNS. The self-managed or managed type of this add-on was installed, by default, when you created your cluster. When you launch an Amazon EKS cluster with at least one node, two replicas of the [.noloc]`CoreDNS` image are deployed by default, regardless of the number of nodes deployed in your cluster. The [.noloc]`CoreDNS` [.noloc]`Pods` provide name resolution for all [.noloc]`Pods` in the cluster. You can deploy the [.noloc]`CoreDNS` [.noloc]`Pods` to Fargate nodes if your cluster includes a Fargate profile with a namespace that matches the namespace for the [.noloc]`CoreDNS` deployment. For more information, see <> +The CoreDNS Amazon EKS add-on is a flexible, extensible DNS server that can serve as the Kubernetes cluster DNS. The self-managed or managed type of this add-on was installed, by default, when you created your cluster. When you launch an Amazon EKS cluster with at least one node, two replicas of the CoreDNS image are deployed by default, regardless of the number of nodes deployed in your cluster. The CoreDNS Pods provide name resolution for all Pods in the cluster. You can deploy the CoreDNS Pods to Fargate nodes if your cluster includes a Fargate profile with a namespace that matches the namespace for the CoreDNS deployment. For more information, see <> [NOTE] ==== @@ -140,17 +157,17 @@ You do not need to install this add-on on Amazon EKS Auto Mode clusters. For mor The Amazon EKS add-on name is `coredns`. -[[add-ons-coredns-iam-permissions,add-ons-coredns-iam-permissions.title]] +[#add-ons-coredns-iam-permissions] === Required IAM permissions This add-on doesn't require any permissions. -[[add-ons-coredns-information,add-ons-coredns-information.title]] +[#add-ons-coredns-information] === Additional information -To learn more about CoreDNS, see https://kubernetes.io/docs/tasks/administer-cluster/coredns/[Using CoreDNS for Service Discovery] and https://kubernetes.io/docs/tasks/administer-cluster/dns-custom-nameservers/[Customizing DNS Service] in the [.noloc]`Kubernetes` documentation. +To learn more about CoreDNS, see https://kubernetes.io/docs/tasks/administer-cluster/coredns/[Using CoreDNS for Service Discovery] and https://kubernetes.io/docs/tasks/administer-cluster/dns-custom-nameservers/[Customizing DNS Service] in the Kubernetes documentation. -[[add-ons-kube-proxy,add-ons-kube-proxy.title]] +[#add-ons-kube-proxy] == `Kube-proxy` [abstract] @@ -158,7 +175,7 @@ To learn more about CoreDNS, see https://kubernetes.io/docs/tasks/administer-clu Learn about the Kube-proxy Amazon EKS add-on. -- -The `Kube-proxy` Amazon EKS add-on maintains network rules on each Amazon EC2 node. It enables network communication to your [.noloc]`Pods`. The self-managed or managed type of this add-on is installed on each Amazon EC2 node in your cluster, by default. +The `Kube-proxy` Amazon EKS add-on maintains network rules on each Amazon EC2 node. It enables network communication to your Pods. The self-managed or managed type of this add-on is installed on each Amazon EC2 node in your cluster, by default. [NOTE] ==== @@ -167,12 +184,12 @@ You do not need to install this add-on on Amazon EKS Auto Mode clusters. For mor The Amazon EKS add-on name is `kube-proxy`. -[[add-ons-kube-proxy-iam-permissions,add-ons-kube-proxy-iam-permissions.title]] +[#add-ons-kube-proxy-iam-permissions] === Required IAM permissions This add-on doesn't require any permissions. -[[add-ons-kube-proxy-update-information,add-ons-kube-proxy-update-information.title]] +[#add-ons-kube-proxy-update-information] === Update information Before updating your current version, consider the following requirements: @@ -182,12 +199,12 @@ Before updating your current version, consider the following requirements: * `Kube-proxy` on an Amazon EKS cluster has the same https://kubernetes.io/releases/version-skew-policy/#kube-proxy[compatibility and skew policy as Kubernetes]. -[[add-ons-kube-proxy-information,add-ons-kube-proxy-information.title]] +[#add-ons-kube-proxy-information] === Additional information -To learn more about `kube-proxy`, see https://kubernetes.io/docs/reference/command-line-tools-reference/kube-proxy/[kube-proxy] in the [.noloc]`Kubernetes` documentation. +To learn more about `kube-proxy`, see https://kubernetes.io/docs/reference/command-line-tools-reference/kube-proxy/[kube-proxy] in the Kubernetes documentation. -[[add-ons-aws-ebs-csi-driver,add-ons-aws-ebs-csi-driver.title]] +[#add-ons-aws-ebs-csi-driver] == Amazon EBS CSI driver [abstract] @@ -195,7 +212,7 @@ To learn more about `kube-proxy`, see https://kubernetes.io/docs/reference/comma Learn about the Amazon EBS CSI driver Amazon EKS add-on. -- -The Amazon EBS CSI driver Amazon EKS add-on is a [.noloc]`Kubernetes` Container Storage Interface (CSI) plugin that provides Amazon EBS storage for your cluster. +The Amazon EBS CSI driver Amazon EKS add-on is a Kubernetes Container Storage Interface (CSI) plugin that provides Amazon EBS storage for your cluster. [NOTE] ==== @@ -204,7 +221,7 @@ You do not need to install this add-on on Amazon EKS Auto Mode clusters. Auto Mo The Amazon EKS add-on name is `aws-ebs-csi-driver`. -[[add-ons-aws-ebs-csi-driver-iam-permissions,add-ons-aws-ebs-csi-driver-iam-permissions.title]] +[#add-ons-aws-ebs-csi-driver-iam-permissions] === Required IAM permissions This add-on utilizes the IAM roles for service accounts capability of Amazon EKS. For more information, see <>. The permissions in the link:aws-managed-policy/latest/reference/AmazonEBSCSIDriverPolicy.html[AmazonEBSCSIDriverPolicy,type="documentation"] {aws} managed policy are required. You can create an IAM role and attach the managed policy to it with the following command. Replace [.replaceable]`my-cluster` with the name of your cluster and [.replaceable]`AmazonEKS_EBS_CSI_DriverRole` with the name for your role. This command requires that you have https://eksctl.io[eksctl] installed on your device. If you need to use a different tool or you need to use a custom link:kms/[KMS key,type="marketing"] for encryption, see <>. @@ -222,12 +239,12 @@ eksctl create iamserviceaccount \ ---- -[[add-ons-aws-ebs-csi-driver-information,add-ons-aws-ebs-csi-driver-information.title]] +[#add-ons-aws-ebs-csi-driver-information] === Additional information To learn more about the add-on, see <>. -[[add-ons-aws-efs-csi-driver,add-ons-aws-efs-csi-driver.title]] +[#add-ons-aws-efs-csi-driver] == Amazon EFS CSI driver [abstract] @@ -235,11 +252,11 @@ To learn more about the add-on, see <>. Learn about the Amazon EFS CSI driver Amazon EKS add-on. -- -The Amazon EFS CSI driver Amazon EKS add-on is a [.noloc]`Kubernetes` Container Storage Interface (CSI) plugin that provides Amazon EFS storage for your cluster. +The Amazon EFS CSI driver Amazon EKS add-on is a Kubernetes Container Storage Interface (CSI) plugin that provides Amazon EFS storage for your cluster. The Amazon EKS add-on name is `aws-efs-csi-driver`. -[[add-ons-aws-efs-csi-driver-iam-permissions,add-ons-aws-efs-csi-driver-iam-permissions.title]] +[#add-ons-aws-efs-csi-driver-iam-permissions] === Required IAM permissions *Required IAM permissions* – This add-on utilizes the IAM roles for service accounts capability of Amazon EKS. For more information, see <>. The permissions in the link:aws-managed-policy/latest/reference/AmazonEFSCSIDriverPolicy.html[AmazonEFSCSIDriverPolicy,type="documentation"] {aws} managed policy are required. You can create an IAM role and attach the managed policy to it with the following commands. Replace [.replaceable]`my-cluster` with the name of your cluster and [.replaceable]`AmazonEKS_EFS_CSI_DriverRole` with the name for your role. These commands require that you have https://eksctl.io[eksctl] installed on your device. If you need to use a different tool, see <>. @@ -256,35 +273,82 @@ eksctl create iamserviceaccount \ --role-only \ --attach-policy-arn {arn-aws}iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy \ --approve -TRUST_POLICY=$(aws iam get-role --role-name $role_name --query 'Role.AssumeRolePolicyDocument' | \ +TRUST_POLICY=$(aws iam get-role --output json --role-name $role_name --query 'Role.AssumeRolePolicyDocument' | \ sed -e 's/efs-csi-controller-sa/efs-csi-*/' -e 's/StringEquals/StringLike/') aws iam update-assume-role-policy --role-name $role_name --policy-document "$TRUST_POLICY" ---- -[[add-ons-aws-efs-csi-driver-information,add-ons-aws-efs-csi-driver-information.title]] +[#add-ons-aws-efs-csi-driver-information] === Additional information To learn more about the add-on, see <>. -[[mountpoint-for-s3-add-on,mountpoint-for-s3-add-on.title]] -== [.noloc]`Mountpoint` for Amazon S3 CSI Driver +[#add-ons-aws-fsx-csi-driver] +== Amazon FSx CSI driver [abstract] -- -Learn about the [.noloc]`Mountpoint` for Amazon S3 CSI Driver Amazon EKS add-on. +Learn about the Amazon FSx CSI driver Amazon EKS add-on. -- -The [.noloc]`Mountpoint` for Amazon S3 CSI Driver Amazon EKS add-on is a [.noloc]`Kubernetes` Container Storage Interface (CSI) plugin that provides Amazon S3 storage for your cluster. +The Amazon FSx CSI driver Amazon EKS add-on is a Kubernetes Container Storage Interface (CSI) plugin that provides Amazon FSx for Lustre storage for your cluster. + +The Amazon EKS add-on name is `aws-fsx-csi-driver`. + +[NOTE] +==== +* Pre-existing Amazon FSx CSI driver installations in the cluster can cause add-on installation failures. When you attempt to install the Amazon EKS add-on version while a non-EKS FSx CSI Driver exists, the installation will fail due to resource conflicts. Use the `OVERWRITE` flag during installation to resolve this issue: ++ +[source,bash] +---- +aws eks create-addon --addon-name aws-fsx-csi-driver --cluster-name my-cluster --resolve-conflicts OVERWRITE +---- + +* The Amazon FSx CSI Driver EKS add-on requires the EKS Pod Identity agent for authentication. Without this component, the add-on will fail with the error `Amazon EKS Pod Identity agent is not installed in the cluster`, preventing volume operations. Install the Pod Identity agent before or after deploying the FSx CSI Driver add-on. For more information, see <>. + +==== + +[#add-ons-aws-fsx-csi-driver-iam-permissions] +=== Required IAM permissions + +This add-on utilizes the IAM roles for service accounts capability of Amazon EKS. For more information, see <>. The permissions in the link:aws-managed-policy/latest/reference/AmazonFSxFullAccess.html[AmazonFSxFullAccess,type="documentation"] {aws} managed policy are required. You can create an IAM role and attach the managed policy to it with the following command. Replace [.replaceable]`my-cluster` with the name of your cluster and [.replaceable]`AmazonEKS_FSx_CSI_DriverRole` with the name for your role. This command requires that you have https://eksctl.io[eksctl] installed on your device. + +[source,bash,subs="verbatim,attributes"] +---- +eksctl create iamserviceaccount \ + --name fsx-csi-controller-sa \ + --namespace kube-system \ + --cluster my-cluster \ + --role-name AmazonEKS_FSx_CSI_DriverRole \ + --role-only \ + --attach-policy-arn {arn-aws}iam::aws:policy/AmazonFSxFullAccess \ + --approve +---- + +[#add-ons-aws-fsx-csi-driver-information] +=== Additional information + +To learn more about the add-on, see <>. + +[#mountpoint-for-s3-add-on] +== Mountpoint for Amazon S3 CSI Driver + +[abstract] +-- +Learn about the Mountpoint for Amazon S3 CSI Driver Amazon EKS add-on. +-- + +The Mountpoint for Amazon S3 CSI Driver Amazon EKS add-on is a Kubernetes Container Storage Interface (CSI) plugin that provides Amazon S3 storage for your cluster. The Amazon EKS add-on name is `aws-mountpoint-s3-csi-driver`. -[[add-ons-mountpoint-for-s3-add-on-iam-permissions,add-ons-mountpoint-for-s3-add-on-iam-permissions.title]] +[#add-ons-mountpoint-for-s3-add-on-iam-permissions] === Required IAM permissions This add-on uses the IAM roles for service accounts capability of Amazon EKS. For more information, see <>. -The IAM role that is created will require a policy that gives access to S3. Follow the https://github.com/awslabs/mountpoint-s3/blob/main/doc/CONFIGURATION.md#iam-permissions[Mountpoint IAM permissions recommendations] when creating the policy. Alternatively, you may use the {aws} managed policy link:iam/home?#/policies/arn:aws:iam::aws:policy/AmazonS3FullAccess$jsonEditor[AmazonS3FullAccess,type="console"], but this managed policy grants more permissions than are needed for [.noloc]`Mountpoint`. +The IAM role that is created will require a policy that gives access to S3. Follow the https://github.com/awslabs/mountpoint-s3/blob/main/doc/CONFIGURATION.md#iam-permissions[Mountpoint IAM permissions recommendations] when creating the policy. Alternatively, you may use the {aws} managed policy link:iam/home?#/policies/arn:aws:iam::aws:policy/AmazonS3FullAccess$jsonEditor[AmazonS3FullAccess,type="console"], but this managed policy grants more permissions than are needed for Mountpoint. You can create an IAM role and attach your policy to it with the following commands. Replace [.replaceable]`my-cluster` with the name of your cluster, [.replaceable]`region-code` with the correct {aws} Region code, [.replaceable]`AmazonEKS_S3_CSI_DriverRole` with the name for your role, and [.replaceable]`AmazonEKS_S3_CSI_DriverRole_ARN` with the role ARN. These commands require that you have https://eksctl.io[eksctl] installed on your device. For instructions on using the IAM console or {aws} CLI, see <>. @@ -306,12 +370,13 @@ eksctl create iamserviceaccount \ ---- -[[add-ons-mountpoint-for-s3-add-on-information,add-ons-mountpoint-for-s3-add-on-information.title]] +[#add-ons-mountpoint-for-s3-add-on-information] === Additional information To learn more about the add-on, see <>. -[[addons-csi-snapshot-controller,addons-csi-snapshot-controller.title]] + +[#addons-csi-snapshot-controller] == CSI snapshot controller [abstract] @@ -323,18 +388,18 @@ The Container Storage Interface (CSI) snapshot controller enables the use of sna The Amazon EKS add-on name is `snapshot-controller`. -[[add-ons-csi-snapshot-controller-iam-permissions,add-ons-csi-snapshot-controller-iam-permissions.title]] +[#add-ons-csi-snapshot-controller-iam-permissions] === Required IAM permissions This add-on doesn't require any permissions. -[[add-ons-csi-snapshot-controller-information,add-ons-csi-snapshot-controller-information.title]] +[#add-ons-csi-snapshot-controller-information] === Additional information To learn more about the add-on, see <>. -[[addons-hyperpod,addons-hyperpod.title]] +[#addons-hyperpod] == Amazon SageMaker HyperPod task governance SageMaker HyperPod task governance is a robust management system designed to streamline resource allocation and ensure efficient utilization of compute resources across teams and projects for your Amazon EKS clusters. This provides administrators with the capability to set: @@ -356,7 +421,50 @@ This add-on doesn't require any permissions. To learn more about the add-on, see link:sagemaker/latest/dg/sagemaker-hyperpod-eks-operate-console-ui-governance.html["SageMaker HyperPod task governance",type="documentation"] -[[addons-network-flow,addons-network-flow.title]] +[#addons-hyperpod-observability] +== Amazon SageMaker HyperPod Observability Add-on + +The Amazon SageMaker HyperPod Observability Add-on provides comprehensive monitoring and observability capabilities for HyperPod clusters. This add-on automatically deploys and manages essential monitoring components including node exporter, DCGM exporter, kube-state-metrics, and EFA exporter. It collects and forwards metrics to a customer-designated Amazon Managed Prometheus (AMP) instance and exposes an OTLP endpoint for custom metrics and event ingestion from customer training jobs. + +The add-on integrates with the broader HyperPod ecosystem by scraping metrics from various components including HyperPod Task Governance add-on, HyperPod Training Operator, Kubeflow, and KEDA. All collected metrics are centralized in Amazon Managed Prometheus, enabling customers to achieve a unified observability view through Amazon Managed Grafana dashboards. This provides end-to-end visibility into cluster health, resource utilization, and training job performance across the entire HyperPod environment. + +The Amazon EKS add-on name is `amazon-sagemaker-hyperpod-observability`. + +=== Required IAM permissions + +This add-on uses the IAM roles for service accounts capability of Amazon EKS. For more information, see <>. The following managed policies are required: + +* `AmazonPrometheusRemoteWriteAccess` - for remote writing metrics from the cluster to AMP +* `CloudWatchAgentServerPolicy` - for remote writing the logs from the cluster to CloudWatch + +=== Additional information + +To learn more about the add-on and its capabilities, see link:sagemaker/latest/dg/sagemaker-hyperpod-eks-cluster-observability-cluster.html["SageMaker HyperPod Observability",type="documentation"]. + +[#addons-hyperpod-training-operator] +== Amazon SageMaker HyperPod training operator + +The Amazon SageMaker HyperPod training operator helps you accelerate generative AI model development by efficiently managing distributed training across large GPU clusters. It introduces intelligent fault recovery, hang job detection, and process-level management capabilities that minimize training disruptions and reduce costs. Unlike traditional training infrastructure that requires complete job restarts when failures occur, this operator implements surgical process recovery to keep your training jobs running smoothly. + +The operator also works with HyperPod's health monitoring and observability functions, providing real-time visibility into training execution and automatic monitoring of critical metrics like loss spikes and throughput degradation. You can define recovery policies through simple YAML configurations without code changes, allowing you to quickly respond to and recover from unrecoverable training states. These monitoring and recovery capabilities work together to maintain optimal training performance while minimizing operational overhead. + +The Amazon EKS add-on name is `amazon-sagemaker-hyperpod-training-operator`. + +For more information, see link:sagemaker/latest/dg/sagemaker-eks-operator.html[Using the HyperPod training operatorr,type="documentation"] in the _Amazon SageMaker Developer Guide_. + +=== Required IAM permissions + +This add-on requires IAM permissions, and uses EKS Pod Identity. + +{aws} suggests the `AmazonSageMakerHyperPodTrainingOperatorAccess` link:aws-managed-policy/latest/reference/AmazonSageMakerHyperPodTrainingOperatorAccess.html["managed policy",type="documentation"]. + +For more information, see link:sagemaker/latest/dg/sagemaker-eks-operator-install.html#sagemaker-eks-operator-install-operator[Installing the training operator,type="documentation"] in the _Amazon SageMaker Developer Guide_. + +=== Additional information + +To learn more about the add-on, see link:sagemaker/latest/dg/sagemaker-eks-operator.html["SageMaker HyperPod training operator",type="documentation"]. + +[#addons-network-flow] == {aws} Network Flow Monitor Agent The Amazon CloudWatch Network Flow Monitor Agent is a Kubernetes application that collects TCP connection statistics from all nodes in a cluster and publishes network flow reports to Amazon CloudWatch Network Flow Monitor Ingestion APIs. @@ -377,7 +485,7 @@ For more information about the managed policy, see link:AmazonCloudWatch/latest/ To learn more about the add-on, see the https://github.com/aws/network-flow-monitor-agent?tab=readme-ov-file[Amazon CloudWatch Network Flow Monitor Agent GitHub repo]. -[[add-ons-eks-node-monitoring-agent,add-ons-eks-node-monitoring-agent.title]] +[#add-ons-eks-node-monitoring-agent] == Node monitoring agent The node monitoring agent Amazon EKS add-on can detect additional node health issues. These extra health signals can also be leveraged by the optional node auto repair feature to automatically replace nodes as needed. @@ -389,18 +497,18 @@ You do not need to install this add-on on Amazon EKS Auto Mode clusters. For mor The Amazon EKS add-on name is `eks-node-monitoring-agent`. -[[add-ons-eks-node-monitoring-agent-iam-permissions,add-ons-eks-node-monitoring-agent-iam-permissions.title]] +[#add-ons-eks-node-monitoring-agent-iam-permissions] === Required IAM permissions This add-on doesn't require additional permissions. -[[add-ons-eks-node-monitoring-agent-information,add-ons-eks-node-monitoring-agent-information.title]] +[#add-ons-eks-node-monitoring-agent-information] === Additional information For more information, see <>. -[[add-ons-adot,add-ons-adot.title]] +[#add-ons-adot] == {aws} Distro for OpenTelemetry [abstract] @@ -412,19 +520,19 @@ The {aws} Distro for OpenTelemetry Amazon EKS add-on is a secure, production-rea The Amazon EKS add-on name is `adot`. -[[add-ons-adot-iam-permissions,add-ons-adot-iam-permissions.title]] +[#add-ons-adot-iam-permissions] === Required IAM permissions This add-on only requires IAM permissions if you're using one of the preconfigured custom resources that can be opted into through advanced configuration. -[[add-ons-adot-information,add-ons-adot-information.title]] +[#add-ons-adot-information] === Additional information -For more information, see https://aws-otel.github.io/docs/getting-started/adot-eks-add-on[Getting Started with {aws} Distro for OpenTelemetry using EKS Add-Ons] in the {aws} Distro for [.noloc]`OpenTelemetry` documentation. +For more information, see https://aws-otel.github.io/docs/getting-started/adot-eks-add-on[Getting Started with {aws} Distro for OpenTelemetry using EKS Add-Ons] in the {aws} Distro for OpenTelemetry documentation. -ADOT requires that `cert-manager` is deployed on the cluster as a prerequisite, otherwise this add-on won't work if deployed directly using the https://registry.terraform.io/modules/terraform-aws-modules/eks/aws/latest[Amazon EKS Terraform]``cluster_addons`` property. For more requirements, see https://aws-otel.github.io/docs/getting-started/adot-eks-add-on/requirements[Requirements for Getting Started with {aws} Distro for OpenTelemetry using EKS Add-Ons] in the {aws} Distro for [.noloc]`OpenTelemetry` documentation. +ADOT requires that the `cert-manager` add-on is deployed on the cluster as a prerequisite, otherwise this add-on won't work if deployed directly using the https://registry.terraform.io/modules/terraform-aws-modules/eks/aws/latest[Amazon EKS Terraform]``cluster_addons`` property. For more requirements, see https://aws-otel.github.io/docs/getting-started/adot-eks-add-on/requirements[Requirements for Getting Started with {aws} Distro for OpenTelemetry using EKS Add-Ons] in the {aws} Distro for OpenTelemetry documentation. -[[add-ons-guard-duty,add-ons-guard-duty.title]] +[#add-ons-guard-duty] == Amazon GuardDuty agent [abstract] @@ -432,16 +540,16 @@ ADOT requires that `cert-manager` is deployed on the cluster as a prerequisite, Learn about the Amazon GuardDuty agent Amazon EKS add-on. -- -The Amazon GuardDuty agent Amazon EKS add-on is a security monitoring service that analyzes and processes link:guardduty/latest/ug/guardduty_data-sources.html[foundational data sources,type="documentation"] including {aws} CloudTrail management events and Amazon VPC flow logs. Amazon GuardDuty also processes link:guardduty/latest/ug/guardduty-features-activation-model.html[features,type="documentation"], such as [.noloc]`Kubernetes` audit logs and runtime monitoring. +The Amazon GuardDuty agent Amazon EKS add-on collects link:guardduty/latest/ug/runtime-monitoring-collected-events.html[runtime events,type="documentation"] (file access, process execution, network connections) from your EKS cluster nodes for analysis by GuardDuty Runtime Monitoring. GuardDuty itself (not the agent) is the security monitoring service that analyzes and processes link:guardduty/latest/ug/guardduty_data-sources.html[foundational data sources,type="documentation"] including {aws} CloudTrail management events and Amazon VPC flow logs, as well as link:guardduty/latest/ug/guardduty-features-activation-model.html[features,type="documentation"], such as Kubernetes audit logs and runtime monitoring. The Amazon EKS add-on name is `aws-guardduty-agent`. -[[add-ons-guard-duty-iam-permissions,add-ons-guard-duty-iam-permissions.title]] +[#add-ons-guard-duty-iam-permissions] === Required IAM permissions This add-on doesn't require any permissions. -[[add-ons-guard-duty-information,add-ons-guard-duty-information.title]] +[#add-ons-guard-duty-information] === Additional information For more information, see link:guardduty/latest/ug/how-runtime-monitoring-works-eks.html[Runtime Monitoring for Amazon EKS clusters in Amazon GuardDuty,type="documentation"]. @@ -451,7 +559,7 @@ For more information, see link:guardduty/latest/ug/how-runtime-monitoring-works- * To detect potential security threats in your Amazon EKS clusters, enable Amazon GuardDuty runtime monitoring and deploy the GuardDuty security agent to your Amazon EKS clusters. -[[amazon-cloudwatch-observability,amazon-cloudwatch-observability.title]] +[#amazon-cloudwatch-observability] == Amazon CloudWatch Observability agent [abstract] @@ -463,10 +571,10 @@ The Amazon CloudWatch Observability agent Amazon EKS add-on the monitoring and o The Amazon EKS add-on name is `amazon-cloudwatch-observability`. -[[amazon-cloudwatch-observability-iam-permissions,amazon-cloudwatch-observability-iam-permissions.title]] +[#amazon-cloudwatch-observability-iam-permissions] === Required IAM permissions -This add-on uses the IAM roles for service accounts capability of Amazon EKS. For more information, see <>. The permissions in the link:iam/home#/policies/arn:aws:iam::aws:policy/AWSXrayWriteOnlyAccess[AWSXrayWriteOnlyAccess,type="console"] and link:iam/home#/policies/arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy[CloudWatchAgentServerPolicy,type="console"] {aws} managed policies are required. You can create an IAM role, attach the managed policies to it, and annotate the [.noloc]`Kubernetes` service account used by the add-on with the following command. Replace [.replaceable]`my-cluster` with the name of your cluster and [.replaceable]`AmazonEKS_Observability_role` with the name for your role. This command requires that you have https://eksctl.io[eksctl] installed on your device. If you need to use a different tool to create the role, attach the policy to it, and annotate the [.noloc]`Kubernetes` service account, see <>. +This add-on uses the IAM roles for service accounts capability of Amazon EKS. For more information, see <>. The permissions in the link:iam/home#/policies/arn:aws:iam::aws:policy/AWSXrayWriteOnlyAccess[AWSXrayWriteOnlyAccess,type="console"] and link:iam/home#/policies/arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy[CloudWatchAgentServerPolicy,type="console"] {aws} managed policies are required. You can create an IAM role, attach the managed policies to it, and annotate the Kubernetes service account used by the add-on with the following command. Replace [.replaceable]`my-cluster` with the name of your cluster and [.replaceable]`AmazonEKS_Observability_role` with the name for your role. This command requires that you have https://eksctl.io[eksctl] installed on your device. If you need to use a different tool to create the role, attach the policy to it, and annotate the Kubernetes service account, see <>. [source,bash,subs="verbatim,attributes"] ---- @@ -482,12 +590,45 @@ eksctl create iamserviceaccount \ ---- -[[amazon-cloudwatch-observability-information,amazon-cloudwatch-observability-information.title]] +[#amazon-cloudwatch-observability-information] === Additional information For more information, see link:AmazonCloudWatch/latest/monitoring/install-CloudWatch-Observability-EKS-addon.html[Install the CloudWatch agent,type="documentation"]. -[[add-ons-pod-id,add-ons-pod-id.title]] +[#add-ons-aws-privateca-connector] +== {aws} Private CA Connector for Kubernetes + +[abstract] +-- +Learn about the {aws} Private CA Connector for Kubernetes Amazon EKS add-on. +-- + +The {aws} Private CA Connector for Kubernetes is an add-on for cert-manager that enables users to obtain Certificates from {aws} Private Certificate Authority ({aws} Private CA). + +* The Amazon EKS add-on name is `aws-privateca-connector-for-kubernetes`. +* The add-on namespace is `aws-privateca-issuer`. + +This add-on requires `cert-manager`. `cert-manager` is available on Amazon EKS as a community add-on. For more information about this add-on, see <>. For more information about installing add-ons, see <>. + +[#add-ons-aws-privateca-connector-iam-permissions] +=== Required IAM permissions + +This add-on requires IAM permissions. + +Use EKS Pod Identities to attach the `AWSPrivateCAConnectorForKubernetesPolicy` IAM Policy to the `aws-privateca-issuer` Kubernetes Service Account. For more information, see <>. + +For information about the required permissions, see link:aws-managed-policy/latest/reference/AWSPrivateCAConnectorForKubernetesPolicy.html[AWSPrivateCAConnectorForKubernetesPolicy,type="documentation"] in the {aws} Managed Policy Reference. + +[#add-ons-aws-privateca-connector-information] +=== Additional information + +For more information, see the https://github.com/cert-manager/aws-privateca-issuer[{aws} Private CA Issuer for Kubernetes GitHub repository]. + +For more information about configuring the add-on, see https://github.com/cert-manager/aws-privateca-issuer/blob/main/charts/aws-pca-issuer/values.yaml[values.yaml] in the `aws-privateca-issuer` GitHub repo. Confirm the version of values.yaml matches the version of the add-on installed on your cluster. + +This add-on tolerates the `CriticalAddonsOnly` taint used by the `system` NodePool of EKS Auto Mode. For more information, see <>. + +[#add-ons-pod-id] == EKS Pod Identity Agent [abstract] @@ -504,12 +645,44 @@ You do not need to install this add-on on Amazon EKS Auto Mode clusters. Amazon The Amazon EKS add-on name is `eks-pod-identity-agent`. -[[add-ons-pod-id-iam-permissions,add-ons-pod-id-iam-permissions.title]] +[#add-ons-pod-id-iam-permissions] === Required IAM permissions -This add-on users permissions from the <>. +The Pod Identity Agent add-on itself does not require an IAM role. It uses permissions from the <> to function, but does not need a dedicated IAM role for the add-on. -[[add-ons-pod-id-update-information,add-ons-pod-id-update-information.title]] +[#add-ons-pod-id-update-information] === Update information -You can only update one minor version at a time. For example, if your current version is `1.28.x-eksbuild.y` and you want to update to `1.30.x-eksbuild.y`, then you must update your current version to `1.29.x-eksbuild.y` and then update it again to `1.30.x-eksbuild.y`. For more information about updating the add-on, see <>. +You can only update one minor version at a time. For example, if your current version is `1.28.x-eksbuild.y` and you want to update to `1.30.x-eksbuild.y`, then you must update your current version to `1.29.x-eksbuild.y` and then update it again to `1.30.x-eksbuild.y`. For more information about updating the add-on, see <>. + +[#add-ons-sriov-network-metrics-exporter] +== SR-IOV Network Metrics Exporter + +The SR-IOV Network Metrics Exporter Amazon EKS add-on collects and exposes metrics about SR-IOV network devices in Prometheus format. It enables monitoring of SR-IOV network performance on EKS bare metal nodes. The exporter runs as a DaemonSet on nodes with SR-IOV-capable network interfaces and exports metrics that can be scraped by Prometheus. + +NOTE: This add-on requires nodes with SR-IOV-capable network interfaces. + +[%header,cols="2"] +|=== +|Property +|Value + +|Add-on name +|`sriov-network-metrics-exporter` + +|Namespace +|`monitoring` + +|Documentation +|https://github.com/k8snetworkplumbingwg/sriov-network-metrics-exporter[SR-IOV Network Metrics Exporter GitHub repo] + +|Service account name +|None + +|Managed IAM policy +|None + +|Custom IAM permissions +|None + +|=== diff --git a/latest/ug/workloads/workloads-add-ons-available-vendors.adoc b/latest/ug/workloads/workloads-add-ons-available-vendors.adoc index 1b5d266f5..a0134eda9 100644 --- a/latest/ug/workloads/workloads-add-ons-available-vendors.adoc +++ b/latest/ug/workloads/workloads-add-ons-available-vendors.adoc @@ -1,13 +1,10 @@ -//!!NODE_ROOT
+include::../attributes.txt[] [.topic] -[[workloads-add-ons-available-vendors,workloads-add-ons-available-vendors.title]] +[#workloads-add-ons-available-vendors] = {aws} Marketplace add-ons :info_titleabbrev: Marketplace add-ons -include::../attributes.txt[] - - [abstract] -- Learn about the Amazon EKS add-ons from independent software vendors. @@ -20,86 +17,86 @@ In addition to the previous list of Amazon EKS add-ons, you can also add a wide video::IIPj119mspc[youtube,align = center,height = 405,fileref = https://www.youtube.com/embed/IIPj119mspc,width = 720] -[[add-on-accuknox,add-on-accuknox.title]] -== [.noloc]`Accuknox` +[#add-on-accuknox] +== Accuknox [abstract] -- -Learn about the [.noloc]`Accuknox` Amazon EKS add-on. +Learn about the Accuknox Amazon EKS add-on. -- -The add-on name is `accuknox_kubearmor` and the namespace is `kubearmor`. [.noloc]`Accuknox` publishes the add-on. +The add-on name is `accuknox_kubearmor` and the namespace is `kubearmor`. Accuknox publishes the add-on. For information about the add-on, see https://docs.kubearmor.io/kubearmor/quick-links/deployment_guide[Getting Started with KubeArmor] in the KubeArmor documentation. -[[add-on-accuknox-service-account-name,add-on-accuknox-service-account-name.title]] +[#add-on-accuknox-service-account-name] === Service account name A service account isn't used with this add-on. -[[add-on-accuknox-managed-policy,add-on-accuknox-managed-policy.title]] +[#add-on-accuknox-managed-policy] === {aws} managed IAM policy A managed policy isn't used with this add-on. -[[add-on-accuknox-custom-permissions,add-on-accuknox-custom-permissions.title]] +[#add-on-accuknox-custom-permissions] === Custom IAM permissions Custom permissions aren't used with this add-on. -[[add-on-akuity,add-on-akuity.title]] -== [.noloc]`Akuity` +[#add-on-akuity] +== Akuity [abstract] -- -Learn about the [.noloc]`Akuity` Amazon EKS add-on. +Learn about the Akuity Amazon EKS add-on. -- -The add-on name is `akuity_agent` and the namespace is `akuity`. [.noloc]`Akuity` publishes the add-on. +The add-on name is `akuity_agent` and the namespace is `akuity`. Akuity publishes the add-on. For information about how the add-on, see https://docs.akuity.io/tutorials/eks-addon-agent-install/[Installing the Akuity Agent on Amazon EKS with the Akuity EKS add-on] in the Akuity Platform documentation. -[[add-on-akuity-service-account-name,add-on-akuity-service-account-name.title]] +[#add-on-akuity-service-account-name] === Service account name A service account isn't used with this add-on. -[[add-on-akuity-managed-policy,add-on-akuity-managed-policy.title]] +[#add-on-akuity-managed-policy] === {aws} managed IAM policy A managed policy isn't used with this add-on. -[[add-on-akuity-custom-permissions,add-on-akuity-custom-permissions.title]] +[#add-on-akuity-custom-permissions] === Custom IAM permissions Custom permissions aren't used with this add-on. -[[add-on-calyptia,add-on-calyptia.title]] -== [.noloc]`Calyptia` +[#add-on-calyptia] +== Calyptia [abstract] -- -Learn about the [.noloc]`Calyptia` Amazon EKS add-on. +Learn about the Calyptia Amazon EKS add-on. -- -The add-on name is `calyptia_fluent-bit` and the namespace is `calytia-fluentbit`. [.noloc]`Calyptia` publishes the add-on. +The add-on name is `calyptia_fluent-bit` and the namespace is `calytia-fluentbit`. Calyptia publishes the add-on. For information about the add-on, see https://docs.akuity.io/tutorials/eks-addon-agent-install/[Getting Started with Calyptia Core Agent] on the Calyptia documentation website. -[[add-on-calyptia-service-account-name,add-on-calyptia-service-account-name.title]] +[#add-on-calyptia-service-account-name] === Service account name The service account name is `clyptia-fluentbit`. -[[add-on-calyptia-managed-policy,add-on-calyptia-managed-policy.title]] +[#add-on-calyptia-managed-policy] === {aws} managed IAM policy This add-on uses the `AWSMarketplaceMeteringRegisterUsage` managed policy. For more information, see link:aws-managed-policy/latest/reference/AWSMarketplaceMeteringRegisterUsage.html[AWSMarketplaceMeteringRegisterUsage,type="documentation"] in the {aws} Managed Policy Reference Guide. -[[add-on-calyptia-custom-permissions,add-on-calyptia-custom-permissions.title]] +[#add-on-calyptia-custom-permissions] === Command to create required IAM role -The following command requires that you have an IAM [.noloc]`OpenID Connect` ([.noloc]`OIDC`) provider for your cluster. To determine whether you have one, or to create one, see <>. Replace [.replaceable]`my-cluster` with the name of your cluster and [.replaceable]`my-calyptia-role` with the name for your role. This command requires that you have https://eksctl.io[eksctl] installed on your device. If you need to use a different tool to create the role and annotate the [.noloc]`Kubernetes` service account, see <>. +The following command requires that you have an IAM OpenID Connect (OIDC) provider for your cluster. To determine whether you have one, or to create one, see <>. Replace [.replaceable]`my-cluster` with the name of your cluster and [.replaceable]`my-calyptia-role` with the name for your role. This command requires that you have https://eksctl.io[eksctl] installed on your device. If you need to use a different tool to create the role and annotate the Kubernetes service account, see <>. [source,bash,subs="verbatim,attributes"] ---- @@ -108,167 +105,167 @@ eksctl create iamserviceaccount --name service-account-name --namespace calypti ---- -[[add-on-cisco-collector,add-on-cisco-collector.title]] -== [.noloc]`Cisco Observability Collector` +[#add-on-cisco-collector] +== Cisco Observability Collector [abstract] -- -Learn about the [.noloc]`Cisco Observability Collector` Amazon EKS add-on. +Learn about the Cisco Observability Collector Amazon EKS add-on. -- -The add-on name is `cisco_cisco-cloud-observability-collectors` and the namespace is `appdynamics`. [.noloc]`Cisco` pubishes the add-on. +The add-on name is `cisco_cisco-cloud-observability-collectors` and the namespace is `appdynamics`. Cisco pubishes the add-on. For information about the add-on, see https://docs.appdynamics.com/observability/cisco-cloud-observability/en/kubernetes-and-app-service-monitoring/install-kubernetes-and-app-service-monitoring-with-amazon-elastic-kubernetes-service/use-the-cisco-cloud-observability-aws-marketplace-add-ons[Use the Cisco Cloud Observability {aws} Marketplace Add-Ons] in the Cisco AppDynamics documentation. -[[add-on-cisco-collector-service-account-name,add-on-cisco-collector-service-account-name.title]] +[#add-on-cisco-collector-service-account-name] === Service account name A service account isn't used with this add-on. -[[add-on-cisco-collector-managed-policy,add-on-cisco-collector-managed-policy.title]] +[#add-on-cisco-collector-managed-policy] === {aws} managed IAM policy A managed policy isn't used with this add-on. -[[add-on-cisco-collector-custom-permissions,add-on-cisco-collector-custom-permissions.title]] +[#add-on-cisco-collector-custom-permissions] === Custom IAM permissions Custom permissions aren't used with this add-on. -[[add-on-cisco-operator,add-on-cisco-operator.title]] -== [.noloc]`Cisco Observability Operator` +[#add-on-cisco-operator] +== Cisco Observability Operator [abstract] -- -Learn about the [.noloc]`Cisco Observability Operator` Amazon EKS add-on. +Learn about the Cisco Observability Operator Amazon EKS add-on. -- -The add-on name is `cisco_cisco-cloud-observability-operators` and the namespace is `appdynamics`. [.noloc]`Cisco` publishes the add-on. +The add-on name is `cisco_cisco-cloud-observability-operators` and the namespace is `appdynamics`. Cisco publishes the add-on. For information about the add-on, see https://docs.appdynamics.com/observability/cisco-cloud-observability/en/kubernetes-and-app-service-monitoring/install-kubernetes-and-app-service-monitoring-with-amazon-elastic-kubernetes-service/use-the-cisco-cloud-observability-aws-marketplace-add-ons[Use the Cisco Cloud Observability {aws} Marketplace Add-Ons] in the Cisco AppDynamics documentation. -[[add-on-cisco-operator-service-account-name,add-on-cisco-operator-service-account-name.title]] +[#add-on-cisco-operator-service-account-name] === Service account name A service account isn't used with this add-on. -[[add-on-cisco-operator-managed-policy,add-on-cisco-operator-managed-policy.title]] +[#add-on-cisco-operator-managed-policy] === {aws} managed IAM policy A managed policy isn't used with this add-on. -[[add-on-cisco-operator-custom-permissions,add-on-cisco-operator-custom-permissions.title]] +[#add-on-cisco-operator-custom-permissions] === Custom IAM permissions Custom permissions aren't used with this add-on. -[[add-on-cloudsoft,add-on-cloudsoft.title]] -== [.noloc]`CLOUDSOFT` +[#add-on-cloudsoft] +== CLOUDSOFT [abstract] -- -Learn about the [.noloc]`CLOUDSOFT` Amazon EKS add-on. +Learn about the CLOUDSOFT Amazon EKS add-on. -- -The add-on name is `cloudsoft_cloudsoft-amp` and the namespace is `cloudsoft-amp`. [.noloc]`CLOUDSOFT` publishes the add-on. +The add-on name is `cloudsoft_cloudsoft-amp` and the namespace is `cloudsoft-amp`. CLOUDSOFT publishes the add-on. For information about the add-on, see https://docs.cloudsoft.io/operations/configuration/aws-eks-addon.html[Amazon EKS ADDON] in the CLOUDSOFT documentation. -[[add-on-cloudsoft-service-account-name,add-on-cloudsoft-service-account-name.title]] +[#add-on-cloudsoft-service-account-name] === Service account name A service account isn't used with this add-on. -[[add-on-cloudsoft-managed-policy,add-on-cloudsoft-managed-policy.title]] +[#add-on-cloudsoft-managed-policy] === {aws} managed IAM policy A managed policy isn't used with this add-on. -[[add-on-cloudsoft-custom-permissions,add-on-cloudsoft-custom-permissions.title]] +[#add-on-cloudsoft-custom-permissions] === Custom IAM permissions Custom permissions aren't used with this add-on. -[[add-on-cribl,add-on-cribl.title]] -== [.noloc]`Cribl` +[#add-on-cribl] +== Cribl [abstract] -- -Learn about the [.noloc]`Cribl` Amazon EKS add-on. +Learn about the Cribl Amazon EKS add-on. -- -The add-on name is `cribl_cribledge` and the namespace is `cribledge`. [.noloc]`Cribl` publishes the add-on. +The add-on name is `cribl_cribledge` and the namespace is `cribledge`. Cribl publishes the add-on. For information about the add-on, see https://docs.cribl.io/edge/usecase-edge-aws-eks/[Installing the Cribl Amazon EKS Add-on for Edge] in the Cribl documentation -[[add-on-cribl-service-account-name,add-on-cribl-service-account-name.title]] +[#add-on-cribl-service-account-name] === Service account name A service account isn't used with this add-on. -[[add-on-cribl-managed-policy,add-on-cribl-managed-policy.title]] +[#add-on-cribl-managed-policy] === {aws} managed IAM policy A managed policy isn't used with this add-on. -[[add-on-cribl-custom-permissions,add-on-cribl-custom-permissions.title]] +[#add-on-cribl-custom-permissions] === Custom IAM permissions Custom permissions aren't used with this add-on. -[[add-on-dynatrace,add-on-dynatrace.title]] -== [.noloc]`Dynatrace` +[#add-on-dynatrace] +== Dynatrace [abstract] -- -Learn about the [.noloc]`Dynatrace` Amazon EKS add-on. +Learn about the Dynatrace Amazon EKS add-on. -- -The add-on name is `dynatrace_dynatrace-operator` and the namespace is `dynatrace`. [.noloc]`Dynatrace` publishes the add-on. +The add-on name is `dynatrace_dynatrace-operator` and the namespace is `dynatrace`. Dynatrace publishes the add-on. -For information about the add-on, see https://www.dynatrace.com/technologies/kubernetes-monitoring/[Kubernetes monitoring] in the [.noloc]`dynatrace` documentation. +For information about the add-on, see https://www.dynatrace.com/technologies/kubernetes-monitoring/[Kubernetes monitoring] in the dynatrace documentation. -[[add-on-dynatrace-service-account-name,add-on-dynatrace-service-account-name.title]] +[#add-on-dynatrace-service-account-name] === Service account name A service account isn't used with this add-on. -[[add-on-dynatrace-managed-policy,add-on-dynatrace-managed-policy.title]] +[#add-on-dynatrace-managed-policy] === {aws} managed IAM policy A managed policy isn't used with this add-on. -[[add-on-dynatrace-custom-permissions,add-on-dynatrace-custom-permissions.title]] +[#add-on-dynatrace-custom-permissions] === Custom IAM permissions Custom permissions aren't used with this add-on. -[[add-on-datree-pro,add-on-datree-pro.title]] -== [.noloc]`Datree` +[#add-on-datree-pro] +== Datree [abstract] -- -Learn about the [.noloc]`Datree` Amazon EKS add-on. +Learn about the Datree Amazon EKS add-on. -- -The add-on name is `datree_engine-pro` and the namespace is `datree`. [.noloc]`Datree` publishes the add-on. +The add-on name is `datree_engine-pro` and the namespace is `datree`. Datree publishes the add-on. For information about the add-on, see https://hub.datree.io/integrations/eks-integration[Amazon EKS-intergration] in the Datree documentation. -[[add-on-datree-pro-service-account-name,add-on-datree-pro-service-account-name.title]] +[#add-on-datree-pro-service-account-name] === Service account name The service account name is datree-webhook-server-awsmp. -[[add-on-datree-pro-managed-policy,add-on-datree-pro-managed-policy.title]] +[#add-on-datree-pro-managed-policy] === {aws} managed IAM policy The managed policy is AWSLicenseManagerConsumptionPolicy. For more information, see link:aws-managed-policy/latest/reference/AWSLicenseManagerConsumptionPolicy.html[AWSLicenseManagerConsumptionPolicy,type="documentation"] in the {aws} Managed Policy Reference Guide.. -[[add-on-datree-pro-iam-command,add-on-datree-pro-iam-command.title]] +[#add-on-datree-pro-iam-command] === Command to create required IAM role -The following command requires that you have an IAM [.noloc]`OpenID Connect` ([.noloc]`OIDC`) provider for your cluster. To determine whether you have one, or to create one, see <>. Replace [.replaceable]`my-cluster` with the name of your cluster and [.replaceable]`my-datree-role` with the name for your role. This command requires that you have https://eksctl.io[eksctl] installed on your device. If you need to use a different tool to create the role and annotate the [.noloc]`Kubernetes` service account, see <>. +The following command requires that you have an IAM OpenID Connect (OIDC) provider for your cluster. To determine whether you have one, or to create one, see <>. Replace [.replaceable]`my-cluster` with the name of your cluster and [.replaceable]`my-datree-role` with the name for your role. This command requires that you have https://eksctl.io[eksctl] installed on your device. If you need to use a different tool to create the role and annotate the Kubernetes service account, see <>. [source,bash,subs="verbatim,attributes"] ---- @@ -277,96 +274,120 @@ eksctl create iamserviceaccount --name datree-webhook-server-awsmp --namespace d ---- -[[add-on-datree-pro-custom-permissions,add-on-datree-pro-custom-permissions.title]] +[#add-on-datree-pro-custom-permissions] === Custom permissions Custom permissions aren't used with this add-on. -[[add-on-datadog,add-on-datadog.title]] -== [.noloc]`Datadog` +[#add-on-datadog] +== Datadog [abstract] -- -Learn about the [.noloc]`Datadog` Amazon EKS add-on. +Learn about the Datadog Amazon EKS add-on. -- -The add-on name is `datadog_operator` and the namespace is `datadog-agent`. [.noloc]`Datadog` publishes the add-on. +The add-on name is `datadog_operator` and the namespace is `datadog-agent`. Datadog publishes the add-on. For information about the add-on, see https://docs.datadoghq.com/containers/guide/operator-eks-addon/?tab=console[Installing the Datadog Agent on Amazon EKS with the Datadog Operator Add-on] in the Datadog documentation. -[[add-on-datadog-service-account-name,add-on-datadog-service-account-name.title]] +[#add-on-datadog-service-account-name] === Service account name A service account isn't used with this add-on. -[[add-on-datadog-managed-policy,add-on-datadog-managed-policy.title]] +[#add-on-datadog-managed-policy] === {aws} managed IAM policy A managed policy isn't used with this add-on. -[[add-on-datadog-custom-permissions,add-on-datadog-custom-permissions.title]] +[#add-on-datadog-custom-permissions] === Custom IAM permissions Custom permissions aren't used with this add-on. -[[add-on-groundcover,add-on-groundcover.title]] -== [.noloc]`Groundcover` +[#add-on-groundcover] +== Groundcover [abstract] -- -Learn about the [.noloc]`Groundcover` Amazon EKS add-on. +Learn about the Groundcover Amazon EKS add-on. -- -The add-on name is `groundcover_agent` and the namespace is `groundcover`. [.noloc]`groundcover` publishes the add-on. +The add-on name is `groundcover_agent` and the namespace is `groundcover`. groundcover publishes the add-on. For information about the add-on, see https://docs.groundcover.com/docs/~/changes/VhDDAl1gy1VIO3RIcgxD/configuration/customization-guide/customize-deployment/eks-add-on[Installing the groundcover Amazon EKS Add-on] in the groundcover documentation. -[[add-on-groundcover-service-account-name,add-on-groundcover-service-account-name.title]] +[#add-on-groundcover-service-account-name] +=== Service account name + +A service account isn't used with this add-on. + +[#add-on-groundcover-managed-policy] +=== {aws} managed IAM policy + +A managed policy isn't used with this add-on. + +[#add-on-groundcover-custom-permissions] +=== Custom IAM permissions + +Custom permissions aren't used with this add-on. + +[#add-on-instana] +== IBM Instana + +The add-on name is `instana-agent` and the namespace is `instana-agent`. IBM publishes the add-on. + +For information about the add-on, see link:blogs/ibm-redhat/implement-observability-for-amazon-eks-workloads-using-the-instana-amazon-eks-add-on/[Implement observability for Amazon EKS workloads using the Instana Amazon EKS add-on,type="marketing"] and link:blogs/ibm-redhat/monitor-and-optimize-amazon-eks-costs-with-ibm-instana-and-kubecost/[Monitor and optimize Amazon EKS costs with IBM Instana and Kubecost,type="marketing"] in the {aws} Blog. + +Instana Observability (Instana) offers an Amazon EKS Add-on that deploys Instana agents to Amazon EKS clusters. Customers can use this add-on to collect and analyze real-time performance data to gain insights into their containerized applications. The Instana Amazon EKS add-on provides visibility across your Kubernetes environments. Once deployed, the Instana agent automatically discovers components within your Amazon EKS clusters including nodes, namespaces, deployments, services, and pods. + +[#add-on-instana-service-account-name] === Service account name A service account isn't used with this add-on. -[[add-on-groundcover-managed-policy,add-on-groundcover-managed-policy.title]] +[#add-on-instana-managed-policy] === {aws} managed IAM policy A managed policy isn't used with this add-on. -[[add-on-groundcover-custom-permissions,add-on-groundcover-custom-permissions.title]] +[#add-on-instana-custom-permissions] === Custom IAM permissions Custom permissions aren't used with this add-on. -[[add-on-grafana,add-on-grafana.title]] -== [.noloc]`Grafana Labs` +[#add-on-grafana] +== Grafana Labs [abstract] -- -Learn about the [.noloc]`Grafana Labs` Amazon EKS add-on. +Learn about the Grafana Labs Amazon EKS add-on. -- -The add-on name is `grafana-labs_kubernetes-monitoring` and the namespace is `monitoring`. [.noloc]`Grafana Labs` publishes the add-on. +The add-on name is `grafana-labs_kubernetes-monitoring` and the namespace is `monitoring`. Grafana Labs publishes the add-on. For information about the add-on, see https://grafana.com/docs/grafana-cloud/monitor-infrastructure/kubernetes-monitoring/configuration/config-aws-eks/[Configure Kubernetes Monitoring as an Add-on with Amazon EKS] in the Grafana Labs documentation. -[[add-on-grafana-service-account-name,add-on-grafana-service-account-name.title]] +[#add-on-grafana-service-account-name] === Service account name A service account isn't used with this add-on. -[[add-on-grafana-managed-policy,add-on-grafana-managed-policy.title]] +[#add-on-grafana-managed-policy] === {aws} managed IAM policy A managed policy isn't used with this add-on. -[[add-on-grafana-custom-permissions,add-on-grafana-custom-permissions.title]] +[#add-on-grafana-custom-permissions] === Custom IAM permissions Custom permissions aren't used with this add-on. -[[add-on-guance,add-on-guance.title]] -== [.noloc]`Guance` +[#add-on-guance] +== Guance -* *Publisher* – [.noloc]`GUANCE` +* *Publisher* – GUANCE * *Name* – `guance_datakit` * *Namespace* – `datakit` * *Service account name* – A service account isn't used with this add-on. @@ -375,32 +396,32 @@ Custom permissions aren't used with this add-on. * *Setup and usage instructions* – See https://docs.guance.com/en/datakit/datakit-eks-deploy/#add-on-install[Using Amazon EKS add-on] in the Guance documentation. -[[add-on-ha-proxy,add-on-ha-proxy.title]] -== [.noloc]`HA Proxy` +[#add-on-ha-proxy] +== HA Proxy [abstract] -- -Learn about the [.noloc]`HA Proxy` Amazon EKS add-on. +Learn about the HA Proxy Amazon EKS add-on. -- -The name is `haproxy-technologies_kubernetes-ingress-ee` and the namespace is `haproxy-controller`. [.noloc]`HA Proxy` publishes the add-on. +The name is `haproxy-technologies_kubernetes-ingress-ee` and the namespace is `haproxy-controller`. HA Proxy publishes the add-on. For information about the add-on, see https://hub.datree.io/integrations/eks-integration[Amazon EKS-intergration] in the Datree documentation. -[[add-on-ha-proxy-service-account-name,add-on-ha-proxy-service-account-name.title]] +[#add-on-ha-proxy-service-account-name] === Service account name The service account name is `customer defined`. -[[add-on-ha-proxy-managed-policy,add-on-ha-proxy-managed-policy.title]] +[#add-on-ha-proxy-managed-policy] === {aws} managed IAM policy The managed policy is AWSLicenseManagerConsumptionPolicy. For more information, see link:aws-managed-policy/latest/reference/AWSLicenseManagerConsumptionPolicy.html[AWSLicenseManagerConsumptionPolicy,type="documentation"] in the {aws} Managed Policy Reference Guide.. -[[add-on-ha-proxy-iam-command,add-on-ha-proxy-iam-command.title]] +[#add-on-ha-proxy-iam-command] === Command to create required IAM role -The following command requires that you have an IAM [.noloc]`OpenID Connect` ([.noloc]`OIDC`) provider for your cluster. To determine whether you have one, or to create one, see <>. Replace [.replaceable]`my-cluster` with the name of your cluster and [.replaceable]`my-haproxy-role` with the name for your role. This command requires that you have https://eksctl.io[eksctl] installed on your device. If you need to use a different tool to create the role and annotate the [.noloc]`Kubernetes` service account, see <>. +The following command requires that you have an IAM OpenID Connect (OIDC) provider for your cluster. To determine whether you have one, or to create one, see <>. Replace [.replaceable]`my-cluster` with the name of your cluster and [.replaceable]`my-haproxy-role` with the name for your role. This command requires that you have https://eksctl.io[eksctl] installed on your device. If you need to use a different tool to create the role and annotate the Kubernetes service account, see <>. [source,bash,subs="verbatim,attributes"] ---- @@ -409,37 +430,37 @@ eksctl create iamserviceaccount --name service-account-name --namespace haproxy ---- -[[add-on-ha-proxy-custom-permissions,add-on-ha-proxy-custom-permissions.title]] +[#add-on-ha-proxy-custom-permissions] === Custom permissions Custom permissions aren't used with this add-on. -[[add-on-kpow,add-on-kpow.title]] -== [.noloc]`Kpow` +[#add-on-kpow] +== Kpow [abstract] -- -Learn about the [.noloc]`Kpow` Amazon EKS add-on. +Learn about the Kpow Amazon EKS add-on. -- -The add-on name is `factorhouse_kpow` and the namespace is `factorhouse`. [.noloc]`Factorhouse` publishes the add-on. +The add-on name is `factorhouse_kpow` and the namespace is `factorhouse`. Factorhouse publishes the add-on. -For information about the add-on, see https://docs.kpow.io/installation/aws-marketplace-lm/[{aws} Marketplace LM] in the [.noloc]`Kpow` documentation. +For information about the add-on, see https://docs.kpow.io/installation/aws-marketplace-lm/[{aws} Marketplace LM] in the Kpow documentation. -[[add-on-kpow-service-account-name,add-on-kpow-service-account-name.title]] +[#add-on-kpow-service-account-name] === Service account name The service account name is `kpow`. -[[add-on-kpow-managed-policy,add-on-kpow-managed-policy.title]] +[#add-on-kpow-managed-policy] === {aws} managed IAM policy The managed policy is AWSLicenseManagerConsumptionPolicy. For more information, see link:aws-managed-policy/latest/reference/AWSLicenseManagerConsumptionPolicy.html[AWSLicenseManagerConsumptionPolicy,type="documentation"] in the {aws} Managed Policy Reference Guide.. -[[add-on-kpow-iam-command,add-on-kpow-iam-command.title]] +[#add-on-kpow-iam-command] === Command to create required IAM role -The following command requires that you have an IAM [.noloc]`OpenID Connect` ([.noloc]`OIDC`) provider for your cluster. To determine whether you have one, or to create one, see <>. Replace [.replaceable]`my-cluster` with the name of your cluster and [.replaceable]`my-kpow-role` with the name for your role. This command requires that you have https://eksctl.io[eksctl] installed on your device. If you need to use a different tool to create the role and annotate the [.noloc]`Kubernetes` service account, see <>. +The following command requires that you have an IAM OpenID Connect (OIDC) provider for your cluster. To determine whether you have one, or to create one, see <>. Replace [.replaceable]`my-cluster` with the name of your cluster and [.replaceable]`my-kpow-role` with the name for your role. This command requires that you have https://eksctl.io[eksctl] installed on your device. If you need to use a different tool to create the role and annotate the Kubernetes service account, see <>. [source,bash,subs="verbatim,attributes"] ---- @@ -448,68 +469,68 @@ eksctl create iamserviceaccount --name kpow --namespace factorhouse --cluster my ---- -[[add-on-kpow-custom-permissions,add-on-kpow-custom-permissions.title]] +[#add-on-kpow-custom-permissions] === Custom permissions Custom permissions aren't used with this add-on. -[[add-on-kubecost,add-on-kubecost.title]] -== [.noloc]`Kubecost` +[#add-on-kubecost] +== Kubecost [abstract] -- -Learn about the [.noloc]`Kubecost` Amazon EKS add-on. +Learn about the Kubecost Amazon EKS add-on. -- -The add-on name is `kubecost_kubecost` and the namespace is `kubecost`. [.noloc]`Kubecost` publishes the add-on. +The add-on name is `kubecost_kubecost` and the namespace is `kubecost`. Kubecost publishes the add-on. -For information about the add-on, see https://docs.kubecost.com/install-and-configure/install/cloud-integration/aws-cloud-integrations[{aws} Cloud Billing Integration] in the [.noloc]`Kubecost` documentation. +For information about the add-on, see https://docs.kubecost.com/install-and-configure/install/cloud-integration/aws-cloud-integrations[{aws} Cloud Billing Integration] in the Kubecost documentation. -If your cluster is version `1.23` or later, you must have the <> installed on your cluster. otherwise you will receive an error. +You must have the <> installed on your cluster. otherwise you will receive an error. -[[add-on-kubecost-service-account-name,add-on-kubecost-service-account-name.title]] +[#add-on-kubecost-service-account-name] === Service account name A service account isn't used with this add-on. -[[add-on-kubecost-managed-policy,add-on-kubecost-managed-policy.title]] +[#add-on-kubecost-managed-policy] === {aws} managed IAM policy A managed policy isn't used with this add-on. -[[add-on-kubecost-custom-permissions,add-on-kubecost-custom-permissions.title]] +[#add-on-kubecost-custom-permissions] === Custom IAM permissions Custom permissions aren't used with this add-on. -[[add-on-kasten,add-on-kasten.title]] -== [.noloc]`Kasten` +[#add-on-kasten] +== Kasten [abstract] -- -Learn about the [.noloc]`Kasten` Amazon EKS add-on. +Learn about the Kasten Amazon EKS add-on. -- -The add-on name is `kasten_k10` and the namespace is `kasten-io`. [.noloc]`Kasten by Veeam` publishes the add-on. +The add-on name is `kasten_k10` and the namespace is `kasten-io`. Kasten by Veeam publishes the add-on. For information about the add-on, see https://docs.kasten.io/latest/install/aws-eks-addon/aws-eks-addon.html[Installing K10 on {aws} using Amazon EKS Add-on] in the Kasten documentation. -If your Amazon EKS cluster is version [.noloc]`Kubernetes` `1.23` or later, you must have the Amazon EBS CSI driver installed on your cluster with a default `StorageClass`. +You must have the Amazon EBS CSI driver installed on your cluster with a default `StorageClass`. -[[add-on-kasten-service-account-name,add-on-kasten-service-account-name.title]] +[#add-on-kasten-service-account-name] === Service account name The service account name is `k10-k10`. -[[add-on-kasten-managed-policy,add-on-kasten-managed-policy.title]] +[#add-on-kasten-managed-policy] === {aws} managed IAM policy The managed policy is AWSLicenseManagerConsumptionPolicy. For more information, see link:aws-managed-policy/latest/reference/AWSLicenseManagerConsumptionPolicy.html[AWSLicenseManagerConsumptionPolicy,type="documentation"] in the {aws} Managed Policy Reference Guide.. -[[add-on-kasten-iam-command,add-on-kasten-iam-command.title]] +[#add-on-kasten-iam-command] === Command to create required IAM role -The following command requires that you have an IAM [.noloc]`OpenID Connect` ([.noloc]`OIDC`) provider for your cluster. To determine whether you have one, or to create one, see <>. Replace [.replaceable]`my-cluster` with the name of your cluster and [.replaceable]`my-kasten-role` with the name for your role. This command requires that you have https://eksctl.io[eksctl] installed on your device. If you need to use a different tool to create the role and annotate the [.noloc]`Kubernetes` service account, see <>. +The following command requires that you have an IAM OpenID Connect (OIDC) provider for your cluster. To determine whether you have one, or to create one, see <>. Replace [.replaceable]`my-cluster` with the name of your cluster and [.replaceable]`my-kasten-role` with the name for your role. This command requires that you have https://eksctl.io[eksctl] installed on your device. If you need to use a different tool to create the role and annotate the Kubernetes service account, see <>. [source,bash,subs="verbatim,attributes"] ---- @@ -518,154 +539,154 @@ eksctl create iamserviceaccount --name k10-k10 --namespace kasten-io --cluster m ---- -[[add-on-kasten-custom-permissions,add-on-kasten-custom-permissions.title]] +[#add-on-kasten-custom-permissions] === Custom permissions Custom permissions aren't used with this add-on. -[[add-on-kong,add-on-kong.title]] -== [.noloc]`Kong` +[#add-on-kong] +== Kong [abstract] -- -Learn about the [.noloc]`Kong` Amazon EKS add-on. +Learn about the Kong Amazon EKS add-on. -- -The add-on name is `kong_konnect-ri` and the namespace is `kong`. [.noloc]`Kong` publishes the add-on. +The add-on name is `kong_konnect-ri` and the namespace is `kong`. Kong publishes the add-on. For information about the add-on, see https://kong.github.io/aws-marketplace-addon-kong-gateway/[Installing the Kong Gateway EKS Add-on] in the Kong documentation. -If your cluster is version `1.23` or later, you must have the <> installed on your cluster. otherwise you will receive an error. +You must have the <> installed on your cluster. otherwise you will receive an error. -[[add-on-kong-service-account-name,add-on-kong-service-account-name.title]] +[#add-on-kong-service-account-name] === Service account name A service account isn't used with this add-on. -[[add-on-kong-managed-policy,add-on-kong-managed-policy.title]] +[#add-on-kong-managed-policy] === {aws} managed IAM policy A managed policy isn't used with this add-on. -[[add-on-kong-custom-permissions,add-on-kong-custom-permissions.title]] +[#add-on-kong-custom-permissions] === Custom IAM permissions Custom permissions aren't used with this add-on. -[[add-on-leaksignal,add-on-leaksignal.title]] -== [.noloc]`LeakSignal` +[#add-on-leaksignal] +== LeakSignal [abstract] -- -Learn about the [.noloc]`LeakSignal` Amazon EKS add-on. +Learn about the LeakSignal Amazon EKS add-on. -- -The add-on name is `leaksignal_leakagent` and the namespace is `leakagent`. [.noloc]`LeakSignal` publishes the add-on. +The add-on name is `leaksignal_leakagent` and the namespace is `leakagent`. LeakSignal publishes the add-on. For information about the add-on, see https://www.leaksignal.com/docs/LeakAgent/Deployment/{aws}%20EKS%20Addon/[Install the LeakAgent add-on] in the LeakSignal documentation -If your cluster is version `1.23` or later, you must have the <> installed on your cluster. otherwise you will receive an error. +You must have the <> installed on your cluster. otherwise you will receive an error. -[[add-on-leaksignal-service-account-name,add-on-leaksignal-service-account-name.title]] +[#add-on-leaksignal-service-account-name] === Service account name A service account isn't used with this add-on. -[[add-on-leaksignal-managed-policy,add-on-leaksignal-managed-policy.title]] +[#add-on-leaksignal-managed-policy] === {aws} managed IAM policy A managed policy isn't used with this add-on. -[[add-on-leaksignal-custom-permissions,add-on-leaksignal-custom-permissions.title]] +[#add-on-leaksignal-custom-permissions] === Custom IAM permissions Custom permissions aren't used with this add-on. -[[add-on-netapp,add-on-netapp.title]] -== [.noloc]`NetApp` +[#add-on-netapp] +== NetApp [abstract] -- -Learn about the [.noloc]`NetApp` Amazon EKS add-on. +Learn about the NetApp Amazon EKS add-on. -- -The add-on name is `netapp_trident-operator` and the namespace is `trident`. [.noloc]`NetApp` publishes the add-on. +The add-on name is `netapp_trident-operator` and the namespace is `trident`. NetApp publishes the add-on. For information about the add-on, see https://docs.netapp.com/us-en/trident/trident-use/trident-aws-addon.html[Configure the Trident EKS add-on] in the NetApp documentation. -[[add-on-netapp-service-account-name,add-on-netapp-service-account-name.title]] +[#add-on-netapp-service-account-name] === Service account name A service account isn't used with this add-on. -[[add-on-netapp-managed-policy,add-on-netapp-managed-policy.title]] +[#add-on-netapp-managed-policy] === {aws} managed IAM policy A managed policy isn't used with this add-on. -[[add-on-netapp-custom-permissions,add-on-netapp-custom-permissions.title]] +[#add-on-netapp-custom-permissions] === Custom IAM permissions Custom permissions aren't used with this add-on. -[[add-on-new-relic,add-on-new-relic.title]] -== [.noloc]`New Relic` +[#add-on-new-relic] +== New Relic [abstract] -- -Learn about the [.noloc]`New Relic` Amazon EKS add-on. +Learn about the New Relic Amazon EKS add-on. -- -The add-on name is `new-relic_kubernetes-operator` and the namespace is `newrelic`. [.noloc]`New Relic` publishes the add-on. +The add-on name is `new-relic_kubernetes-operator` and the namespace is `newrelic`. New Relic publishes the add-on. For information about the add-on, see https://docs.newrelic.com/docs/infrastructure/amazon-integrations/connect/eks-add-on[Installing the New Relic Add-on for EKS] in the New Relic documentation. -[[add-on-new-relic-service-account-name,add-on-new-relic-service-account-name.title]] +[#add-on-new-relic-service-account-name] === Service account name A service account isn't used with this add-on. -[[add-on-new-relic-managed-policy,add-on-new-relic-managed-policy.title]] +[#add-on-new-relic-managed-policy] === {aws} managed IAM policy A managed policy isn't used with this add-on. -[[add-on-new-relic-custom-permissions,add-on-new-relic-custom-permissions.title]] +[#add-on-new-relic-custom-permissions] === Custom IAM permissions Custom permissions aren't used with this add-on. -[[add-on-rafay,add-on-rafay.title]] -== [.noloc]`Rafay` +[#add-on-rafay] +== Rafay [abstract] -- -Learn about the [.noloc]`Rafay` Amazon EKS add-on. +Learn about the Rafay Amazon EKS add-on. -- -The add-on name is `rafay-systems_rafay-operator` and the namespace is `rafay-system`. [.noloc]`Rafay` publishes the add-on. +The add-on name is `rafay-systems_rafay-operator` and the namespace is `rafay-system`. Rafay publishes the add-on. For information about the add-on, see https://docs.rafay.co/clusters/import/eksaddon/[Installing the Rafay Amazon EKS Add-on] in the Rafay documentation. -[[add-on-rafay-service-account-name,add-on-rafay-service-account-name.title]] +[#add-on-rafay-service-account-name] === Service account name A service account isn't used with this add-on. -[[add-on-rafay-managed-policy,add-on-rafay-managed-policy.title]] +[#add-on-rafay-managed-policy] === {aws} managed IAM policy A managed policy isn't used with this add-on. -[[add-on-rafay-custom-permissions,add-on-rafay-custom-permissions.title]] +[#add-on-rafay-custom-permissions] === Custom IAM permissions Custom permissions aren't used with this add-on. -[[add-on-rad,add-on-rad.title]] -== [.noloc]`Rad Security` +[#add-on-rad] +== Rad Security -* *Publisher* – [.noloc]`RAD SECURITY` +* *Publisher* – RAD SECURITY * *Name* – `rad-security_rad-security` * *Namespace* – `ksoc` * *Service account name* – A service account isn't used with this add-on. @@ -674,10 +695,10 @@ Custom permissions aren't used with this add-on. * *Setup and usage instructions* – See https://docs.rad.security/docs/installing-ksoc-in-the-aws-marketplace[Installing Rad Through The {aws} Marketplace] in the Rad Security documentation. -[[add-on-solarwinds,add-on-solarwinds.title]] -== [.noloc]`SolarWinds` +[#add-on-solarwinds] +== SolarWinds -* *Publisher* – [.noloc]`SOLARWINDS` +* *Publisher* – SOLARWINDS * *Name* – `solarwinds_swo-k8s-collector-addon` * *Namespace* – `solarwinds` * *Service account name* – A service account isn't used with this add-on. @@ -686,37 +707,37 @@ Custom permissions aren't used with this add-on. * *Setup and usage instructions* – See https://documentation.solarwinds.com/en/success_center/observability/content/configure/configure-kubernetes.htm#MonitorAmazonEKS[Monitor an Amazon EKS cluster] in the SolarWinds documentation. -[[add-on-solo,add-on-solo.title]] -== [.noloc]`Solo` +[#add-on-solo] +== Solo [abstract] -- -Learn about the [.noloc]`Solo` Amazon EKS add-on. +Learn about the Solo Amazon EKS add-on. -- -The add-on name is `solo-io_istio-distro` and the namespace is `istio-system`. [.noloc]`Solo` publishes the add-on. +The add-on name is `solo-io_istio-distro` and the namespace is `istio-system`. Solo publishes the add-on. For information about the add-on, see https://docs.solo.io/gloo-mesh-enterprise/main/setup/install/eks_addon/[Installing Istio] in the Solo.io documentation.. -[[add-on-solo-service-account-name,add-on-solo-service-account-name.title]] +[#add-on-solo-service-account-name] === Service account name A service account isn't used with this add-on. -[[add-on-solo-managed-policy,add-on-solo-managed-policy.title]] +[#add-on-solo-managed-policy] === {aws} managed IAM policy A managed policy isn't used with this add-on. -[[add-on-solo-custom-permissions,add-on-solo-custom-permissions.title]] +[#add-on-solo-custom-permissions] === Custom IAM permissions Custom permissions aren't used with this add-on. -[[add-on-snyk,add-on-snyk.title]] -== [.noloc]`Snyk` +[#add-on-snyk] +== Snyk -* *Publisher* – [.noloc]`SNYK` +* *Publisher* – SNYK * *Name* – `snyk_runtime-sensor` * *Namespace* – `snyk_runtime-sensor` * *Service account name* – A service account isn't used with this add-on. @@ -725,166 +746,174 @@ Custom permissions aren't used with this add-on. * *Setup and usage instructions* – See https://docs.snyk.io/integrate-with-snyk/snyk-runtime-sensor[Snyk runtime sensor] in the Snyk user docs. -[[add-on-stormforge,add-on-stormforge.title]] -== [.noloc]`Stormforge` +[#add-on-stormforge] +== Stormforge [abstract] -- -Learn about the [.noloc]`Stormforge` Amazon EKS add-on. +Learn about the Stormforge Amazon EKS add-on. -- -The add-on name is `stormforge_optimize-Live` and the namespace is `stormforge-system`. [.noloc]`Stormforge` publishes the add-on. +The add-on name is `stormforge_optimize-Live` and the namespace is `stormforge-system`. Stormforge publishes the add-on. For information about the add-on, see https://docs.stormforge.io/optimize-live/getting-started/install-v2/[Installing the StormForge Agent] in the StormForge documentation. -[[add-on-stormforge-service-account-name,add-on-stormforge-service-account-name.title]] +[#add-on-stormforge-service-account-name] === Service account name A service account isn't used with this add-on. -[[add-on-stormforge-managed-policy,add-on-stormforge-managed-policy.title]] +[#add-on-stormforge-managed-policy] === {aws} managed IAM policy A managed policy isn't used with this add-on. -[[add-on-stormforge-custom-permissions,add-on-stormforge-custom-permissions.title]] +[#add-on-stormforge-custom-permissions] === Custom IAM permissions Custom permissions aren't used with this add-on. -[[add-on-splunk,add-on-splunk.title]] -== [.noloc]`Splunk` +[#add-on-suse] +== SUSE +* *Publisher* – SUSE +* *Name* – `suse_observability-agent` +* *Namespace* – `suse-observability` +* *Service account name* – A service account isn't used with this add-on. +* *{aws} managed IAM policy* – A managed policy isn't used with this add-on. +* *Custom IAM permissions* – Custom permissions aren't used with this add-on. +* *Setup and usage instructions* – See https://docs.stackstate.com/get-started/k8s-quick-start-guide#amazon-eks[Quick Start] in the SUSE documentation. + +[#add-on-splunk] +== Splunk [abstract] -- -Learn about the [.noloc]`Splunk` Amazon EKS add-on. +Learn about the Splunk Amazon EKS add-on. -- -The add-on name is `splunk_splunk-otel-collector-chart` and the namespace is `splunk-monitoring`. [.noloc]`Splunk` publishes the add-on. +The add-on name is `splunk_splunk-otel-collector-chart` and the namespace is `splunk-monitoring`. Splunk publishes the add-on. -For information about the add-on, see https://docs.splunk.com/observability/en/gdi/opentelemetry/install-k8s-addon-eks.html[Install the Splunk add-on for Amazon EKS] in the Splunk documentation. +For information about the add-on, see https://help.splunk.com/en/splunk-observability-cloud/manage-data/splunk-distribution-of-the-opentelemetry-collector/get-started-with-the-splunk-distribution-of-the-opentelemetry-collector/collector-for-kubernetes/kubernetes-eks-add-on[Install the Splunk add-on for Amazon EKS] in the Splunk documentation. -[[add-on-splunk-service-account-name,add-on-splunk-service-account-name.title]] +[#add-on-splunk-service-account-name] === Service account name A service account isn't used with this add-on. -[[add-on-splunk-managed-policy,add-on-splunk-managed-policy.title]] +[#add-on-splunk-managed-policy] === {aws} managed IAM policy A managed policy isn't used with this add-on. -[[add-on-splunk-custom-permissions,add-on-splunk-custom-permissions.title]] +[#add-on-splunk-custom-permissions] === Custom IAM permissions Custom permissions aren't used with this add-on. -[[add-on-teleport,add-on-teleport.title]] -== [.noloc]`Teleport` +[#add-on-teleport] +== Teleport [abstract] -- -Learn about the [.noloc]`Teleport` Amazon EKS add-on. +Learn about the Teleport Amazon EKS add-on. -- -The add-on name is `teleport_teleport` and the namespace is `teleport`. [.noloc]`Teleport` publishes the add-on. +The add-on name is `teleport_teleport` and the namespace is `teleport`. Teleport publishes the add-on. -For information about the add-on, see https://goteleport.com/how-it-works/[How Teleport Works] in the [.noloc]`Teleport` documentation. +For information about the add-on, see https://goteleport.com/how-it-works/[How Teleport Works] in the Teleport documentation. -[[add-on-teleport-service-account-name,add-on-teleport-service-account-name.title]] +[#add-on-teleport-service-account-name] === Service account name A service account isn't used with this add-on. -[[add-on-teleport-managed-policy,add-on-teleport-managed-policy.title]] +[#add-on-teleport-managed-policy] === {aws} managed IAM policy A managed policy isn't used with this add-on. -[[add-on-teleport-custom-permissions,add-on-teleport-custom-permissions.title]] +[#add-on-teleport-custom-permissions] === Custom IAM permissions Custom permissions aren't used with this add-on. -[[add-on-tetrate,add-on-tetrate.title]] -== [.noloc]`Tetrate` +[#add-on-tetrate] +== Tetrate [abstract] -- -Learn about the [.noloc]`Tetrate` Amazon EKS add-on. +Learn about the Tetrate Amazon EKS add-on. -- -The add-on name is `tetrate-io_istio-distro` and the namespace is `istio-system`. [.noloc]`Tetrate Io` publishes the add-on. +The add-on name is `tetrate-io_istio-distro` and the namespace is `istio-system`. Tetrate Io publishes the add-on. For information about the add-on, see the https://tetratelabs.io/[Tetrate Istio Distro] website. -[[add-on-tetrate-service-account-name,add-on-tetrate-service-account-name.title]] +[#add-on-tetrate-service-account-name] === Service account name A service account isn't used with this add-on. -[[add-on-tetrate-managed-policy,add-on-tetrate-managed-policy.title]] +[#add-on-tetrate-managed-policy] === {aws} managed IAM policy A managed policy isn't used with this add-on. -[[add-on-tetrate-custom-permissions,add-on-tetrate-custom-permissions.title]] +[#add-on-tetrate-custom-permissions] === Custom IAM permissions Custom permissions aren't used with this add-on. -[[add-on-upbound,add-on-upbound.title]] -== [.noloc]`Upbound Universal Crossplane` +[#add-on-upbound] +== Upbound Universal Crossplane [abstract] -- -Learn about the [.noloc]`Upbound Universal Crossplane` Amazon EKS add-on. +Learn about the Upbound Universal Crossplane Amazon EKS add-on. -- -The add-on name is `upbound_universal-crossplane` and the namespace is `upbound-system`. [.noloc]`Upbound` publishes the add-on. +The add-on name is `upbound_universal-crossplane` and the namespace is `upbound-system`. Upbound publishes the add-on. For information about the add-on, see https://docs.upbound.io/uxp/[Upbound Universal Crossplane (UXP)] in the Upbound documentation. -[[add-on-upbound-service-account-name,add-on-upbound-service-account-name.title]] +[#add-on-upbound-service-account-name] === Service account name A service account isn't used with this add-on. -[[add-on-upbound-managed-policy,add-on-upbound-managed-policy.title]] +[#add-on-upbound-managed-policy] === {aws} managed IAM policy A managed policy isn't used with this add-on. -[[add-on-upbound-custom-permissions,add-on-upbound-custom-permissions.title]] +[#add-on-upbound-custom-permissions] === Custom IAM permissions Custom permissions aren't used with this add-on. -[[add-on-upwind,add-on-upwind.title]] -== [.noloc]`Upwind` +[#add-on-upwind] +== Upwind [abstract] -- -Learn about the [.noloc]`Upwind` Amazon EKS add-on. +Learn about the Upwind Amazon EKS add-on. -- -The add-on name is `upwind` and the namespace is `upwind`. [.noloc]`Upwind` publishes the add-on. +The add-on name is `upwind` and the namespace is `upwind`. Upwind publishes the add-on. For information about the add-on, see https://docs.upwind.io/install-sensor/kubernetes/install?installation-method=amazon-eks-addon[Upwind documentation]. -[[add-on-upwind-service-account-name,add-on-upwind-service-account-name.title]] +[#add-on-upwind-service-account-name] === Service account name A service account isn't used with this add-on. -[[add-on-upwind-managed-policy,add-on-upwind-managed-policy.title]] +[#add-on-upwind-managed-policy] === {aws} managed IAM policy A managed policy isn't used with this add-on. -[[add-on-upwind-custom-permissions,add-on-upwind-custom-permissions.title]] +[#add-on-upwind-custom-permissions] === Custom IAM permissions Custom permissions aren't used with this add-on. - - diff --git a/latest/ug/zonbook.redirects.xml b/latest/ug/zonbook.redirects.xml index 413eb381d..87e71ba7a 100644 --- a/latest/ug/zonbook.redirects.xml +++ b/latest/ug/zonbook.redirects.xml @@ -146,9 +146,12 @@ + + + diff --git a/team/squish.adoc b/team/squish.adoc new file mode 100644 index 000000000..f81de1222 --- /dev/null +++ b/team/squish.adoc @@ -0,0 +1,125 @@ += AmazonEKSDocs Git History Squish Procedure + +== Overview +This procedure outlines the steps to clear the history of the "mainline" branch while maintaining a backup. The backup branch will be named with the current date (e.g., `13JAN2025-mainline`). + +== Prerequisites +* Ensure you have the latest version of the repository +* Have appropriate permissions to push to mainline +* Verify you can build and deploy to alpha environment + +== How to: Squish + +* Only one person needs to do the branch work +* Then everyone needs to reset mainline on their devices + +=== Initial Setup Steps + +[source,bash] +---- +# Sync and checkout mainline: +git fetch origin +git checkout mainline +git pull origin mainline + +# Create a backup branch: +git checkout -b 13JAN2025-mainline +git push origin 13JAN2025-mainline +---- + +=== History Clean-up Steps + +[source,bash] +---- +# Create new orphan branch: +git checkout --orphan new/mainline + +# Stage and commit all files: +git add . +git commit -m "Fresh start: History squish for mainline branch" +---- + +Verify build functionality: +* Build the project locally + +[source,bash] +---- +# Push the new branch: +git push origin new/mainline +---- + +NOTE: If a branch with this name already exists, deprecate it first + +Deploy to alpha environment. Verify all functionality works as expected. + +=== Production Switch + +. After alpha verification, deprecate mainline on code.amazon.com + +[source,bash] +---- +# Clean up local branches: +git branch -D mainline +git checkout new/mainline + +# Rename and establish new mainline: +git checkout -b mainline +git push origin mainline +---- + +=== Troubleshooting +* If you encounter push errors, ensure you've properly deprecated the old branches +* If build fails, verify all files were properly carried over in the orphan branch +* For any issues, consult with the team lead before proceeding + +=== Rollback Plan +If issues arise, the backup branch (`13JAN2025-mainline`) can be used to restore the previous state. + +== How to: update local copy + +All team members must execute these steps to sync with the new mainline: + +[source,bash] +---- +# Switch to the new branch structure: +git checkout new/mainline + +# Remove old mainline reference: +git branch -D mainline + +# Update and switch to new mainline: +git fetch +git checkout mainline +---- + +== How to: Deprecate branch of code.amazon.com + +. Open package on code.amazon.com +. In Repo info tab, scroll down to "Deprecate branch" dropdown menu +. Select the name of the branch and choose deprecate + +NOTE: The branch is just archived and hidden, and it can be restored. + +== How to: Add github remote + +Ensure your SSH is configured with GitHub + +[source,bash] +---- +git remote add github git@github.com:awsdocs/amazon-eks-user-guide.git +---- + + +== How to: Merge in GitHub changes + +[source,bash] +---- +git fetch github/mainline +---- + +== How to: Publish changes to github + +[source,bash] +---- +git push github +---- \ No newline at end of file diff --git a/vale/styles/EksDocs/ExternalDomains.yml b/vale/styles/EksDocs/ExternalDomains.yml index a4431cc33..73dd0b766 100644 --- a/vale/styles/EksDocs/ExternalDomains.yml +++ b/vale/styles/EksDocs/ExternalDomains.yml @@ -4,115 +4,218 @@ level: error scope: raw #ignorecase: true tokens: - - '(?:^|\s)(http(s)?://[^\s]+)(?:$|\s)' + - '(?:^|\b)(http(s)?:\/\/[^\s\[]+)(?=$|\s|\[)' + # note: this version no longer detects the valid URL that uses replacement syntax https://raw.githubusercontent.com/projectcalico/calico/\[\.replaceable\]\`CALICO_VERSION\`/manifests/operator-crds.yaml exceptions: - - https://aws.github.io/ - - https://kubernetes-sigs.github.io/ - - https://aws-observability.github.io/ - - https://github.com/aws/ - - https://github.com/aws-samples/ - - https://github.com/kubernetes/ - - https://repost.aws/ - - https://github.com/bottlerocket-os/ - - https://kubernetes.io - - https://karpenter.sh - - https://anywhere.eks.amazonaws.com - - https://aws-ia.github.io - - https://eksctl.io - - https://catalog.workshops.aws - - https://github.com/awslabs/ - - https://github.com/aws-controllers-k8s/ - - https://raw.githubusercontent.com/aws-observability/ - - https://github.com/kubernetes-sigs/ - - https://d1.awsstatic.com/ - - https://docs.github.com/ - - https://code.visualstudio.com/ - - https://cli.github.com/ - - https://marketplace.visualstudio.com/ - - https://docs.asciidoctor.org/ - - https://brew.sh/ - - https://github.dev/aws/ - - https://catalog.us-east-1.prod.workshops.aws/ - - http://developers.eksworkshop.com - - https://s3.us-west-2.amazonaws.com/amazon-eks/ - - https://www.eksworkshop.com - - https://community.aws/ - - https://www.youtube.com/ - - https://kind.sigs.k8s.io - - https://minikube.sigs.k8s.io - - https://www.terraform.io - - https://developers.eksworkshop.com - - https://docs.aws.amazon.com/ - - https://groups.google.com/forum/#!topic/kubernetes-security-announce - - https://raw.githubusercontent.com/kubernetes-sigs - - https://helm.sh - - https://public.ecr.aws - - https://gallery.ecr.aws - - https://amazon-eks.s3.us-west-2.amazonaws.com/eks-connector/ - - https://www.intel.com/ - - https://www.tensorflow.org/ - - https://docs.upbound.io/ - - https://docs.upwind.io/ - - https://tetratelabs.io/ - - https://goteleport.com/ - - https://docs.splunk.com/ - - https://docs.stormforge.io/ - - https://docs.snyk.io/ - - https://docs.solo.io/ + - 'https://bugs.launchpad.net/ubuntu/\+source/containerd-app/\+bug/2065423' - alas.aws.amazon.com - amazon.awsapps.com + - anywhere.eks.amazonaws.com - apparmor.net - appdynamics.com + - aws-ia.github.io + - aws-observability.github.io - aws-otel.github.io + - aws.amazon.com + - aws.github.io - awsdocs-neuron.readthedocs-hosted.com - awslabs.github.io + - boto3.amazonaws.com + - bottlerocket.dev - brew.sh - catalog.redhat.com + - catalog.us-east-1.prod.workshops.aws + - catalog.workshops.aws - chocolatey.org - cloud-images.ubuntu.com - cloudinit.readthedocs.io + - community.aws + - console.aws.amazon.com - containerd.io + - cve.mitre.org + - d1.awsstatic.com/ + - developer.hashicorp.com + - developer.nvidia.com + - developers.eksworkshop.com - distro.eks.amazonaws.com + - docs.aws.amazon.com + - docs.cilium.io - docs.docker.com + - docs.fluentbit.io + - docs.helm.sh + - docs.kubecost.com - docs.netapp.com - docs.newrelic.com - docs.pingidentity.com - docs.podman.io + - docs.rad.security + - docs.ray.io + - docs.tigera.io + - docs.vllm.ai - documentation.solarwinds.com - documentation.ubuntu.com - ebpf.io - eks.amazonaws.com + - eksctl.io - eksworkshop.com - etcd.io - explore.skillbuilder.aws - gateway-api.sigs.k8s.io - github.com - grafana.com + - guide.kubecost.com + - http://developers.eksworkshop.com + - http://192.0.2.4:3128 + - http://localhost:8080/utility/stress/1000000 + - http://localhost:9090 + - http://retail-store-sample-ui.default.svc/utility/stress/1000000 + - https://access.redhat.com/security/security-updates/security-advisories + - https://amazon-eks.s3.us-west-2.amazonaws.com/eks-connector/ + - https://antrea.io/docs/main/docs/eks-installation + - https://anywhere.eks.amazonaws.com + - https://aws-ia.github.io + - https://aws-observability.github.io/ + - https://aws.github.io/ + - https://brew.sh/ + - https://catalog.us-east-1.prod.workshops.aws/ + - https://catalog.workshops.aws + - https://cert-manager.io + - https://cilium.io + - https://cli.github.com/ + - https://code.visualstudio.com/ + - https://community.aws/ + - https://d1.awsstatic.com/ + - https://developers.eksworkshop.com + - https://distribution.github.io/distribution/ + - https://docs.akuity.io/tutorials/eks-addon-agent-install/ + - https://docs.asciidoctor.org/ + - https://docs.aws.amazon.com/ + - https://docs.cloudsoft.io/operations/configuration/aws-eks-addon.html + - https://docs.cribl.io/edge/usecase-edge-aws-eks/ + - https://docs.datadoghq.com/containers/guide/operator-eks-addon/ + - https://docs.datadoghq.com/containers/guide/operator-eks-addon/\?tab=console + - https://docs.github.com/ + - https://docs.groundcover.com/docs/~/changes/VhDDAl1gy1VIO3RIcgxD/configuration/customization-guide/customize-deployment/eks-add-on + - https://docs.guance.com/en/datakit/datakit-eks-deploy/ + - https://docs.guance.com/en/datakit/datakit-eks-deploy/#add-on-install + - https://docs.kasten.io/latest/install/aws-eks-addon/aws-eks-addon.html + - https://docs.kpow.io/installation/aws-marketplace-lm/ + - https://docs.kubearmor.io/kubearmor/quick-links/deployment_guide + - https://docs.nvidia.com/cuda/cuda-runtime-api/group_%5FCUDART%5F_DEVICE.html + - https://docs.nvidia.com/deploy/gpu-debug-guidelines/index.html#understanding-xid-messages + - https://docs.nvidia.com/deploy/xid-errors/index.html#topic_5_1 + - https://docs.projectcalico.org/getting-started/kubernetes/managed-public-cloud/eks + - https://docs.rafay.co/clusters/import/eksaddon/ + - https://docs.snyk.io/ + - https://docs.solo.io/ + - https://docs.splunk.com/ + - https://docs.stormforge.io/ + - https://docs.upbound.io/ + - https://docs.upwind.io/ + - https://download.docker.com/linux/ubuntu + - https://download.docker.com/linux/ubuntu/gpg + - https://eksctl.io + - https://en.wikipedia.org/wiki/Reserved_IP_addresses + - https://example.com + - https://gallery.ecr.aws + - https://github.com/aws-controllers-k8s/ + - https://github.com/aws-samples/ + - https://github.com/aws/ + - https://github.com/awslabs/ + - https://github.com/bottlerocket-os/ + - https://github.com/kubernetes-sigs/ + - https://github.com/kubernetes/ + - https://github.dev/aws/ + - https://github.dev/awsdocs/ + - https://goteleport.com/ + - https://groups.google.com/forum/#!msg/kubernetes-security-announce/jk8polzSUxs/dfq6a-MnCQAJ + - https://groups.google.com/forum/#!topic/kubernetes-security-announce + - https://helm.sh + - https://hub.datree.io/integrations/eks-integration + - https://karpenter.sh + - https://kind.sigs.k8s.io + - https://kong.github.io/aws-marketplace-addon-kong-gateway/ + - https://kubernetes-sigs.github.io/ + - https://kubernetes.io + - https://marketplace.visualstudio.com/ + - https://minikube.sigs.k8s.io + - https://play2048.co/ + - https://prometheus-community.github.io/helm-charts + - https://public.ecr.aws + - https://pyjwt.readthedocs.io/en/latest/ + - https://ratify.dev/docs/1.0/quickstarts/ratify-on-aws + - https://raw.githubusercontent.com/aws-observability/ + - https://raw.githubusercontent.com/aws-samples/ + - https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/v1.12.6/config/master/cni-metrics-helper.yaml + - https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/v1.19.2/config/master/aws-k8s-cni.yaml + - https://raw.githubusercontent.com/aws/eks-charts/master/stable/aws-load-balancer-controller/crds/crds.yaml + - https://raw.githubusercontent.com/awslabs/amazon-eks-ami/ + - https://raw.githubusercontent.com/awslabs/amazoneks-ami/ + - https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + - https://raw.githubusercontent.com/kubecost/cost-analyzer-helm-chart/develop/cost-analyzer/values-eks-cost-monitoring.yaml + - https://raw.githubusercontent.com/kubernetes-sigs + - https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/ + - https://raw.githubusercontent.com/projectcalico/calico/ + - https://repost.aws/ + - https://s3.amazonaws.com/EULA/ + - https://s3.us-west-2.amazonaws.com/amazon-eks/ + - https://tetratelabs.io/ + - https://tinkerbell.org + - https://ubuntu.com/security/notices?order=newest&release=focal + - https://ubuntu.com/security/notices?order=newest&release=jammy + - https://ubuntu.com/security/notices?order=newest&release=noble + - https://vale.sh/ + - https://www.cncf.io/ + - https://www.dynatrace.com/technologies/kubernetes-monitoring/ + - https://www.eksworkshop.com + - https://www.intel.com/ + - https://www.leaksignal.com/docs/LeakAgent/Deployment/ + - https://www.powershellgallery.com/packages/{aws}.Tools.Common/4.1.502 + - https://www.powershellgallery.com/packages/Watch-Command/0.1.3 + - https://www.qemu.org/docs/ + - https://www.tensorflow.org/ + - https://www.terraform.io + - https://www.tigera.io/project-calico + - https://www.youtube.com/ - hub.docker.com + - huggingface.co - k8s.io + - karpenter.sh - kubernetes-csi.github.io + - kubernetes.io - learn.microsoft.com + - man7.org - oidc.eks.amazonaws.com - oidc.eks.region-code.amazonaws.com + - oidc.eks.us-west-2.amazonaws.com - opencontainers.org - openid.net + - opensearch.org - operatorhub.io - packer.io - portal.msrc.microsoft.com - prometheus.io + - public.ecr.aws - pytorch.org - registry.terraform.io + - repost.aws - requests.readthedocs.io + - rolesanywhere.amazonaws.com - rubygems.org - support.microsoft.com - tensorflow.org + - tetratelabs.io - ubuntu.com - www.canonical.com - www.cisecurity.org + - www.eksworkshop.com + - www.gateway-api-controller.eks.aws.dev - www.itu.int - www.juniper.net - www.python.org - www.rfc-editor.org - xilinx.github.io - - https://cert-manager.io - - https://www.cncf.io/ + - sdk.amazonaws.com + - efa-installer.amazonaws.com + - amazon-eks.s3.amazonaws.com + - amazon-eks-docs.s3.amazonaws.com \ No newline at end of file diff --git a/vale/styles/config/vocabularies/EksDocsVocab/accept.txt b/vale/styles/config/vocabularies/EksDocsVocab/accept.txt index 3fef970f0..d7a8bb16c 100644 --- a/vale/styles/config/vocabularies/EksDocsVocab/accept.txt +++ b/vale/styles/config/vocabularies/EksDocsVocab/accept.txt @@ -1,3 +1,4 @@ +# Each line is a Ruby regex EKS eksctl Fargate @@ -5,6 +6,23 @@ Bottlerocket EBS StorageClass PersistentVolume -CSI +CSIs? Karpenter -VPC \ No newline at end of file +VPCs? +VPC Reachability Analyzer +reachability +CNIs? +repo +CIDRs? +Firehose +Kinesis +AMIs? +SDKs? +FSx +ENIs? +IPs? +CSRs? +routable +ARN +CloudWatch +CoreDNS \ No newline at end of file