Merge branch 'feature/multiple-imds-endpoints' into 'main'

support IPv6/multiple IMDS endpoints Closes #68 See merge request alpine/cloud/tiny-cloud!155
2026-06-21 00:07:16 +03:00 · 2026-06-15 12:31:48 +00:00 · 2026-06-15 12:31:48 +00:00 · 6bd78dcd45
commit 6bd78dcd45
parent 19688765e9 9efc7c0bb1
10 changed files with 254 additions and 10 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -1,5 +1,14 @@
 # CHANGELOG

+## 2026-06-XX - Tiny Cloud v3.3.3
+
+* Support IPv6 and multiple endpoints
+  [#68](https://gitlab.alpinelinux.org/alpine/cloud/tiny-cloud/-/work_items/68)
+
+* Check for IP routes to IMDS endpoints before trying them; retry if none are
+  routable.  Fixes race condition between `dhcpcd` starting and attempting to
+  reach IMDS before routes are resolved.
+
 ## 2026-06-08 - Tiny Cloud v3.3.2

 * Fixes an autodetect regression introduced in v3.3.1
--- a/README.md
+++ b/README.md
@ -131,6 +131,19 @@ The default endpoint is `169.254.169.254` for most cloud providers. This
 setting allows you to specify a custom IP address and optional port for the
 metadata service.

+Multiple endpoints can be configured with `IMDS_ENDPOINTS`. The list is tried
+in order, and the most recently working endpoint is tried first on later
+queries. `IMDS_ENDPOINTS` defaults to `IMDS_ENDPOINT` when unset. IPv6 endpoints
+must use URL-style brackets:
+
+```sh
+IMDS_ENDPOINTS="169.254.169.254 [fd00:ec2::254]"
+```
+
+Tiny Cloud checks for routes to configured IMDS endpoints before trying metadata
+requests. `IMDS_ENDPOINT_WAIT_ATTEMPTS` controls how many route checks are made
+before metadata requests are tried anyway.
+
 ### Metadata API Version

 Each provider's API has a built-in default version. You can override the
--- a/bin/imds
+++ b/bin/imds
@ -53,6 +53,9 @@ unset -f \
 # Common to many clouds
 # Can be overridden in /etc/tiny-cloud.conf
 : "${IMDS_ENDPOINT:=169.254.169.254}"
+: "${IMDS_ENDPOINTS:=$IMDS_ENDPOINT}"
+: "${IMDS_ENDPOINT_CACHE:=$TINY_CLOUD_VAR/.imds-endpoint}"
+: "${IMDS_ENDPOINT_WAIT_ATTEMPTS:=10}"

 # Common to AWS and NoCloud(ish)
 IMDS_HOSTNAME="meta-data/hostname"
@ -68,10 +71,84 @@ IMDS_IPV6_NET="subnet-ipv6-cidr-blocks"
 IMDS_IPV4_PREFIX="ipv4-prefix"
 IMDS_IPV6_PREFIX="ipv6-prefix"

+_imds_endpoints() {
+	local cached e
+	cached="$(cat "$IMDS_ENDPOINT_CACHE" 2>/dev/null)" || :
+	for e in $IMDS_ENDPOINTS; do
+		[ "$e" = "$cached" ] && echo "$e"
+	done
+	for e in $IMDS_ENDPOINTS; do
+		[ "$e" != "$cached" ] && echo "$e"
+	done
+}
+
+_imds_host_port() {
+	local host port
+	case "$1" in
+		\[*\]:*)
+			host="${1#\[}"
+			host="${host%%\]*}"
+			port="${1##*\]:}"
+			;;
+		\[*\])
+			host="${1#\[}"
+			host="${host%\]}"
+			port=80
+			;;
+		*:*)
+			host="${1%:*}"
+			port="${1##*:}"
+			;;
+		*)
+			host="$1"
+			port=80
+			;;
+	esac
+	echo "$host"
+	echo "$port"
+}
+
+_imds_has_route() {
+	local host
+	set -- $(_imds_host_port "$1")
+	host="$1"
+	case "$host" in
+		*:*) ip -6 route get "$host" >/dev/null 2>&1 ;;
+		[0-9]*.[0-9]*.[0-9]*.[0-9]*) ip route get "$host" >/dev/null 2>&1 ;;
+		*) return 0 ;;
+	esac
+}
+
 _imds() {
-	wget --quiet --timeout 1 --output-document - \
-		--header "$(_imds_header)" \
-		"http://$IMDS_ENDPOINT/$IMDS_URI/$1$IMDS_QUERY"
+	local endpoint endpoints routed attempts=1
+	while :; do
+		endpoints=
+		routed=
+		for endpoint in $(_imds_endpoints); do
+			if _imds_has_route "$endpoint"; then
+				endpoints="$endpoints $endpoint"
+				routed=1
+			fi
+		done
+		[ -n "$routed" ] && break
+		[ "$attempts" -ge "$IMDS_ENDPOINT_WAIT_ATTEMPTS" ] && {
+			endpoints="$(_imds_endpoints)"
+			break
+		}
+		sleep 1
+		attempts=$((attempts + 1))
+	done
+	for endpoint in $endpoints; do
+		IMDS_ENDPOINT="$endpoint"
+		IMDS_CURRENT_ENDPOINT="$endpoint"
+		wget --quiet --timeout 1 --output-document - \
+			--header "$(_imds_header)" \
+			"http://$endpoint/$IMDS_URI/$1$IMDS_QUERY" && {
+				echo "$endpoint" > "$IMDS_ENDPOINT_CACHE"
+				return 0
+			}
+	done
+	return 1
 }

 _imds_userdata() { _imds "$IMDS_USERDATA"; }
--- a/docs/imds.1.scd
+++ b/docs/imds.1.scd
@ -98,6 +98,18 @@ inside *@nic:* queries.
 *CLOUD*
 	Cloud provider name. When set to *auto*, the autodetected provider is used.

+*IMDS_ENDPOINT*
+	Single provider metadata endpoint. Used as the default value for
+	*IMDS_ENDPOINTS*.
+
+*IMDS_ENDPOINTS*
+	Whitespace-separated provider metadata endpoint list. IPv6 endpoints must use
+	brackets, for example *[fd00:ec2::254]*.
+
+*IMDS_ENDPOINT_WAIT_ATTEMPTS*
+	Number of times to check for routes to any IMDS endpoint before trying
+	metadata requests anyway. The default is *10*.
+
 # EXIT STATUS

 *0*
--- a/docs/tiny-cloud.conf.5.scd
+++ b/docs/tiny-cloud.conf.5.scd
@ -37,10 +37,20 @@ Blank lines and shell comments are ignored.
 	Provider's API version to use. Providers that have versioned APIs have
 	built-in default values.

-*IMDS_ENDPOINT*=<ip_address>
+*IMDS_ENDPOINT*=<ip_address[:port]>
 	Provider endpoint IP address to use. Defaults to 169.254.169.254 for many
 	providers.

+*IMDS_ENDPOINTS*=<endpoint> ...
+	Whitespace-separated provider endpoint list. Defaults to *IMDS_ENDPOINT*.
+	Endpoints are tried in order, with the most recently working endpoint tried
+	first on later queries. IPv6 endpoints must use brackets, for example
+	*[fd00:ec2::254]* or *[fd00:ec2::254]:80*.
+
+*IMDS_ENDPOINT_WAIT_ATTEMPTS*=<number>
+	Number of times to check for routes to any IMDS endpoint before trying
+	metadata requests anyway. The default is *10*.
+
 *IMDS_TOKEN_TTL*=<seconds>
 	Metadata token lifetime in seconds for AWS metadata access. This is only used
 	by the AWS provider. The default is *5*.
--- a/lib/tiny-cloud/cloud/aws/imds
+++ b/lib/tiny-cloud/cloud/aws/imds
@ -9,14 +9,18 @@ IMDS_TOKEN_TTL_HEADER="X-aws-ec2-metadata-token-ttl-seconds"
 IMDS_URI="$IMDS_API_VERSION"

 _imds_token() {
+	local host port
 	# Only try to get token if using IMDSv2
 	# IMDSv1: API versions 2009-04-04 and earlier (no token support)
 	# IMDSv2: API versions 2009-04-05 and later, or 'latest' (requires token)
 	expr "$IMDS_API_VERSION" "<=" "2009-04-04" > /dev/null && return
+	set -- $(_imds_host_port "${IMDS_CURRENT_ENDPOINT:-$IMDS_ENDPOINT}")
+	host="$1"
+	port="$2"
 	# IMDSv2 - request token
 	printf "PUT /latest/api/token HTTP/1.0\r\n%s: %s\r\n\r\n" \
 		"$IMDS_TOKEN_TTL_HEADER" "$IMDS_TOKEN_TTL" \
-		| nc -w 1 "$IMDS_ENDPOINT" 80 | tail -n 1
+		| nc -w 1 "$host" "$port" | tail -n 1
 }

 _imds_header() {
--- a/lib/tiny-cloud/tiny-cloud.conf
+++ b/lib/tiny-cloud/tiny-cloud.conf
@ -13,6 +13,14 @@
 # Useful for custom metadata services
 #IMDS_ENDPOINT=169.254.169.254

+# Ordered IMDS endpoint list. Defaults to IMDS_ENDPOINT.
+# IPv6 endpoints must use brackets: [fd00:ec2::254] or [fd00:ec2::254]:80
+#IMDS_ENDPOINTS="169.254.169.254 [fd00:ec2::254]"
+
+# Number of times to check for routes to any IMDS endpoint before trying
+# metadata requests anyway.
+#IMDS_ENDPOINT_WAIT_ATTEMPTS=10
+
 # IMDS API version
 # Most providers have a default version, overrideable here if necessary
 #IMDS_API_VERSION=""
--- a/tests/bin/wget
+++ b/tests/bin/wget
@ -78,11 +78,19 @@ for url; do
 	esac
 done

-host="${url#http*://}"
-host="${host%%/*}"
-path="${url#http*://$host}"
+url_no_scheme="${url#*://}"
+case "$url_no_scheme" in
+	*/*) host="${url_no_scheme%%/*}"; path="/${url_no_scheme#*/}";;
+	*) host="$url_no_scheme"; path=/;;
+esac
 path="${path#${WGET_STRIP_PREFIX:-/}}"
 path="${path%\?*}"
+[ -n "$WGET_HOST_LOG" ] && echo "$host" >> "$WGET_HOST_LOG"
+case "$host" in
+	\[*\]:*) yaml_host="${host#\[}"; yaml_host="${yaml_host%%\]*}";;
+	\[*\]) yaml_host="${host#\[}"; yaml_host="${yaml_host%\]}";;
+	*) yaml_host="$host";;
+esac


 if [ -z "$WGETCONTENT" ]; then
@ -93,7 +101,7 @@ if [ -z "$WGETCONTENT" ]; then
 		(
 		IFS=/
 		set -- ${path#/}
-		yx -f "${WGET_YAML:-$host.yaml}" "$@" 2>/dev/null
+		yx -f "${WGET_YAML:-$yaml_host.yaml}" "$@" 2>/dev/null
 		)
 	fi
 fi
@ -107,4 +115,3 @@ case "$outfile" in
 		echo "$WGETCONTENT" > "$outfile"
 		;;
 esac
-
--- a/tests/imds.test
+++ b/tests/imds.test
@ -10,6 +10,11 @@ PROVIDERS="aws azure digitalocean gcp hetzner incus oci nocloud scaleway"
 init_tests \
 	imds_help \
 	imds_space \
+	imds_endpoint_fallback \
+	imds_endpoint_cache \
+	imds_endpoint_ipv6 \
+	imds_endpoint_route_skip \
+	imds_endpoint_route_wait \
 	\
 	imds_hostname_aws \
 	imds_hostname_azure \
@ -44,6 +49,7 @@ init_tests \
 	imds_aws_api_version_imdsv1 \
 	imds_aws_api_version_imdsv2_explicit \
 	imds_aws_api_version_imdsv2_latest \
+	imds_aws_token_endpoint_port \
 	\
 	imds_nocloud_cmdline_local_hostname \
 	imds_nocloud_smbios_local_hostname \
@ -63,6 +69,81 @@ imds_space_body() {
 	done
 }

+imds_endpoint_fallback_body() {
+	IMDS_API_VERSION=2009-04-04 CLOUD=aws fake_metadata aws <<-EOF
+		hostname: myhostname
+	EOF
+	IMDS_API_VERSION=2009-04-04 IMDS_ENDPOINT_WAIT_ATTEMPTS=0 \
+		IMDS_ENDPOINTS="fail 169.254.169.254" CLOUD=aws atf_check \
+		-o match:"myhostname" \
+		imds @hostname
+	atf_check -o match:"^169.254.169.254$" cat var/lib/cloud/.imds-endpoint
+}
+
+imds_endpoint_cache_body() {
+	mkdir -p var/lib/cloud
+	echo "cached.example" > var/lib/cloud/.imds-endpoint
+	cat > cached.example.yaml <<-EOF
+		hostname: cached-hostname
+	EOF
+	cat > first.example.yaml <<-EOF
+		hostname: first-hostname
+	EOF
+	IMDS_API_VERSION=2009-04-04 IMDS_ENDPOINT_WAIT_ATTEMPTS=0 \
+		WGET_STRIP_PREFIX="/2009-04-04/meta-data" \
+		WGET_HOST_LOG="$PWD/hosts.log" \
+		IMDS_ENDPOINTS="first.example cached.example" CLOUD=aws atf_check \
+		-o match:"cached-hostname" \
+		imds @hostname
+	atf_check -o match:"^cached.example$" head -n 1 hosts.log
+}
+
+imds_endpoint_ipv6_body() {
+	cat > "fd00:ec2::254.yaml" <<-EOF
+		hostname: ipv6-hostname
+	EOF
+	IMDS_API_VERSION=2009-04-04 IMDS_ENDPOINT_WAIT_ATTEMPTS=0 \
+		WGET_STRIP_PREFIX="/2009-04-04/meta-data" \
+		WGET_HOST_LOG="$PWD/hosts.log" \
+		IMDS_ENDPOINTS="[fd00:ec2::254]" CLOUD=aws atf_check \
+		-o match:"ipv6-hostname" \
+		imds @hostname
+	atf_check -o match:"^\\[fd00:ec2::254\\]$" cat hosts.log
+}
+
+imds_endpoint_route_skip_body() {
+	IMDS_API_VERSION=2009-04-04 CLOUD=aws fake_metadata aws <<-EOF
+		hostname: myhostname
+	EOF
+	fake_bin ip <<-'EOF'
+		#!/bin/sh
+		[ "$3" = 169.254.169.254 ]
+	EOF
+	IMDS_API_VERSION=2009-04-04 \
+		IMDS_ENDPOINTS="192.0.2.1 169.254.169.254" CLOUD=aws atf_check \
+		-o match:"myhostname" \
+		imds @hostname
+	atf_check -o match:"^169.254.169.254$" cat var/lib/cloud/.imds-endpoint
+}
+
+imds_endpoint_route_wait_body() {
+	IMDS_API_VERSION=2009-04-04 CLOUD=aws fake_metadata aws <<-EOF
+		hostname: myhostname
+	EOF
+	fake_bin ip <<-'EOF'
+		#!/bin/sh
+		mkdir -p tmp
+		count=$(cat tmp/route-count 2>/dev/null || echo 0)
+		count=$((count + 1))
+		echo "$count" > tmp/route-count
+		[ "$count" -gt 1 ]
+	EOF
+	IMDS_API_VERSION=2009-04-04 IMDS_ENDPOINT_WAIT_ATTEMPTS=5 CLOUD=aws atf_check \
+		-o match:"myhostname" \
+		imds @hostname
+	atf_check -o match:"^2$" cat tmp/route-count
+}
+
 check_hostname() {
 	fake_metadata "$1" <<-EOF
 		# aws, digitalocean, hetzner, nocloud
@ -251,6 +332,28 @@ imds_aws_api_version_imdsv2_latest_body() {
 		imds @hostname
 }

+imds_aws_token_endpoint_port_body() {
+	cat > "fd00:ec2::254.yaml" <<-EOF
+		hostname: test-imdsv2-port
+	EOF
+	fake_bin nc <<-'NCEOF'
+		#!/bin/sh
+		while [ -n "$1" ]; do
+			case "$1" in
+				-w) shift 2;;
+				*) echo "$1" >> nc.args; shift;;
+			esac
+		done
+		cat > /dev/null
+		printf "HTTP/1.0 200 OK\r\n\r\nmock-token"
+	NCEOF
+	IMDS_API_VERSION=latest WGET_STRIP_PREFIX="/latest/meta-data" \
+		IMDS_ENDPOINTS="[fd00:ec2::254]:8080" CLOUD=aws atf_check \
+		-o match:"test-imdsv2-port" \
+		imds @hostname
+	atf_check -o match:"^fd00:ec2::254 8080 $" sh -c "tr '\n' ' ' < nc.args"
+}
+
 imds_nocloud_cmdline_local_hostname_body() {
 	atf_require_prog yx
 	mkdir proc
--- a/tests/test_env.sh
+++ b/tests/test_env.sh
@ -6,6 +6,7 @@ PATH="$atf_srcdir/bin:$srcdir/bin:$srcdir/sbin:$PATH"

 export TINY_CLOUD_BASEDIR="$srcdir"
 export ROOT="$PWD"
+export IMDS_ENDPOINT_WAIT_ATTEMPTS=0


 init_tests() {