1
0
mirror of https://gitlab.alpinelinux.org/alpine/cloud/tiny-cloud.git synced 2025-12-15 11:22:43 +03:00

Compressed User-Data / Initial NoCloud Support

This commit is contained in:
Jake Buchholz Göktürk 2022-08-02 01:24:53 +00:00
parent ed8bdc8c8b
commit d15ac6bed3
12 changed files with 252 additions and 108 deletions

View File

@ -1,22 +1,21 @@
PREFIX?=/
SUBPACKAGES = core network openrc aws azure gcp oci
SUBPACKAGES = core network openrc aws azure gcp oci nocloud
.PHONY: install $(SUBPACKAGES)
# installs all subpackages, then replaces cloud-specific config with example
install: $(SUBPACKAGES)
mv "$(PREFIX)"/etc/conf.d/tiny-cloud.example "$(PREFIX)"/etc/conf.d/tiny-cloud
core:
install -Dm755 -t "$(PREFIX)"/bin \
bin/imds
install -Dm644 -t "$(PREFIX)"/etc/conf.d \
etc/conf.d/tiny-cloud.example
install -Dm644 -t "$(PREFIX)"/lib/tiny-cloud \
lib/tiny-cloud/common \
lib/tiny-cloud/init-* \
lib/tiny-cloud/mdev
lib/tiny-cloud/mdev \
lib/tiny-cloud/tiny-cloud.conf
install -Dm644 lib/tiny-cloud/tiny-cloud.conf \
"$(PREFIX)"/etc/conf.d/tiny-cloud
network:
install -Dm644 -t "$(PREFIX)"/etc/network/interfaces.d \
@ -32,32 +31,24 @@ openrc:
install -Dm755 -t "$(PREFIX)"/etc/init.d \
etc/init.d/*
aws: conf_dir
aws:
install -Dm755 -t "$(PREFIX)"/lib/mdev \
lib/mdev/nvme-ebs-links
install -Dm644 -t "$(PREFIX)"/lib/tiny-cloud/aws \
lib/tiny-cloud/aws/*
sed -Ee 's/^#?CLOUD=.*/CLOUD=aws/' \
-Ee 's/^#?HOTPLUG_MODULES=.*/HOTPLUG_MODULES="vnic_eth_hotplug nvme_ebs_links"/' \
etc/conf.d/tiny-cloud.example > "$(PREFIX)"/etc/conf.d/tiny-cloud
azure: conf_dir
azure:
install -Dm644 -t $(PREFIX)/lib/tiny-cloud/azure \
lib/tiny-cloud/azure/*
sed -Ee 's/^#?CLOUD=.*/CLOUD=azure/' \
etc/conf.d/tiny-cloud.example > "$(PREFIX)"/etc/conf.d/tiny-cloud
gcp: conf_dir
gcp:
install -Dm644 -t $(PREFIX)/lib/tiny-cloud/gcp \
lib/tiny-cloud/gcp/*
sed -Ee 's/^#?CLOUD=.*/CLOUD=gcp/' \
etc/conf.d/tiny-cloud.example > "$(PREFIX)"/etc/conf.d/tiny-cloud
oci: conf_dir
oci:
install -Dm644 -t $(PREFIX)/lib/tiny-cloud/oci \
lib/tiny-cloud/oci/*
sed -Ee 's/^#?CLOUD=.*/CLOUD=oci/' \
etc/conf.d/tiny-cloud.example > "$(PREFIX)"/etc/conf.d/tiny-cloud
conf_dir:
mkdir -p "$(PREFIX)"/etc/conf.d
nocloud:
install -Dm644 -t $(PREFIX)/lib/tiny-cloud/nocloud \
lib/tiny-cloud/nocloud/*

View File

@ -8,10 +8,13 @@ do just what is necessary with a small footprint and minimal dependencies.
A direct descendant of [tiny-ec2-bootstrap](
https://gitlab.alpinelinux.org/alpine/cloud/tiny-ec2-bootstrap), Tiny Cloud
works with multiple cloud providers. Currently, the following are supported:
* AWS (Amazon Web Services)
* Azure (Microsoft Azure)
* GCP (Google Cloud Platform)
* OCI (Oracle Cloud Infrastructure)
* [AWS](https://aws.amazon.com) (Amazon Web Services)
* [Azure](https://azure.microsoft.com) (Microsoft Azure)
* [GCP](https://cloud.google.com) (Google Cloud Platform)
* [OCI](https://cloud.oracle.com) (Oracle Cloud Infrastructure)
* [NoCloud](
https://cloudinit.readthedocs.io/en/latest/topics/datasources/nocloud.html)
(cloud-init's NoCloud AWS-compatible data source)
## Features
@ -44,9 +47,12 @@ As Tiny Cloud is meant to be tiny, it has very few dependencies:
* `partx`
* `resize2fs`
* `sfdisk`
* [`yx`](https://gitlab.com/tomalok/yx)
(optional, allows NoCloud to extract metadata from YAML files)
Tiny Cloud has been developed specifically for use with the
[Alpine Cloud Images](https://gitlab.alpinelinux.org/alpine/cloud/alpine-cloud-images)
[Alpine Cloud Images](
https://gitlab.alpinelinux.org/alpine/cloud/alpine-cloud-images)
project, and as such, it is currently tailored for use with [Alpine Linux](
https://alpinelinux.org), the [OpenRC](https://github.com/OpenRC/openrc) init
system, and the `ext4` root filesystem. If you would like to see Tiny Cloud
@ -57,12 +63,12 @@ open an issue with your request -- or better yet, submit a merge request!
Typically, Tiny Cloud is installed and configured when building a cloud image,
and is available on Alpine Linux as the [`tiny-cloud`](
https://pkgs.alpinelinux.org/packages?name=tiny-cloud) APK...
https://pkgs.alpinelinux.org/packages?name=tiny-cloud*) APKs...
```
apk install tiny-cloud
apk install tiny-cloud-<cloud>
```
This will install the necessary init scripts, libraries, etc. plus any missing
dependencies.
dependencies for Tiny Cloud to support _`<cloud>`_.
Alternately, you can download a release tarball, and use `make` to install it.
@ -76,12 +82,12 @@ rc-update add tiny-cloud-final default
## Configuration
By default, Tiny Cloud expects configuration at `/etc/conf.d/tiny-cloud`,
The stock [`etc/conf.d/tiny-cloud`](etc/conf.d/tiny-cloud) file contains
details of all tuneable settings.
The stock [lib/tiny-cloud/tiny-cloud.conf`](lib/tiny-cloud/tiny-cloud.conf)
file contains details of all tuneable settings.
*Because Tiny Cloud does not currently do auto-detection, you **MUST** set a
_Because Tiny Cloud does not currently do auto-detection, you **MUST** set a
configuration value for `CLOUD` indicating which cloud provider will be used.
Current valid values are `aws`, `azure`, `gcp`, and `oci`.*
Current valid values are `aws`, `azure`, `gcp`, `oci`, and `nocloud`._
## Operation
@ -106,13 +112,18 @@ data, and sets up instance's hostname and the cloud user's SSH keys before
`tiny-cloud-final` should be the very last init script to run in the
**default** runlevel. By default, it saves the instance's user data to
`/var/lib/cloud/user-data`, which is overrideable via the `TINY_CLOUD_VAR`
andr `CLOUD_USERDATA` config settings.
`/var/lib/cloud/user-data`; the directory overrideable via the `TINY_CLOUD_VAR`
config setting.
If the user data is compressed, Tiny Cloud will decompress it. Currently
supported compression algorithms are `gzip`, `bzip2`, `unxz`, `lzma`, `lzop`,
`lz4`, and `zstd`. _(Note that `lz4` and `zstd` are not installed in Alpine
by default, and would need to be added to the image.)_
If the user data is a script starting with `#!/`, it will be executed; its
output (combined STDOUT and STDERR) and exit code are saved to
`/var/log/user-data.log` and `/var/log/user-data.exit`, respectively -- unless
overriden with `TINY_CLOUD_LOGS` and `CLOUD_USERDATA` config settings.
`/var/log/user-data.log` and `/var/log/user-data.exit`, respectively; the
directory is overrideable via the `TINY_CLOUD_LOGS` config setting.
If all went well, the very last thing `tiny-cloud-final` does is touch
a `.bootstrap-complete` file into existence in `/var/lib/cloud` or another

View File

@ -3,27 +3,16 @@
# Tiny Cloud - Instance MetaData Service client
### configuration
### configuration, common functions
[ -f /etc/conf.d/tiny-cloud ] && source /etc/conf.d/tiny-cloud
source /lib/tiny-cloud/common
### cloud-specific variables/functions
unset \
IMDS_HEADER \
IMDS_URI \
IMDS_QUERY \
IMDS_HOSTNAME \
IMDS_SSH_KEYS \
IMDS_USERDATA \
IMDS_NICS \
IMDS_MAC \
IMDS_IPV4 \
IMDS_IPV6 \
IMDS_IPV4_NET \
IMDS_IPV6_NET \
IMDS_IPV4_PREFIX \
IMDS_IPV6_PREFIX
IMDS_QUERY
unset -f \
_imds_token \
_imds_header \
@ -32,12 +21,40 @@ unset -f \
### default variables/functions
CLOUD="${CLOUD:-unknown}"
# Common to many clouds
IMDS_ENDPOINT="169.254.169.254"
_imds_ssh_keys() { _imds "$IMDS_SSH_KEYS"; }
# Common to AWS and NoCloud(ish)
IMDS_HOSTNAME="meta-data/hostname"
IMDS_SSH_KEYS="meta-data/public-keys"
IMDS_USERDATA="user-data"
IMDS_NICS="meta-data/network/interfaces/macs"
IMDS_MAC="mac"
IMDS_IPV4="local-ipv4s"
IMDS_IPV6="ipv6s"
IMDS_IPV4_NET="subnet-ipv4-cidr-block"
IMDS_IPV6_NET="subnet-ipv6-cidr-blocks"
IMDS_IPV4_PREFIX="ipv4-prefix"
IMDS_IPV6_PREFIX="ipv6-prefix"
_imds() {
wget --quiet --timeout 1 --output-document - \
--header "$(_imds_header)" \
"http://$IMDS_ENDPOINT/$IMDS_URI/$1$IMDS_QUERY"
}
_imds_userdata() { _imds "$IMDS_USERDATA"; }
_imds_ssh_keys() {
local key
for key in $(_imds "$IMDS_SSH_KEYS"); do
_imds "$IMDS_SSH_KEYS/${key%=*}/openssh-key"
echo
done | sort -u
}
_imds_nic_index() { cat "/sys/class/net/$1/address"; }
### load cloud-specific variables and functions
if [ ! -d /lib/tiny-cloud/"$CLOUD" ]; then
@ -48,12 +65,6 @@ source /lib/tiny-cloud/"$CLOUD"/imds
### non-overrideable functions
_imds() {
wget --quiet --timeout 1 --output-document - \
--header "$(_imds_header)" \
"http://$IMDS_ENDPOINT/$IMDS_URI/$1$IMDS_QUERY"
}
imds() {
local cmd args key rv err=1
if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then

View File

@ -6,19 +6,6 @@ IMDS_TOKEN_TTL_HEADER="X-aws-ec2-metadata-token-ttl-seconds"
IMDS_TOKEN_TTL=${IMDS_TOKEN_TTL:-5}
IMDS_URI="latest"
IMDS_HOSTNAME="meta-data/hostname"
IMDS_SSH_KEYS="meta-data/public-keys"
IMDS_USERDATA="user-data"
IMDS_NICS="meta-data/network/interfaces/macs"
IMDS_MAC="mac"
IMDS_IPV4="local-ipv4s"
IMDS_IPV6="ipv6s"
IMDS_IPV4_NET="subnet-ipv4-cidr-block"
IMDS_IPV6_NET="subnet-ipv6-cidr-blocks"
IMDS_IPV4_PREFIX="ipv4-prefix"
IMDS_IPV6_PREFIX="ipv6-prefix"
_imds_token() {
echo -ne "PUT /latest/api/token" \
"HTTP/1.0\r\n$IMDS_TOKEN_TTL_HEADER: $IMDS_TOKEN_TTL\r\n\r\n" |
@ -27,14 +14,4 @@ _imds_token() {
_imds_header() {
echo "$IMDS_HEADER: $(_imds_token)"
}
# digs deeper than the default
_imds_ssh_keys() {
local key
for key in $(imds "$IMDS_SSH_KEYS"); do
imds "$IMDS_SSH_KEYS/${key%=*}/openssh-key"
done | sort -u
}
_imds_nic_index() { cat "/sys/class/net/$1/address"; }
}

View File

@ -8,8 +8,17 @@ IMDS_URI="metadata/instance"
IMDS_HOSTNAME="compute/name"
IMDS_SSH_KEYS="compute/publicKeys"
IMDS_USERDATA="compute/userData"
IMDS_NICS="network/interface"
IMDS_NICS=""
# TODO: flesh out networking
unset \
IMDS_MAC \
IMDS_IPV4 \
IMDS_IPV6 \
IMDS_IPV4_NET \
IMDS_IPV6_NET \
IMDS_IPV4_PREFIX \
IMDS_IPV6_PREFIX
_imds_header() {
echo "$IMDS_HEADER: true"

View File

@ -1,8 +1,15 @@
# Tiny Cloud - common script functions
# vim: ts=4 et ft=sh:
# set defaults
[ -f /etc/conf.d/tiny-cloud ] && source /etc/conf.d/tiny-cloud
CLOUD="${CLOUD:-unknown}"
CLOUD_USER=${CLOUD_USER:-alpine}
TINY_CLOUD_LOGS=${TINY_CLOUD_LOGS:-/var/log}
TINY_CLOUD_VAR=${TINY_CLOUD_VAR:-/var/lib/cloud}
log() {
local facility=kern
local facility="kern"
local stderr
local tag=$(basename "$0")
while [ "${1:0:1}" = '-' ]; do
@ -17,7 +24,7 @@ log() {
[ -z "$DEBUG" ] && [ "$level" = debug ] && return
shift
logger $stderr -p "$facility.$level" -t "$tag" "$@"
logger $stderr -p "$facility.$level" -t "$tag[$$]" "$@"
case "$level" in
crit|alert|emerg) exit 1 ;;
esac

View File

@ -1,13 +1,8 @@
# Tiny Cloud - Common Initialization
# vim:set ts=4 et ft=sh:
[ -f /etc/conf.d/tiny-cloud ] && source /etc/conf.d/tiny-cloud
# set defaults
CLOUD_USER=${CLOUD_USER:-alpine}
CLOUD_USERDATA=${CLOUD_USERDATA:-user-data}
TINY_CLOUD_LOGS=${TINY_CLOUD_LOGS:-/var/log}
TINY_CLOUD_VAR=${TINY_CLOUD_VAR:-/var/lib/cloud}
source /lib/tiny-cloud/common
SKIP_INIT_ACTIONS=${SKIP_INIT_ACTIONS:-}
# is initial bootstrap already done?

View File

@ -3,22 +3,51 @@
source /lib/tiny-cloud/init-common
match_header() {
local bytes=$(echo -en "$1")
[ "$bytes" = "$(dd bs=1 count=${#bytes} if="$2" 2>/dev/null)" ]
}
save_userdata() {
skip_action save_userdata && return
imds -e @userdata > "$TINY_CLOUD_VAR/$CLOUD_USERDATA"
local userdata="$TINY_CLOUD_VAR/user-data"
local tmpfile=$(mktemp "$userdata.XXXXXX")
local cmd
imds -e @userdata > "$tmpfile"
cmd="cat"
if ! skip_action decompress_userdata; then
if match_header '\037\213\010' "$tmpfile"; then
cmd="gzip -dc"
elif match_header 'BZh' "$tmpfile"; then
cmd="bzip2 -dc"
elif match_header '\3757zXZ\000' "$tmpfile"; then
cmd="unxz -c"
elif match_header '\135\0\0\0' "$tmpfile"; then
cmd="lzma -dc"
elif match_header '\211\114\132' "$tmpfile"; then
cmd="lzop -dc"
elif match_header '\002!L\030' "$tmpfile"; then
cmd="lz4 -dc"
elif match_header '(\265/\375' "$tmpfile"; then
cmd="zstd -dc"
fi
fi
$cmd "$tmpfile" > "$userdata"
rm "$tmpfile"
}
is_userdata_script() {
head -n1 "$TINY_CLOUD_VAR/$CLOUD_USERDATA" | grep -q '#!/'
head -n1 "$TINY_CLOUD_VAR/user-data" | grep -q "#!/"
}
run_userdata() {
skip_action run_userdata && return
local log="$TINY_CLOUD_LOGS/$CLOUD_USERDATA.log"
local exit="$TINY_CLOUD_LOGS/$CLOUD_USERDATA.exit"
local userdata="$TINY_CLOUD_VAR/$CLOUD_USERDATA"
local log="$TINY_CLOUD_LOGS/user-data.log"
local exit="$TINY_CLOUD_LOGS/user-data.exit"
local userdata="$TINY_CLOUD_VAR/user-data"
chmod +x "$userdata"
{ "$userdata" 2>& 1; echo $? > "$exit"; } | tee "$log"

View File

@ -15,7 +15,7 @@ install_before() {
line="-$line"
else
# no rule exists, put it before the catch-all fallback
before='^# fallback'
before="^# fallback"
line="$line\n"
fi
sed -i -Ee "s|($before.*)|$line\n\1|" /etc/mdev.conf
@ -25,8 +25,8 @@ install_before() {
mod__vnic_eth_hotplug() {
[ -f /lib/mdev/vnic-eth-hotplug ] || return 1
install_before '^eth' \
'eth[0-9] root:root 0644 */lib/mdev/vnic-eth-hotplug'
install_before "^eth" \
"eth[0-9] root:root 0644 */lib/mdev/vnic-eth-hotplug"
# NICs attached at launch don't get added with mdev -s
assemble-interfaces

103
lib/tiny-cloud/nocloud/imds Normal file
View File

@ -0,0 +1,103 @@
# NoCloud Instance Metadata
# vim: ts=4 et ft=sh:
NOCLOUD_FILES="meta-data user-data vendor-data network-config"
is_nocloud_loaded() { [ -f "$TINY_CLOUD_VAR/.nocloud_loaded" ]; }
_load_nocloud_cmdline() {
local kopt kv k v data
for kopt in $(cat /proc/cmdline); do
echo "$kopt" | grep -qE '(^|=)ds=nocloud(-net)?;' || continue
for kv in $(echo "${kopt#*;}" | tr \; ' '); do
k=$(echo "$kv" | cut -d= -f1)
v=$(echo "$kv" | cut -d= -f2-)
case "$k" in
h|hostname)
echo -e "\nhostname: $v" >> "$TINY_CLOUD_VAR/meta-data"
;;
i|instance-id)
echo -e "\ninstance-id: $v" >> "$TINY_CLOUD_VAR/meta-data"
;;
s|seedfrom)
for data in $NOCLOUD_FILES; do
case "${v#file:/}" in
/*)
cat "$v/$data" >> "$TINY_CLOUD_VAR/$data" || continue
echo >> "$TINY_CLOUD_VAR/$data"
;;
http://*|https://*)
wget -qO - "$v/$data" >> "$TINY_CLOUD_VAR/$data" || continue
echo >> "$TINY_CLOUD_VAR/$data"
;;
*) log -s warning "Unknown NoCloud seedfrom value '$v'"
;;
esac
done
;;
*) log -s warning "Unknown NoCloud kernel cmdline key '$k'"
;;
esac
done
return
done
return 1
}
_load_nocloud_volume() {
local mntdir=$(mktemp -d /mnt/cidata-XXXXXX)
local data mounted
mkdir -p "$mntdir"
for fstype in vfat iso9660; do
[ "$mounted" ] && break
for label in cidata CIDATA; do
[ -n "$mounted" ] && break
mount -o ro -t "$fstype" LABEL="$label" "$mntdir" && mounted=1
done
done
if [ -n "$mounted" ]; then
for data in $NOCLOUD_FILES; do
# lack of source results in empty target
cat "$mntdir/$data" > "$TINY_CLOUD_VAR/$data"
done
umount "$mntdir"
else
return 1
fi
rmdir "$mntdir"
}
load_nocloud() {
# start with a clean slate
(cd "$TINY_CLOUD_VAR" && rm -f $NOCLOUD_FILES)
if _load_nocloud_cmdline || _load_nocloud_volume; then
touch "$TINY_CLOUD_VAR/.nocloud_loaded"
else
log -s err "Unable to load NoCloud datasource"
return 1
fi
# minimally, we expect some content in meta-data
[ -s "$TINY_CLOUD_VAR/meta-data" ] ||
log -s warning "NoCloud 'meta-data' is empty"
}
_imds() {
local file="$TINY_CLOUD_VAR"/$(echo "$1" | cut -d/ -f1)
local keypath=$(echo "$1" | cut -d/ -f2- | tr / ' ')
is_nocloud_loaded || load_nocloud
# does file exist?
[ -f "$file" ] || return 1
# use 'file/' to get top-level keys
if [ $(basename "$file") = "$keypath" ]; then
cat "$file"
else
yx -f "$file" $keypath
fi
}

View File

@ -7,16 +7,29 @@ IMDS_URI="opc/v2"
IMDS_HOSTNAME="instance/hostname"
IMDS_SSH_KEYS="instance/metadata/ssh_authorized_keys"
IMDS_USERDATA="instance/metadata/userdata"
IMDS_NICS="nics"
# TODO: flesh out networking
unset \
IMDS_MAC \
IMDS_IPV4 \
IMDS_IPV6 \
IMDS_IPV4_NET \
IMDS_IPV6_NET \
IMDS_IPV4_PREFIX \
IMDS_IPV6_PREFIX
_imds_header() {
echo "$IMDS_HEADER: Bearer Oracle"
}
_imds_ssh_keys() { _imds "$IMDS_SSH_KEYS"; }
_imds_nic_index() {
local m n=0
local mac=$(cat /sys/class/net/$1/mac)
local mac=$(cat "/sys/class/net/$1/mac")
while m=$(imds $IMDS_NICS/$n/mac | tr A-F a-f); do
[ "$m" = "$mac" ] && echo $n; return 0
done
return 1
}
}

View File

@ -1,15 +1,13 @@
# Tiny Cloud configuration
# /etc/conf.d/tiny-cloud
# REQUIRED: The instance's cloud provider (valid: aws, azure, gcp, oci)
# valid: aws, azure, gcp, oci
# REQUIRED: The instance's cloud provider
# valid: aws, azure, gcp, oci, nocloud
#CLOUD=
# User account where instance SSH keys are installed
#CLOUD_USER=alpine
# Filename of userdata file (in TINY_CLOUD_VAR directory)
#CLOUD_USERDATA=user-data
# IMDS token validity, in seconds (AWS only)
#IMDS_TOKEN_TTL=5
@ -28,5 +26,5 @@
# Explicitly skip these (whitespace delimited) things during init
# valid: expand_root install_hotplugs set_hostname set_ssh_keys
# save_userdata run_userdata
# save_userdata decompress_userdata run_userdata
#SKIP_INIT_ACTIONS=