diff --git a/Makefile b/Makefile index 35ddd58..69ac9da 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PREFIX?=/ -SUBPACKAGES = core network openrc aws azure gcp oci +SUBPACKAGES = core network openrc aws azure gcp oci nocloud .PHONY: install $(SUBPACKAGES) @@ -59,5 +59,11 @@ oci: conf_dir sed -Ee 's/^#?CLOUD=.*/CLOUD=oci/' \ etc/conf.d/tiny-cloud.example > "$(PREFIX)"/etc/conf.d/tiny-cloud +nocloud: conf_dir + install -Dm644 -t $(PREFIX)/lib/tiny-cloud/nocloud \ + lib/tiny-cloud/nocloud/* + sed -Ee 's/^#?CLOUD=.*/CLOUD=nocloud/' \ + etc/conf.d/tiny-cloud.example > "$(PREFIX)"/etc/conf.d/tiny-cloud + conf_dir: - mkdir -p "$(PREFIX)"/etc/conf.d \ No newline at end of file + mkdir -p "$(PREFIX)"/etc/conf.d diff --git a/README.md b/README.md index 5b8dc74..c533892 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,13 @@ do just what is necessary with a small footprint and minimal dependencies. A direct descendant of [tiny-ec2-bootstrap]( https://gitlab.alpinelinux.org/alpine/cloud/tiny-ec2-bootstrap), Tiny Cloud works with multiple cloud providers. Currently, the following are supported: -* AWS (Amazon Web Services) -* Azure (Microsoft Azure) -* GCP (Google Cloud Platform) -* OCI (Oracle Cloud Infrastructure) +* [AWS](https://aws.amazon.com) (Amazon Web Services) +* [Azure](https://azure.microsoft.com) (Microsoft Azure) +* [GCP](https://cloud.google.com) (Google Cloud Platform) +* [OCI](https://cloud.oracle.com) (Oracle Cloud Infrastructure) +* [NoCloud]( + https://cloudinit.readthedocs.io/en/latest/topics/datasources/nocloud.html) + (cloud-init's NoCloud AWS-compatible data source) ## Features @@ -44,9 +47,12 @@ As Tiny Cloud is meant to be tiny, it has very few dependencies: * `partx` * `resize2fs` * `sfdisk` +* [`yx`](https://gitlab.com/tomalok/yx) + (optional, allows NoCloud to extract metadata from YAML files) Tiny Cloud has been developed specifically for use with the -[Alpine Cloud Images](https://gitlab.alpinelinux.org/alpine/cloud/alpine-cloud-images) +[Alpine Cloud Images]( + https://gitlab.alpinelinux.org/alpine/cloud/alpine-cloud-images) project, and as such, it is currently tailored for use with [Alpine Linux]( https://alpinelinux.org), the [OpenRC](https://github.com/OpenRC/openrc) init system, and the `ext4` root filesystem. If you would like to see Tiny Cloud @@ -57,12 +63,12 @@ open an issue with your request -- or better yet, submit a merge request! Typically, Tiny Cloud is installed and configured when building a cloud image, and is available on Alpine Linux as the [`tiny-cloud`]( -https://pkgs.alpinelinux.org/packages?name=tiny-cloud) APK... + https://pkgs.alpinelinux.org/packages?name=tiny-cloud*) APKs... ``` -apk install tiny-cloud +apk install tiny-cloud- ``` This will install the necessary init scripts, libraries, etc. plus any missing -dependencies. +dependencies for Tiny Cloud to support _``_. Alternately, you can download a release tarball, and use `make` to install it. @@ -79,9 +85,9 @@ By default, Tiny Cloud expects configuration at `/etc/conf.d/tiny-cloud`, The stock [`etc/conf.d/tiny-cloud`](etc/conf.d/tiny-cloud) file contains details of all tuneable settings. -*Because Tiny Cloud does not currently do auto-detection, you **MUST** set a +_Because Tiny Cloud does not currently do auto-detection, you **MUST** set a configuration value for `CLOUD` indicating which cloud provider will be used. -Current valid values are `aws`, `azure`, `gcp`, and `oci`.* +Current valid values are `aws`, `azure`, `gcp`, `oci`, and `nocloud`._ ## Operation @@ -106,13 +112,18 @@ data, and sets up instance's hostname and the cloud user's SSH keys before `tiny-cloud-final` should be the very last init script to run in the **default** runlevel. By default, it saves the instance's user data to -`/var/lib/cloud/user-data`, which is overrideable via the `TINY_CLOUD_VAR` -andr `CLOUD_USERDATA` config settings. +`/var/lib/cloud/user-data`; the directory overrideable via the `TINY_CLOUD_VAR` +config setting. -If the user data is a script starting with `#!/`, it will be executed; its +If the user data is compressed, Tiny Cloud will decompress it. Currently +supported compression algorithms are `gzip`, `bzip2`, `unxz`, `lzma`, `lzop`, +`lz4`, and `zstd`. _(Note that `lz4` and `zstd` are not installed in Alpine +by default, and would need to be added to the image.)_ + +If the user data is a script starting with `#!`, it will be executed; its output (combined STDOUT and STDERR) and exit code are saved to -`/var/log/user-data.log` and `/var/log/user-data.exit`, respectively -- unless -overriden with `TINY_CLOUD_LOGS` and `CLOUD_USERDATA` config settings. +`/var/log/user-data.log` and `/var/log/user-data.exit`, respectively; the +directory is overrideable via the `TINY_CLOUD_LOGS` config setting. If all went well, the very last thing `tiny-cloud-final` does is touch a `.bootstrap-complete` file into existence in `/var/lib/cloud` or another diff --git a/bin/imds b/bin/imds index 2a1de30..146d806 100755 --- a/bin/imds +++ b/bin/imds @@ -12,18 +12,7 @@ unset \ IMDS_HEADER \ IMDS_URI \ - IMDS_QUERY \ - IMDS_HOSTNAME \ - IMDS_SSH_KEYS \ - IMDS_USERDATA \ - IMDS_NICS \ - IMDS_MAC \ - IMDS_IPV4 \ - IMDS_IPV6 \ - IMDS_IPV4_NET \ - IMDS_IPV6_NET \ - IMDS_IPV4_PREFIX \ - IMDS_IPV6_PREFIX + IMDS_QUERY unset -f \ _imds_token \ _imds_header \ @@ -35,15 +24,36 @@ unset -f \ CLOUD="${CLOUD:-unknown}" IMDS_ENDPOINT="169.254.169.254" +# Common to AWS and NoCloud(ish) +IMDS_HOSTNAME="meta-data/hostname" +IMDS_SSH_KEYS="meta-data/public-keys" +IMDS_USERDATA="user-data" +IMDS_NICS="meta-data/network/interfaces/macs" +IMDS_MAC="mac" +IMDS_IPV4="local-ipv4s" +IMDS_IPV6="ipv6s" +IMDS_IPV4_NET="subnet-ipv4-cidr-block" +IMDS_IPV6_NET="subnet-ipv6-cidr-blocks" +IMDS_IPV4_PREFIX="ipv4-prefix" +IMDS_IPV6_PREFIX="ipv6-prefix" + _imds() { wget --quiet --timeout 1 --output-document - \ --header "$(_imds_header)" \ "http://$IMDS_ENDPOINT/$IMDS_URI/$1$IMDS_QUERY" } -_imds_ssh_keys() { _imds "$IMDS_SSH_KEYS"; } _imds_userdata() { _imds "$IMDS_USERDATA"; } +_imds_ssh_keys() { + local key + for key in $(imds "$IMDS_SSH_KEYS"); do + imds "$IMDS_SSH_KEYS/${key%=*}/openssh-key" + done | sort -u +} + +_imds_nic_index() { cat "/sys/class/net/$1/address"; } + ### load cloud-specific variables and functions if [ ! -d /lib/tiny-cloud/"$CLOUD" ]; then diff --git a/etc/conf.d/tiny-cloud.example b/etc/conf.d/tiny-cloud.example index 904e11b..5cc8b59 100644 --- a/etc/conf.d/tiny-cloud.example +++ b/etc/conf.d/tiny-cloud.example @@ -1,15 +1,12 @@ # Tiny Cloud configuration -# REQUIRED: The instance's cloud provider (valid: aws, azure, gcp, oci) -# valid: aws, azure, gcp, oci +# REQUIRED: The instance's cloud provider +# valid: aws, azure, gcp, oci, nocloud #CLOUD= # User account where instance SSH keys are installed #CLOUD_USER=alpine -# Filename of userdata file (in TINY_CLOUD_VAR directory) -#CLOUD_USERDATA=user-data - # IMDS token validity, in seconds (AWS only) #IMDS_TOKEN_TTL=5 @@ -28,5 +25,5 @@ # Explicitly skip these (whitespace delimited) things during init # valid: expand_root install_hotplugs set_hostname set_ssh_keys -# save_userdata run_userdata +# save_userdata decompress_userdata run_userdata #SKIP_INIT_ACTIONS= diff --git a/lib/tiny-cloud/aws/imds b/lib/tiny-cloud/aws/imds index 2501544..3ade15f 100644 --- a/lib/tiny-cloud/aws/imds +++ b/lib/tiny-cloud/aws/imds @@ -6,19 +6,6 @@ IMDS_TOKEN_TTL_HEADER="X-aws-ec2-metadata-token-ttl-seconds" IMDS_TOKEN_TTL=${IMDS_TOKEN_TTL:-5} IMDS_URI="latest" -IMDS_HOSTNAME="meta-data/hostname" -IMDS_SSH_KEYS="meta-data/public-keys" -IMDS_USERDATA="user-data" - -IMDS_NICS="meta-data/network/interfaces/macs" -IMDS_MAC="mac" -IMDS_IPV4="local-ipv4s" -IMDS_IPV6="ipv6s" -IMDS_IPV4_NET="subnet-ipv4-cidr-block" -IMDS_IPV6_NET="subnet-ipv6-cidr-blocks" -IMDS_IPV4_PREFIX="ipv4-prefix" -IMDS_IPV6_PREFIX="ipv6-prefix" - _imds_token() { echo -ne "PUT /latest/api/token" \ "HTTP/1.0\r\n$IMDS_TOKEN_TTL_HEADER: $IMDS_TOKEN_TTL\r\n\r\n" | @@ -27,14 +14,4 @@ _imds_token() { _imds_header() { echo "$IMDS_HEADER: $(_imds_token)" -} - -# digs deeper than the default -_imds_ssh_keys() { - local key - for key in $(imds "$IMDS_SSH_KEYS"); do - imds "$IMDS_SSH_KEYS/${key%=*}/openssh-key" - done | sort -u -} - -_imds_nic_index() { cat "/sys/class/net/$1/address"; } +} \ No newline at end of file diff --git a/lib/tiny-cloud/azure/imds b/lib/tiny-cloud/azure/imds index b04dc79..c638ca2 100644 --- a/lib/tiny-cloud/azure/imds +++ b/lib/tiny-cloud/azure/imds @@ -8,8 +8,17 @@ IMDS_URI="metadata/instance" IMDS_HOSTNAME="compute/name" IMDS_SSH_KEYS="compute/publicKeys" IMDS_USERDATA="compute/userData" +IMDS_NICS="network/interface" -IMDS_NICS="" +# TODO: flesh out networking +unset \ + IMDS_MAC \ + IMDS_IPV4 \ + IMDS_IPV6 \ + IMDS_IPV4_NET \ + IMDS_IPV6_NET \ + IMDS_IPV4_PREFIX \ + IMDS_IPV6_PREFIX _imds_header() { echo "$IMDS_HEADER: true" diff --git a/lib/tiny-cloud/common b/lib/tiny-cloud/common index 2590796..206b39a 100644 --- a/lib/tiny-cloud/common +++ b/lib/tiny-cloud/common @@ -2,7 +2,7 @@ # vim: ts=4 et ft=sh: log() { - local facility=kern + local facility="kern" local stderr local tag=$(basename "$0") while [ "${1:0:1}" = '-' ]; do diff --git a/lib/tiny-cloud/init-common b/lib/tiny-cloud/init-common index 6d23b84..2632c78 100644 --- a/lib/tiny-cloud/init-common +++ b/lib/tiny-cloud/init-common @@ -5,7 +5,6 @@ # set defaults CLOUD_USER=${CLOUD_USER:-alpine} -CLOUD_USERDATA=${CLOUD_USERDATA:-user-data} TINY_CLOUD_LOGS=${TINY_CLOUD_LOGS:-/var/log} TINY_CLOUD_VAR=${TINY_CLOUD_VAR:-/var/lib/cloud} SKIP_INIT_ACTIONS=${SKIP_INIT_ACTIONS:-} diff --git a/lib/tiny-cloud/init-final b/lib/tiny-cloud/init-final index 696d3eb..97f4716 100644 --- a/lib/tiny-cloud/init-final +++ b/lib/tiny-cloud/init-final @@ -9,7 +9,6 @@ match_header() { [ "$bytes" = $(dd bs=1 count=${#bytes} if="$2" 2>/dev/null) ] } -# TODO: also do this for nocloud meta-data and vendor-data? save_userdata() { skip_action save_userdata && return @@ -18,29 +17,30 @@ save_userdata() { local cmd imds -e @userdata > "$tmpfile" - if match_header '\037\213\010' "$tmpfile"; then - cmd='gzip -dc' - elif match_header 'BZh' "$tmpfile"; then - cmd='bzip2 -dc' - elif match_header '\3757zXZ\000' "$tmpfile"; then - cmd='xz -dc' - elif match_header '\135\0\0\0' "$tmpfile"; then - cmd='lzma -dc' - elif match_header '\211\114\132' "$tmpfile"; then - cmd='lzop -dc' - elif match_header '\002!L\030' "$tmpfile"; then - cmd='lz4 -dc' - elif match_header '(\265/\375' "$tmpfile"; then - cmd='zstd -dc' - else - cmd='cat' + cmd="cat" + if ! skip_action decompress_userdata; then + if match_header '\037\213\010' "$tmpfile"; then + cmd="gzip -dc" + elif match_header 'BZh' "$tmpfile"; then + cmd="bzip2 -dc" + elif match_header '\3757zXZ\000' "$tmpfile"; then + cmd="unxz -c" + elif match_header '\135\0\0\0' "$tmpfile"; then + cmd="lzma -dc" + elif match_header '\211\114\132' "$tmpfile"; then + cmd="lzop -dc" + elif match_header '\002!L\030' "$tmpfile"; then + cmd="lz4 -dc" + elif match_header '(\265/\375' "$tmpfile"; then + cmd="zstd -dc" + fi fi $cmd "$tmpfile" > "$userdata" rm "$tmpfile" } is_userdata_script() { - head -n1 "$TINY_CLOUD_VAR/$CLOUD_USERDATA" | grep -q '#!/' + head -n1 "$TINY_CLOUD_VAR/$CLOUD_USERDATA" | grep -q "#!/" } run_userdata() { diff --git a/lib/tiny-cloud/mdev b/lib/tiny-cloud/mdev index 7e5ea25..345a945 100644 --- a/lib/tiny-cloud/mdev +++ b/lib/tiny-cloud/mdev @@ -15,7 +15,7 @@ install_before() { line="-$line" else # no rule exists, put it before the catch-all fallback - before='^# fallback' + before="^# fallback" line="$line\n" fi sed -i -Ee "s|($before.*)|$line\n\1|" /etc/mdev.conf @@ -25,8 +25,8 @@ install_before() { mod__vnic_eth_hotplug() { [ -f /lib/mdev/vnic-eth-hotplug ] || return 1 - install_before '^eth' \ - 'eth[0-9] root:root 0644 */lib/mdev/vnic-eth-hotplug' + install_before "^eth" \ + "eth[0-9] root:root 0644 */lib/mdev/vnic-eth-hotplug" # NICs attached at launch don't get added with mdev -s assemble-interfaces diff --git a/lib/tiny-cloud/nocloud/imds b/lib/tiny-cloud/nocloud/imds index f45e7d2..97c3e30 100644 --- a/lib/tiny-cloud/nocloud/imds +++ b/lib/tiny-cloud/nocloud/imds @@ -1,46 +1,76 @@ # NoCloud Instance Metadata # vim: ts=4 et ft=sh: -IMDS_HOSTNAME=meta-data/hostname -IMDS_SSH_KEYS=meta-data/ssh-keys -IMDS_USERDATA=user-data +NOCLOUD_FILES="meta-data user-data vendor-data network-config" -# have we loaded the nocloud meta/user/vendor data? is_nocloud_loaded() { [ -f "$TINY_CLOUD_VAR/.nocloud_loaded" ]; } -# from location specified in kernel cmdline _load_nocloud_cmdline() { - # TODO + local kopt kv k v data + + for kopt in $(cat /proc/cmdline); do + echo "$kopt" | grep -qE '(^|=)ds=nocloud(-net)?(;|$)' || continue + for kv in $(echo "$kopt" | cut -d\; -f2-); do + k=$(echo "$kv" | cut -d= -f1) + v=$(echo "$kv" | cut -d= -f2- | sed -Ee s'|^file://|/|') + case "$k" in + h|hostname) + echo -e "\nhostname: $hostname" >> "$TINY_CLOUD_VAR/meta-data" + ;; + i|instance-id) + echo -e "\ninstance-id: $hostname" >> "$TINY_CLOUD_VAR/meta-data" + ;; + s|seedfrom) + for data in $NOCLOUD_FILES; do + case "$v" in + /*) + cat "$v" >> "$TINY_CLOUD_VAR/$data" || continue + echo >> "$TINY_CLOUD_VAR/$data" + ;; + http://*|https://*) + wget -qO - "$v" >> "$TINY_CLOUD_VAR/$data" || continue + echo >> "$TINY_CLOUD_VAR/$data" + ;; + *) echo "WARNING: Ignoring unknown seedfrom value '$v'" >&2 + ;; + esac + ;; + done + ;; + *) echo "WARNING: Ignoring unknown nocloud kernel cmdline key '$k'" >&2 + ;; + esac + done + return + done } -# from volume labeled cidata|CIDATA _load_nocloud_volume() { - local mntdir='/mnt/cidata' + local mntdir=$(mktmemp /mnt/cidata-XXXXXX) + local data - mkdir "$mntdir" - mount -L cidata "$mntdir" || mount -L CIDATA "$mntdir" || return 1 - for data in meta user vendor; do - if ! cp "$mntdir/$data-data" "$TINY_CLOUD_VAR/$data-data"; then - [ "$data" = vendor ] && continue - echo "ERROR: Required $data-data not found on CIDATA volume" - umount "$mntdir" - return 1 - fi + mkdir -p "$mntdir" + # TODO: are lables case insensitive? + mount LABEL=cidata "$mntdir" || mount LABEL=CIDATA "$mntdir" || return 1 + for data in $NOCLOUD_FILES; do + cp "$mntdir/$data" "$TINY_CLOUD_VAR/$data" 2>/dev/null done umount "$mntdir" + rmdir "$mntdir" } load_nocloud() { is_nocloud_loaded && return - if grep -qE ' ds=nocloud(-net)?[; ]' /proc/cmdline; then - if ! _load_nocloud_cmdline; then - echo 'ERROR: Unable to load nocloud data specified in kernel cmdline' >&2 - return 1 - fi - elif ! _load_nocloud_volume; then - echo 'ERROR: Unable to load nocloud data from CIDATA volume' >&2 + + # start with a clean slate + rm -f $NOCLOUD_FILES + + if ! _load_nocloud_cmdline || _load_nocloud_volume; then + echo "ERROR: Unable to load NoCloud data" >&2 return 1 fi + # at the very minimum, we expect something in meta-data + touch "$TINY_CLOUD_VAR/.nocloud_loaded" } @@ -56,4 +86,4 @@ _imds() { # use 'file/' to get top-level keys [ "$1" = "$file" ] && cat "$file" || yx -f "$file" "$key" -} +} \ No newline at end of file diff --git a/lib/tiny-cloud/oci/imds b/lib/tiny-cloud/oci/imds index 9c78e16..caface5 100644 --- a/lib/tiny-cloud/oci/imds +++ b/lib/tiny-cloud/oci/imds @@ -7,11 +7,24 @@ IMDS_URI="opc/v2" IMDS_HOSTNAME="instance/hostname" IMDS_SSH_KEYS="instance/metadata/ssh_authorized_keys" IMDS_USERDATA="instance/metadata/userdata" +IMDS_NICS="nics" + +# TODO: flesh out networking +unset \ + IMDS_MAC \ + IMDS_IPV4 \ + IMDS_IPV6 \ + IMDS_IPV4_NET \ + IMDS_IPV6_NET \ + IMDS_IPV4_PREFIX \ + IMDS_IPV6_PREFIX _imds_header() { echo "$IMDS_HEADER: Bearer Oracle" } +_imds_ssh_keys() { _imds "$IMDS_SSH_KEYS"; } + _imds_nic_index() { local m n=0 local mac=$(cat /sys/class/net/$1/mac) @@ -19,4 +32,4 @@ _imds_nic_index() { [ "$m" = "$mac" ] && echo $n; return 0 done return 1 -} +} \ No newline at end of file