nuageinit: add basic support for cloudinit.

this is a very early script to support cloudinit, it does not intend to
be a full featured cloudinit client, but will support a good enough
subset to be viable in most case.

It support nocloud and openstack config-2 config drive mode (iso9660 or
msdosfs)

The following features are currently supported:
- adding users (including a default user named 'freebsd' with password
  'freebsd'
- adding groups
- adding ssh keys
- static ipv4, static ipv6, dynamic ipv4

With this one is able to use the 'bring your own image feature" out of
box.

It is expected that the script grows the support of other clouds
supporting cloud-init, contributions are welcomed.

It is designed to be only run once via the firstboot mecanism.

Sponsored by:	OVHCloud
Differential Revision:	https://reviews.freebsd.org/D44141

(cherry picked from commit a42d6f76018e4ed8324e319ab48aac904bda437c)
(cherry picked from commit c051f22bce42d920abba61bd7cf4ef5b6a270ffa)
(cherry picked from commit b8c053c9a612651d4909f7a323088f3e92485b7b)
(cherry picked from commit 9eae9233fdcc946945f4191e1413f548adfa2943)

11 files changed, 476 insertions, 0 deletions

diff --git a/libexec/nuageinit/tests/Makefile b/libexec/nuageinit/tests/Makefile
new file mode 100644
index 000000000000..d5b3bd9dcc82
--- /dev/null
+++ b/libexec/nuageinit/tests/Makefile
@@ -0,0 +1,13 @@
+PACKAGE=	tests
+
+ATF_TESTS_SH=	nuage utils nuageinit
+
+${PACKAGE}FILES+=	warn.lua
+${PACKAGE}FILES+=	err.lua
+${PACKAGE}FILES+=	dirname.lua
+${PACKAGE}FILES+=	sethostname.lua
+${PACKAGE}FILES+=	addsshkey.lua
+${PACKAGE}FILES+=	adduser.lua
+${PACKAGE}FILES+=	addgroup.lua
+
+.include <bsd.test.mk>
diff --git a/libexec/nuageinit/tests/addgroup.lua b/libexec/nuageinit/tests/addgroup.lua
new file mode 100644
index 000000000000..60a0d8346793
--- /dev/null
+++ b/libexec/nuageinit/tests/addgroup.lua
@@ -0,0 +1,15 @@
+#!/usr/libexec/flua
+
+local n = require("nuage")
+if n.addgroup() then
+	n.err("addgroup should not accept empty value")
+end
+if n.addgroup("plop") then
+	n.err("addgroup should not accept empty value")
+end
+local gr = {}
+gr.name = "impossible_groupname"
+local res = n.addgroup(gr)
+if not res then
+	n.err("valid addgroup should return a path")
+end
diff --git a/libexec/nuageinit/tests/addsshkey.lua b/libexec/nuageinit/tests/addsshkey.lua
new file mode 100644
index 000000000000..3aa5f7619ec2
--- /dev/null
+++ b/libexec/nuageinit/tests/addsshkey.lua
@@ -0,0 +1,2 @@
+local n = require("nuage")
+n.addsshkey(".", "mykey")
diff --git a/libexec/nuageinit/tests/adduser.lua b/libexec/nuageinit/tests/adduser.lua
new file mode 100644
index 000000000000..9366d2abd0f4
--- /dev/null
+++ b/libexec/nuageinit/tests/adduser.lua
@@ -0,0 +1,15 @@
+#!/usr/libexec/flua
+
+local n = require("nuage")
+if n.adduser() then
+	n.err("adduser should not accept empty value")
+end
+if n.adduser("plop") then
+	n.err("adduser should not accept empty value")
+end
+local pw = {}
+pw.name = "impossible_username"
+local res = n.adduser(pw)
+if not res then
+	n.err("valid adduser should return a path")
+end
diff --git a/libexec/nuageinit/tests/dirname.lua b/libexec/nuageinit/tests/dirname.lua
new file mode 100644
index 000000000000..d1268e48575c
--- /dev/null
+++ b/libexec/nuageinit/tests/dirname.lua
@@ -0,0 +1,8 @@
+local n = require("nuage")
+print(n.dirname("/my/path/path1"))
+if n.dirname("path") then
+	nuage.err("Expecting nil for n.dirname(\"path\")")
+end
+if n.dirname() then
+	nuage.err("Expecting nil for n.dirname")
+end
diff --git a/libexec/nuageinit/tests/err.lua b/libexec/nuageinit/tests/err.lua
new file mode 100644
index 000000000000..c62fa1098f09
--- /dev/null
+++ b/libexec/nuageinit/tests/err.lua
@@ -0,0 +1,4 @@
+#!/usr/libexec/flua
+
+local n = require("nuage")
+n.err("plop")
diff --git a/libexec/nuageinit/tests/nuage.sh b/libexec/nuageinit/tests/nuage.sh
new file mode 100644
index 000000000000..bbf306eae51f
--- /dev/null
+++ b/libexec/nuageinit/tests/nuage.sh
@@ -0,0 +1,52 @@
+atf_test_case sethostname
+atf_test_case addsshkey
+atf_test_case adduser
+atf_test_case addgroup
+
+sethostname_body() {
+	export NUAGE_FAKE_ROOTDIR="$(pwd)"
+	atf_check /usr/libexec/flua $(atf_get_srcdir)/sethostname.lua
+	if [ ! -f etc/rc.conf.d/hostname ]; then
+		atf_fail "hostname not written"
+	fi
+	atf_check -o inline:"hostname=\"myhostname\"\n" cat etc/rc.conf.d/hostname
+}
+
+addsshkey_body() {
+	atf_check /usr/libexec/flua $(atf_get_srcdir)/addsshkey.lua
+	if [ ! -f .ssh/authorized_keys ]; then
+		atf_fail "ssh key not added"
+	fi
+	atf_check -o inline:"mykey\n" cat .ssh/authorized_keys
+	atf_check /usr/libexec/flua $(atf_get_srcdir)/addsshkey.lua
+	atf_check -o inline:"mykey\nmykey\n" cat .ssh/authorized_keys
+}
+
+adduser_body() {
+	export NUAGE_FAKE_ROOTDIR="$(pwd)"
+	if [ $(id -u) -ne 0 ]; then
+		atf_skip "root required"
+	fi
+	mkdir etc
+	printf "root:*:0:0::0:0:Charlie &:/root:/bin/csh\n" > etc/master.passwd
+	pwd_mkdb -d etc etc/master.passwd
+	printf "wheel:*:0:root\n" > etc/group
+	atf_check -e inline:"Argument should be a table\nArgument should be a table\n" /usr/libexec/flua $(atf_get_srcdir)/adduser.lua
+	test -d home/impossible_username || atf_fail "home not created"
+	atf_check -o inline:"impossible_username::1001:1001::0:0:impossible_username User:/home/impossible_username:/bin/sh\n" grep impossible_username etc/master.passwd
+}
+
+addgroup_body() {
+	export NUAGE_FAKE_ROOTDIR="$(pwd)"
+	mkdir etc
+	printf "wheel:*:0:root\n" > etc/group
+	atf_check -e inline:"Argument should be a table\nArgument should be a table\n" /usr/libexec/flua $(atf_get_srcdir)/addgroup.lua
+	atf_check -o inline:"impossible_groupname:*:1001:\n" grep impossible_groupname etc/group
+}
+
+atf_init_test_cases() {
+	atf_add_test_case sethostname
+	atf_add_test_case addsshkey
+	atf_add_test_case adduser
+	atf_add_test_case addgroup
+}
diff --git a/libexec/nuageinit/tests/nuageinit.sh b/libexec/nuageinit/tests/nuageinit.sh
new file mode 100644
index 000000000000..926233bcf66d
--- /dev/null
+++ b/libexec/nuageinit/tests/nuageinit.sh
@@ -0,0 +1,338 @@
+atf_test_case args
+atf_test_case nocloud
+atf_test_case nocloud_userdata_script
+atf_test_case nocloud_userdata_cloudconfig
+atf_test_case nocloud_userdata_cloudconfig_users
+atf_test_case nocloud_network
+atf_test_case config2
+atf_test_case config2_pubkeys
+atf_test_case config2_network
+atf_test_case config2_network_static_v4
+
+
+args_body()
+{
+	atf_check -s exit:1 -e inline:"Usage /usr/libexec/nuageinit <cloud-init directory> [config-2|nocloud]\n" /usr/libexec/nuageinit
+	atf_check -s exit:1 -e inline:"Usage /usr/libexec/nuageinit <cloud-init directory> [config-2|nocloud]\n" /usr/libexec/nuageinit bla
+	atf_check -s exit:1 -e inline:"Usage /usr/libexec/nuageinit <cloud-init directory> [config-2|nocloud]\n" /usr/libexec/nuageinit bla meh plop
+	atf_check -s exit:1 -e inline:"Unknown cloud init type: meh\n" /usr/libexec/nuageinit bla meh
+}
+
+nocloud_body()
+{
+	here=$(pwd)
+	mkdir -p media/nuageinit
+	atf_check -s exit:1 -e match:"nuageinit: error parsing nocloud.*" /usr/libexec/nuageinit ${here}/media/nuageinit/ nocloud
+	export NUAGE_FAKE_ROOTDIR=$(pwd)
+	printf "instance-id: iid-local01\nlocal-hostname: cloudimg\n" > ${here}/media/nuageinit/meta-data
+	atf_check -s exit:0 /usr/libexec/nuageinit ${here}/media/nuageinit nocloud
+	atf_check -o inline:"hostname=\"cloudimg\"\n" cat etc/rc.conf.d/hostname
+	cat > media/nuageinit/meta-data << EOF
+instance-id: iid-local01
+hostname: myhost
+EOF
+	atf_check -s exit:0 /usr/libexec/nuageinit ${here}/media/nuageinit nocloud
+	atf_check -o inline:"hostname=\"myhost\"\n" cat etc/rc.conf.d/hostname
+}
+
+nocloud_userdata_script_body()
+{
+	here=$(pwd)
+	mkdir -p media/nuageinit
+	printf "instance-id: iid-local01\n" > ${here}/media/nuageinit/meta-data
+	printf "#!/bin/sh\necho "yeah"\n" > ${here}/media/nuageinit/user-data
+	chmod 755  ${here}/media/nuageinit/user-data
+	atf_check -s exit:0 -o inline:"yeah\n" /usr/libexec/nuageinit ${here}/media/nuageinit nocloud
+}
+
+nocloud_userdata_cloudconfig_users_body()
+{
+	here=$(pwd)
+	export NUAGE_FAKE_ROOTDIR=$(pwd)
+	if [ $(id -u) -ne 0 ]; then
+		atf_skip "root required"
+	fi
+	mkdir -p media/nuageinit
+	printf "instance-id: iid-local01\n" > ${here}/media/nuageinit/meta-data
+	mkdir -p etc
+	cat > etc/master.passwd <<EOF
+root:*:0:0::0:0:Charlie &:/root:/bin/csh
+sys:*:1:0::0:0:Sys:/home/sys:/bin/csh
+EOF
+	pwd_mkdb -d etc ${here}/etc/master.passwd
+	cat > etc/group <<EOF
+wheel:*:0:root
+users:*:1:
+EOF
+	cat > media/nuageinit/user-data <<EOF
+#cloud-config
+groups:
+  - admingroup: [root,sys]
+  - cloud-users
+users:
+  - default
+  - name: foobar
+    gecos: Foo B. Bar
+    primary_group: foobar
+    groups: users
+    passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/
+EOF
+	atf_check /usr/libexec/nuageinit ${here}/media/nuageinit nocloud
+	cat > expectedgroup << EOF
+wheel:*:0:root,freebsd
+users:*:1:foobar
+admingroup:*:1001:root,sys
+cloud-users:*:1002:
+freebsd:*:1003:
+foobar:*:1004:
+EOF
+	cat > expectedpasswd << EOF
+root:*:0:0::0:0:Charlie &:/root:/bin/csh
+sys:*:1:0::0:0:Sys:/home/sys:/bin/csh
+freebsd:freebsd:1001:1003::0:0:FreeBSD User:/home/freebsd:/bin/sh
+foobar:H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/:1002:1004::0:0:Foo B. Bar:/home/foobar:/bin/sh
+EOF
+	atf_check -o file:expectedpasswd cat ${here}/etc/master.passwd
+	atf_check -o file:expectedgroup cat ${here}/etc/group
+}
+
+nocloud_network_body()
+{
+	here=$(pwd)
+	mkdir -p media/nuageinit
+	mkdir -p etc
+	cat > etc/master.passwd <<EOF
+root:*:0:0::0:0:Charlie &:/root:/bin/csh
+sys:*:1:0::0:0:Sys:/home/sys:/bin/csh
+EOF
+	pwd_mkdb -d etc ${here}/etc/master.passwd
+	cat > etc/group <<EOF
+wheel:*:0:root
+users:*:1:
+EOF
+	if [ $(id -u) -ne 0 ]; then
+		atf_skip "root required"
+	fi
+	mynetworks=$(ifconfig -l ether)
+	if [ -z "$mynetworks" ]; then
+		atf_skip "a network interface is needed"
+	fi
+	set -- $mynetworks
+	myiface=$1
+	myaddr=$(ifconfig $myiface ether | awk '/ether/ { print $2 }')
+	printf "instance-id: iid-local01\n" > ${here}/media/nuageinit/meta-data
+	cat > media/nuageinit/user-data <<EOF
+#cloud-config
+#
+network:
+  version: 2
+  ethernets:
+    # opaque ID for physical interfaces, only referred to by other stanzas
+    id0:
+      match:
+        macaddress: '${myaddr}'
+      addresses:
+        - 192.168.14.2/24
+        - 2001:1::1/64
+      gateway4: 192.168.14.1
+      gateway6: 2001:1::2
+EOF
+	export NUAGE_FAKE_ROOTDIR=$(pwd)
+	atf_check /usr/libexec/nuageinit ${here}/media/nuageinit nocloud
+	cat > network <<EOF
+ifconfig_${myiface}="inet 192.168.14.2/24"
+ifconfig_${myiface}_ipv6="inet6 2001:1::1/64"
+ipv6_network_interfaces="${myiface}"
+ipv6_default_interface="${myiface}"
+EOF
+	cat > routing <<EOF
+defaultrouter="192.168.14.1"
+ipv6_defaultrouter="2001:1::2"
+ipv6_route_${myiface}="2001:1::2 -prefixlen 128 -interface ${myiface}"
+EOF
+	atf_check -o file:network cat ${here}/etc/rc.conf.d/network
+	atf_check -o file:routing cat ${here}/etc/rc.conf.d/routing
+}
+config2_body()
+{
+	here=$(pwd)
+	mkdir -p media/nuageinit
+	atf_check -s exit:1 -e match:"nuageinit: error parsing config-2: meta_data.json.*" /usr/libexec/nuageinit ${here}/media/nuageinit config-2
+	printf "{}" > media/nuageinit/meta_data.json
+	atf_check /usr/libexec/nuageinit ${here}/media/nuageinit config-2
+	cat > media/nuageinit/meta_data.json << EOF
+{
+   "hostname": "cloudimg",
+}
+EOF
+	export NUAGE_FAKE_ROOTDIR=$(pwd)
+	atf_check /usr/libexec/nuageinit ${here}/media/nuageinit config-2
+	atf_check -o inline:"hostname=\"cloudimg\"\n" cat etc/rc.conf.d/hostname
+}
+
+config2_pubkeys_body()
+{
+	here=$(pwd)
+	if [ $(id -u) -ne 0 ]; then
+		atf_skip "root required"
+	fi
+	mkdir -p media/nuageinit
+	cat > media/nuageinit/meta_data.json << EOF
+{
+   "public_keys": {
+       "mykey": "ssh-rsa AAAAB3NzaC1y...== Generated by Nova"
+   },
+}
+EOF
+	mkdir -p etc
+	cat > etc/master.passwd <<EOF
+root:*:0:0::0:0:Charlie &:/root:/bin/csh
+sys:*:1:0::0:0:Sys:/home/sys:/bin/csh
+EOF
+	pwd_mkdb -d etc ${here}/etc/master.passwd
+	cat > etc/group <<EOF
+wheel:*:0:root
+users:*:1:
+EOF
+	export NUAGE_FAKE_ROOTDIR=$(pwd)
+	atf_check /usr/libexec/nuageinit ${here}/media/nuageinit config-2
+	atf_check -o inline:"ssh-rsa AAAAB3NzaC1y...== Generated by Nova\n" cat home/freebsd/.ssh/authorized_keys
+}
+
+config2_network_body() {
+	here=$(pwd)
+	mkdir -p media/nuageinit
+	printf "{}" > media/nuageinit/meta_data.json
+	mynetworks=$(ifconfig -l ether)
+	if [ -z "$mynetworks" ]; then
+		atf_skip "a network interface is needed"
+	fi
+	set -- $mynetworks
+	myiface=$1
+	myaddr=$(ifconfig $myiface ether | awk '/ether/ { print $2 }')
+cat > media/nuageinit/network_data.json <<EOF
+{
+    "links": [
+        {
+            "ethernet_mac_address": "$myaddr",
+            "id": "iface0",
+            "mtu": null,
+        }
+    ],
+    "networks": [
+        {
+            "id": "network0",
+            "link": "iface0",
+            "type": "ipv4_dhcp"
+        },
+	{ // IPv6
+	"id": "private-ipv4",
+	"type": "ipv6",
+	"link": "iface0",
+	// supports condensed IPv6 with CIDR netmask
+	"ip_address": "2001:cdba::3257:9652/24",
+	"gateway": "fd00::1"
+	"routes": [
+	    {
+		"network": "::",
+		"netmask": "::",
+		"gateway": "fd00::1"
+	},
+	    {
+		"network": "::",
+		"netmask": "ffff:ffff:ffff::",
+		"gateway": "fd00::1:1"
+	},
+	],
+	"network_id": "da5bb487-5193-4a65-a3df-4a0055a8c0d8"
+},
+    ],
+}
+EOF
+	export NUAGE_FAKE_ROOTDIR=$(pwd)
+	atf_check /usr/libexec/nuageinit ${here}/media/nuageinit config-2
+	cat > network <<EOF
+ifconfig_${myiface}="DHCP"
+ifconfig_${myiface}_ipv6="inet6 2001:cdba::3257:9652/24"
+ipv6_network_interfaces="${myiface}"
+ipv6_default_interface="${myiface}"
+EOF
+	cat > routing <<EOF
+ipv6_defaultrouter="fd00::1"
+ipv6_route_${myiface}="fd00::1 -prefixlen 128 -interface ${myiface}"
+ipv6_static_routes="${myiface}"
+EOF
+	atf_check -o file:network cat ${here}/etc/rc.conf.d/network
+	atf_check -o file:routing cat ${here}/etc/rc.conf.d/routing
+}
+
+config2_network_static_v4_body() {
+	here=$(pwd)
+	mkdir -p media/nuageinit
+	printf "{}" > media/nuageinit/meta_data.json
+	mynetworks=$(ifconfig -l ether)
+	if [ -z "$mynetworks" ]; then
+		atf_skip "a network interface is needed"
+	fi
+	set -- $mynetworks
+	myiface=$1
+	myaddr=$(ifconfig $myiface ether | awk '/ether/ { print $2 }')
+cat > media/nuageinit/network_data.json <<EOF
+{
+    "links": [
+        {
+            "ethernet_mac_address": "$myaddr",
+            "id": "iface0",
+            "mtu": null,
+        }
+    ],
+    "networks": [
+        {
+            "id": "network0",
+            "link": "iface0",
+            "type": "ipv4"
+            "ip_address": "10.184.0.244",
+            "netmask": "255.255.240.0",
+            "routes": [
+            {
+                "network": "10.0.0.0",
+                "netmask": "255.0.0.0",
+                "gateway": "11.0.0.1"
+            },
+            {
+                "network": "0.0.0.0",
+                "netmask": "0.0.0.0",
+                "gateway": "23.253.157.1"
+            }
+         ]
+     }
+]
+}
+EOF
+
+	export NUAGE_FAKE_ROOTDIR=$(pwd)
+	atf_check /usr/libexec/nuageinit ${here}/media/nuageinit config-2
+	cat > network <<EOF
+ifconfig_${myiface}="inet 10.184.0.244 netmask 255.255.240.0"
+EOF
+	cat > routing <<EOF
+route_cloudinit1_${myiface}="-net 10.0.0.0 11.0.0.1 255.0.0.0"
+defaultrouter="23.253.157.1"
+static_routes="cloudinit1_${myiface}"
+EOF
+	atf_check -o file:network cat ${here}/etc/rc.conf.d/network
+	atf_check -o file:routing cat ${here}/etc/rc.conf.d/routing
+}
+
+atf_init_test_cases()
+{
+	atf_add_test_case args
+	atf_add_test_case nocloud
+	atf_add_test_case nocloud_userdata_script
+	atf_add_test_case nocloud_userdata_cloudconfig_users
+	atf_add_test_case nocloud_network
+	atf_add_test_case config2
+	atf_add_test_case config2_pubkeys
+	atf_add_test_case config2_network
+	atf_add_test_case config2_network_static_v4
+}
diff --git a/libexec/nuageinit/tests/sethostname.lua b/libexec/nuageinit/tests/sethostname.lua
new file mode 100644
index 000000000000..01434403934b
--- /dev/null
+++ b/libexec/nuageinit/tests/sethostname.lua
@@ -0,0 +1,4 @@
+#!/usr/libexec/flua
+
+local n = require("nuage")
+n.sethostname("myhostname")
diff --git a/libexec/nuageinit/tests/utils.sh b/libexec/nuageinit/tests/utils.sh
new file mode 100644
index 000000000000..192ccfb565cb
--- /dev/null
+++ b/libexec/nuageinit/tests/utils.sh
@@ -0,0 +1,21 @@
+atf_test_case warn
+atf_test_case err
+atf_test_case dirname
+
+warn_body() {
+	atf_check -e "inline:plop\n" -s exit:0 /usr/libexec/flua $(atf_get_srcdir)/warn.lua
+}
+
+err_body() {
+	atf_check -e "inline:plop\n" -s exit:1 /usr/libexec/flua $(atf_get_srcdir)/err.lua
+}
+
+dirname_body() {
+	atf_check -o "inline:/my/path/\n" -s exit:0 /usr/libexec/flua $(atf_get_srcdir)/dirname.lua
+}
+
+atf_init_test_cases() {
+	atf_add_test_case warn
+	atf_add_test_case err
+	atf_add_test_case dirname
+}
diff --git a/libexec/nuageinit/tests/warn.lua b/libexec/nuageinit/tests/warn.lua
new file mode 100644
index 000000000000..8575ae92c071
--- /dev/null
+++ b/libexec/nuageinit/tests/warn.lua
@@ -0,0 +1,4 @@
+#!/usr/libexec/flua
+
+local n = require("nuage")
+n.warn("plop")