Support datasource specific instance metadata

This change aligns the instance metadata with cloudinit to
add the support for datasource specific instance metadata.

The datasource specific instance metadata allows more
information to be exposed to the Jinja template for userdata
and script.

Note:
The structure is not standarized but you may refer to
https://cloudinit.readthedocs.io/en/latest/explanation/instancedata.html
to the cloudinit format.

Change-Id: I1ec7e5bdf063709c513b52a02c9251752aafe157

Author: laozc

cloudbaseinit/metadata/services/base.py

@@ -28,6 +28,9 @@
 
 CONF = cloudbaseinit_conf.CONF
 LOG = oslo_logging.getLogger(__name__)
+EXPERIMENTAL_NOTICE = ("EXPERIMENTAL: The structure and format of content "
+                       "scoped under the 'ds' key may change in subsequent "
+                       "releases of cloud-init.")
 
 
 class NotExistingMetadataException(Exception):
@@ -236,28 +239,52 @@ def get_instance_data(self):
         The ds namespace can change without prior notice and should not be
         used in production.
         """
-
         instance_id = self.get_instance_id()
         hostname = self.get_host_name()
 
         v1_data = {
+            "instance-id": instance_id,
             "instance_id": instance_id,
+            "local-hostname": hostname,
             "local_hostname": hostname,
-            "public_ssh_keys": self.get_public_keys()
         }
 
         # Copy the v1 data to the ds.meta_data and add more fields
-        ds_meta_data = copy.deepcopy(v1_data)
-        ds_meta_data.update({
-            "hostname": hostname
-        })
-
-        return {
+        ds_meta_data = self._get_datasource_instance_meta_data()
+        if not ds_meta_data:
+            ds_meta_data = copy.deepcopy(v1_data)
+            ds_meta_data.update({
+                "hostname": hostname
+            })
+
+        v1_data["public_ssh_keys"] = self.get_public_keys()
+        md = {
             "v1": v1_data,
             "ds": {
+                "_doc": EXPERIMENTAL_NOTICE,
                 "meta_data": ds_meta_data,
-            }
+            },
+            "instance-id": instance_id,
+            "instance_id": instance_id,
+            "local-hostname": hostname,
+            "local_hostname": hostname,
+            "public_ssh_keys": self.get_public_keys()
         }
+        return md
+
+    def _get_datasource_instance_meta_data(self):
+        """Returns a dictionary with datasource specific instance data
+
+        The instance data structure is based on the cloud-init specifications:
+        https://cloudinit.readthedocs.io/en/latest/explanation/instancedata.html
+
+        Datasource-specific metadata crawled for the specific cloud platform.
+        It should closely represent the structure of the cloud metadata
+        crawled. The structure of content and details provided are entirely
+        cloud-dependent.
+
+        """
+        pass
 
 
 class BaseHTTPMetadataService(BaseMetadataService):

cloudbaseinit/metadata/services/nocloudservice.py

@@ -30,6 +30,9 @@
 CONF = cloudbaseinit_conf.CONF
 LOG = oslo_logging.getLogger(__name__)
 
+DEFAULT_GATEWAY_CIDR_IPV4 = u"0.0.0.0/0"
+DEFAULT_GATEWAY_CIDR_IPV6 = u"::/0"
+
 
 class NoCloudNetworkConfigV1Parser(object):
     NETWORK_LINK_TYPE_PHY = 'physical'
@@ -280,9 +283,6 @@ def parse(self, network_config):
 
 
 class NoCloudNetworkConfigV2Parser(object):
-    DEFAULT_GATEWAY_CIDR_IPV4 = u"0.0.0.0/0"
-    DEFAULT_GATEWAY_CIDR_IPV6 = u"::/0"
-
     NETWORK_LINK_TYPE_ETHERNET = 'ethernet'
     NETWORK_LINK_TYPE_BOND = 'bond'
     NETWORK_LINK_TYPE_VLAN = 'vlan'
@@ -308,11 +308,11 @@ def _parse_addresses(self, item, link_name):
         default_route = None
         if gateway6 and netaddr.valid_ipv6(gateway6):
             default_route = network_model.Route(
-                network_cidr=self.DEFAULT_GATEWAY_CIDR_IPV6,
+                network_cidr=DEFAULT_GATEWAY_CIDR_IPV6,
                 gateway=gateway6)
         elif gateway4 and netaddr.valid_ipv4(gateway4):
             default_route = network_model.Route(
-                network_cidr=self.DEFAULT_GATEWAY_CIDR_IPV4,
+                network_cidr=DEFAULT_GATEWAY_CIDR_IPV4,
                 gateway=gateway4)
         if default_route:
             routes.append(default_route)
@@ -324,9 +324,9 @@ def _parse_addresses(self, item, link_name):
             gateway = route_config.get("via")
             if network_cidr.lower() == "default":
                 if netaddr.valid_ipv6(gateway):
-                    network_cidr = self.DEFAULT_GATEWAY_CIDR_IPV6
+                    network_cidr = DEFAULT_GATEWAY_CIDR_IPV6
                 else:
-                    network_cidr = self.DEFAULT_GATEWAY_CIDR_IPV4
+                    network_cidr = DEFAULT_GATEWAY_CIDR_IPV4
             route = network_model.Route(
                 network_cidr=network_cidr,
                 gateway=gateway)
@@ -547,6 +547,113 @@ def parse(network_data):
 
         return network_config_parser.parse(network_data)
 
+    @staticmethod
+    def network_details_v1_to_v2(v1_networks):
+        """Converts `NetworkDetails` objects to `NetworkDetailsV2` object.
+
+        """
+        if not v1_networks:
+            return None
+
+        links = []
+        networks = []
+        services = []
+        for nic in v1_networks:
+            link = network_model.Link(
+                id=nic.name,
+                name=nic.name,
+                type=network_model.LINK_TYPE_PHYSICAL,
+                mac_address=nic.mac,
+                enabled=None,
+                mtu=None,
+                bond=None,
+                vlan_link=None,
+                vlan_id=None,
+            )
+            links.append(link)
+
+            dns_addresses_v4 = []
+            dns_addresses_v6 = []
+            if nic.dnsnameservers:
+                for ns in nic.dnsnameservers:
+                    if netaddr.valid_ipv6(ns):
+                        dns_addresses_v6.append(ns)
+                    else:
+                        dns_addresses_v4.append(ns)
+
+            dns_services_v6 = None
+            if dns_addresses_v6:
+                dns_service_v6 = network_model.NameServerService(
+                    addresses=dns_addresses_v6,
+                    search=None,
+                )
+                dns_services_v6 = [dns_service_v6]
+                services.append(dns_service_v6)
+
+            dns_services_v4 = None
+            if dns_addresses_v4:
+                dns_service_v4 = network_model.NameServerService(
+                    addresses=dns_addresses_v4,
+                    search=None,
+                )
+                dns_services_v4 = [dns_service_v4]
+                services.append(dns_service_v4)
+
+            # Note: IPv6 address might be set to IPv4 field
+            # Not sure if it's a bug
+            default_route_v6 = None
+            default_route_v4 = None
+            if nic.gateway6:
+                default_route_v6 = network_model.Route(
+                    network_cidr=DEFAULT_GATEWAY_CIDR_IPV6,
+                    gateway=nic.gateway6)
+
+            if nic.gateway:
+                if netaddr.valid_ipv6(nic.gateway):
+                    default_route_v6 = network_model.Route(
+                        network_cidr=DEFAULT_GATEWAY_CIDR_IPV6,
+                        gateway=nic.gateway)
+                else:
+                    default_route_v4 = network_model.Route(
+                        network_cidr=DEFAULT_GATEWAY_CIDR_IPV4,
+                        gateway=nic.gateway)
+
+            routes_v6 = [default_route_v6] if default_route_v6 else []
+            routes_v4 = [default_route_v4] if default_route_v4 else []
+
+            if nic.address6:
+                net = network_model.Network(
+                    link=link.name,
+                    address_cidr=network_utils.ip_netmask_to_cidr(
+                        nic.address6, nic.netmask6),
+                    routes=routes_v6,
+                    dns_nameservers=dns_services_v6,
+                )
+                networks.append(net)
+
+            if nic.address:
+                if netaddr.valid_ipv6(nic.address):
+                    net = network_model.Network(
+                        link=link.name,
+                        address_cidr=network_utils.ip_netmask_to_cidr(
+                            nic.address, nic.netmask),
+                        routes=routes_v6,
+                        dns_nameservers=dns_services_v6,
+                    )
+                else:
+                    net = network_model.Network(
+                        link=link.name,
+                        address_cidr=network_utils.ip_netmask_to_cidr(
+                            nic.address, nic.netmask),
+                        routes=routes_v4,
+                        dns_nameservers=dns_services_v4,
+                    )
+                networks.append(net)
+
+        return network_model.NetworkDetailsV2(links=links,
+                                              networks=networks,
+                                              services=services)
+
 
 class NoCloudConfigDriveService(baseconfigdrive.BaseConfigDriveService):
 

cloudbaseinit/metadata/services/vmwareguestinfoservice.py

@@ -27,6 +27,7 @@
 from cloudbaseinit.metadata.services import base
 from cloudbaseinit.metadata.services import nocloudservice
 from cloudbaseinit.osutils import factory as osutils_factory
+from cloudbaseinit.utils import network
 from cloudbaseinit.utils import serialization
 
 CONF = cloudbaseinit_conf.CONF
@@ -239,3 +240,22 @@ def _process_network_config(self, data):
 
         LOG.debug("network data %s", network)
         return {"network": network}
+
+    def _get_datasource_instance_meta_data(self):
+        """Returns a dictionary with datasource specific instance data
+
+        The instance data structure is based on the cloud-init specifications:
+        https://cloudinit.readthedocs.io/en/latest/explanation/instancedata.html
+
+        Datasource-specific metadata crawled for the specific cloud platform.
+        It should closely represent the structure of the cloud metadata
+        crawled. The structure of content and details provided are entirely
+        cloud-dependent.
+
+        """
+        ds = dict()
+        network_details = self.get_network_details_v2()
+        host_info = network.get_host_info(self.get_host_name(),
+                                          network_details)
+        ds.update(host_info)
+        return ds

cloudbaseinit/tests/metadata/services/test_base.py

@@ -61,38 +61,63 @@ def test_get_user_pwd_encryption_key(self, mock_get_public_keys):
         result = self._service.get_user_pwd_encryption_key()
         self.assertEqual(result, mock_get_public_keys.return_value[0])
 
+    @mock.patch('cloudbaseinit.metadata.services.base.'
+                'BaseMetadataService._get_datasource_instance_meta_data')
     @mock.patch('cloudbaseinit.metadata.services.base.'
                 'BaseMetadataService.get_public_keys')
     @mock.patch('cloudbaseinit.metadata.services.base.'
                 'BaseMetadataService.get_host_name')
     @mock.patch('cloudbaseinit.metadata.services.base.'
                 'BaseMetadataService.get_instance_id')
-    def test_get_instance_data(self, mock_instance_id, mock_hostname,
-                               mock_public_keys):
+    def _test_get_instance_data_with_datasource_meta_data(
+            self, mock_instance_id, mock_hostname, mock_public_keys,
+            mock_get_datasource_instance_meta_data, datasource_meta_data=None):
         fake_instance_id = 'id'
         mock_instance_id.return_value = fake_instance_id
         fake_hostname = 'host'
         mock_hostname.return_value = fake_hostname
         fake_keys = ['ssh1', 'ssh2']
         mock_public_keys.return_value = fake_keys
+        mock_get_datasource_instance_meta_data.return_value = \
+            datasource_meta_data
 
+        if datasource_meta_data:
+            ds_md = datasource_meta_data
+        else:
+            ds_md = {
+                "instance_id": fake_instance_id,
+                "instance-id": fake_instance_id,
+                "local_hostname": fake_hostname,
+                "local-hostname": fake_hostname,
+                "hostname": fake_hostname
+            }
         expected_response = {
             'v1': {
                 "instance_id": fake_instance_id,
+                "instance-id": fake_instance_id,
                 "local_hostname": fake_hostname,
+                "local-hostname": fake_hostname,
                 "public_ssh_keys": fake_keys
             },
             'ds': {
-                'meta_data': {
-                    "instance_id": fake_instance_id,
-                    "local_hostname": fake_hostname,
-                    "public_ssh_keys": fake_keys,
-                    "hostname": fake_hostname
-                },
-            }
+                '_doc': base.EXPERIMENTAL_NOTICE,
+                'meta_data': ds_md,
+            },
+            "instance_id": fake_instance_id,
+            "instance-id": fake_instance_id,
+            "local_hostname": fake_hostname,
+            "local-hostname": fake_hostname,
+            "public_ssh_keys": fake_keys,
         }
         self.assertEqual(expected_response, self._service.get_instance_data())
 
+    def test_get_instance_data(self):
+        self._test_get_instance_data_with_datasource_meta_data()
+
+    def test_get_instance_data_with_datasource_meta_data(self):
+        self._test_get_instance_data_with_datasource_meta_data(
+            datasource_meta_data={'fake-data': 'fake-value'})
+
 
 class TestBaseHTTPMetadataService(unittest.TestCase):