# -*- encoding: utf-8 -*-
# Copyright (c) 2016 b<>com
#
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import base64
import functools
import json
import os_traits
import random
import textwrap
import time

from datetime import datetime
from datetime import timedelta
from datetime import timezone

from oslo_log import log
from tempest.common import waiters
from tempest import config
from tempest.lib.common import api_microversion_fixture
from tempest.lib.common import api_version_utils
from tempest.lib.common.utils import data_utils
from tempest.lib.common.utils import test_utils
from tempest.lib import exceptions
from tempest.scenario import manager

from watcher_tempest_plugin import infra_optim_clients as clients
from watcher_tempest_plugin.services.infra_optim.v1.json import (
    api_microversion_fixture as watcher_microversion_fixture
)
from watcher_tempest_plugin.tests.common import base


LOG = log.getLogger(__name__)
CONF = config.CONF

# Minimal Nova API required to specify the hosts used when creating
# instances is 2.74.This constant will be used to set the minimal
# version of nova to be used in tests which use create_one_instance_per_host.
# Note that it will not affect the version of the Nova API used by
# Watcher, only the version used by tempest when used from those tests.
NOVA_API_VERSION_CREATE_WITH_HOST = '2.74'
# Nova API version that adds 'pinned_availability_zone' to GET /servers/details
# response.
NOVA_API_VERSION_SERVER_PINNED_AZ = '2.96'


class BaseInfraOptimScenarioTest(manager.ScenarioTest,
                                 base.WatcherHelperMixin):
    """Base class for Infrastructure Optimization API tests."""

    # Default Goal and Strategy to be defined in each test class that inherit
    # from this base class.
    GOAL = None
    STRATEGY = None

    min_microversion = None
    max_microversion = manager.LATEST_MICROVERSION

    # Holds the initial configuration of available compute nodes, to assist
    # the rollback procedure.
    initial_compute_nodes_setup = []

    # Metric map used to add or retrieve metrics on Prometheus
    # NOTE(dviroel): This maps metrics consumed from Prometheus server,
    #  but the datasource may support other metrics from the model.
    PROMETHEUS_METRIC_MAP = dict(
        host_cpu_usage='node_cpu_seconds_total',
        host_ram_usage='node_memory_MemAvailable_bytes',
        host_ram_total='node_memory_MemTotal_bytes',
        instance_cpu_usage='ceilometer_cpu',
        instance_ram_usage='ceilometer_memory_usage')

    # Metric map used to add or retrieve metrics on Gnocchi
    GNOCCHI_METRIC_MAP = dict(
        host_cpu_usage='compute.node.cpu.percent',
        host_ram_usage='hardware.memory.used',
        host_outlet_temp='hardware.ipmi.node.outlet_temperature',
        host_inlet_temp='hardware.ipmi.node.temperature',
        host_airflow='hardware.ipmi.node.airflow',
        host_power='hardware.ipmi.node.power',
        instance_cpu_usage='cpu',
        instance_ram_usage='memory.resident',
        instance_ram_allocated='memory',
        instance_l3_cache_usage='cpu_l3_cache',
        instance_root_disk_size='disk.root.size',)

    @classmethod
    def skip_checks(cls):
        super(BaseInfraOptimScenarioTest, cls).skip_checks()
        if not CONF.service_available.watcher:
            raise cls.skipException('Watcher support is required')

        api_version_utils.check_skip_with_microversion(
            cls.min_microversion,
            cls.max_microversion,
            CONF.optimize.min_microversion,
            CONF.optimize.max_microversion)

    @classmethod
    def setup_credentials(cls):
        cls._check_network_config()
        super(BaseInfraOptimScenarioTest, cls).setup_credentials()
        cls.mgr = clients.AdminManager()

    @classmethod
    def setup_clients(cls):
        super(BaseInfraOptimScenarioTest, cls).setup_clients()
        cls.client = cls.mgr.io_client
        cls.gnocchi = cls.mgr.gn_client
        cls.placement_client = cls.mgr.placement_client
        cls.prometheus_client = cls.mgr.prometheus_client
        cls.flavors_client = cls.mgr.flavors_client

    def setUp(self):
        super(BaseInfraOptimScenarioTest, self).setUp()
        self.useFixture(api_microversion_fixture.APIMicroversionFixture(
            compute_microversion=self.compute_request_microversion))
        self.useFixture(api_microversion_fixture.APIMicroversionFixture(
            placement_microversion=CONF.placement.min_microversion))
        self.useFixture(watcher_microversion_fixture.APIMicroversionFixture(
            optimize_microversion=self.request_microversion))

    @classmethod
    def resource_setup(cls):
        super(BaseInfraOptimScenarioTest, cls).resource_setup()

        cls.request_microversion = (
            api_version_utils.select_request_microversion(
                cls.min_microversion,
                CONF.optimize.min_microversion))

    @classmethod
    def resource_cleanup(cls):
        """Ensure that all created objects get destroyed."""
        super(BaseInfraOptimScenarioTest, cls).resource_cleanup()

    @classmethod
    def get_hypervisors_setup(cls):
        hypervisors_client = cls.mgr.hypervisor_client
        hypervisors = hypervisors_client.list_hypervisors(
            detail=True)['hypervisors']
        return hypervisors

    @classmethod
    def get_hypervisor_details(cls, node_name):
        """Get hypervisor details by node name."""
        hypervisors = cls.get_hypervisors_setup()
        for hyp in hypervisors:
            if hyp['hypervisor_hostname'] == node_name:
                return hyp
        raise exceptions.InvalidConfiguration(
            "Hypervisor %s not found in the list of hypervisors." %
            node_name)

    @classmethod
    def get_compute_nodes_setup(cls):
        services_client = cls.mgr.services_client
        available_services = services_client.list_services()['services']

        return [srv for srv in available_services
                if srv.get('binary') == 'nova-compute']

    @classmethod
    def get_enabled_compute_nodes(cls):
        cls.initial_compute_nodes_setup = cls.get_compute_nodes_setup()
        return [cn for cn in cls.initial_compute_nodes_setup
                if cn.get('status') == 'enabled']

    @classmethod
    def get_host_other_than(cls, server_id):
        source_host = cls.get_host_for_server(server_id)

        svcs = cls.os_admin.services_client.list_services(
            binary='nova-compute')['services']
        hosts = []
        for svc in svcs:
            if CONF.compute.target_hosts_to_avoid in svc['host']:
                continue
            if svc['state'] == 'up' and svc['status'] == 'enabled':
                if CONF.compute.compute_volume_common_az:
                    if svc['zone'] == CONF.compute.compute_volume_common_az:
                        hosts.append(svc['host'])
                else:
                    hosts.append(svc['host'])

        for target_host in hosts:
            if source_host != target_host:
                return target_host

    @classmethod
    def wait_for_compute_node_setup(cls):

        def _are_compute_nodes_setup():
            try:
                hypervisors = cls.get_hypervisors_setup()
                available_hypervisors = set(
                    hyp['hypervisor_hostname'] for hyp in hypervisors
                    if hyp['state'] == 'up')
                available_services = set(
                    service['host']
                    for service in cls.get_compute_nodes_setup()
                    if service['state'] == 'up')
                return (
                    len(available_hypervisors) == len(available_services)
                    and len(hypervisors) >= 2)
            except Exception as exc:
                LOG.exception(exc)
                return False

        assert test_utils.call_until_true(
            func=_are_compute_nodes_setup,
            duration=600,
            sleep_for=2
        )

    @classmethod
    def rollback_compute_nodes_status(cls):
        current_compute_nodes_setup = cls.get_compute_nodes_setup()
        for cn_setup in current_compute_nodes_setup:
            cn_hostname = cn_setup.get('host')
            matching_cns = [
                cns for cns in cls.initial_compute_nodes_setup
                if cns.get('host') == cn_hostname
            ]
            initial_cn_setup = matching_cns[0]  # Should return a single result
            if cn_setup.get('status') != initial_cn_setup.get('status'):
                svr_id = cn_setup.get('id')
                status = initial_cn_setup.get('status')
                # The Nova version Watcher neede is at least 2.56
                # Starting with microversion 2.53 disable/enable API
                # is superseded by PUT /os-services/{service_id}
                rollback_func = cls.mgr.services_client.update_service
                rollback_func(svr_id, status=status)

    @classmethod
    def wait_for(cls, condition, timeout=30):
        start_time = time.time()
        while time.time() - start_time < timeout:
            if condition():
                break
            time.sleep(.5)

    @classmethod
    def _check_network_config(cls):
        if not CONF.network.public_network_id:
            msg = 'public network not defined.'
            LOG.error(msg)
            raise exceptions.InvalidConfiguration(msg)

    @classmethod
    def _are_all_action_plans_finished(cls):
        _, action_plans = cls.client.list_action_plans()
        return all([ap['state'] in cls.ACTIONPLAN_FINISHED_STATES.values()
                    for ap in action_plans['action_plans']])

    def check_min_enabled_compute_nodes(self, min_nodes):
        enabled_compute_nodes = self.get_enabled_compute_nodes()
        msg = ("This test required at least %s enabled compute nodes, but "
               "only %s were found." % (min_nodes, len(enabled_compute_nodes)))
        self.assertGreaterEqual(len(enabled_compute_nodes), min_nodes, msg=msg)

    def wait_for_all_action_plans_to_finish(self):
        assert test_utils.call_until_true(
            func=self._are_all_action_plans_finished,
            duration=300,
            sleep_for=5
        )

    def _migrate_server_to(self, server_id, dest_host):
        # The default value of  block_migration is auto and
        # disk_over_commit is not valid after version 2.25
        block_migration = 'auto'
        body = self.mgr.servers_client.live_migrate_server(
            server_id, host=dest_host, block_migration=block_migration)
        return body

    def _live_migrate(self, server_id, target_host, state):
        self._migrate_server_to(server_id, target_host)
        waiters.wait_for_server_status(self.os_admin.servers_client,
                                       server_id, state)
        migration_list = (self.mgr.migrations_client.list_migrations()
                          ['migrations'])

        msg = ("Live Migration failed. Migrations list for Instance "
               "%s: [" % server_id)
        for live_migration in migration_list:
            if (live_migration['instance_uuid'] == server_id):
                msg += "\n%s" % live_migration
        msg += "]"
        server_host = self.mgr.servers_client.show_server(
            server_id)['server']['OS-EXT-SRV-ATTR:host']

        self.assertEqual(target_host, server_host, msg)

    def _create_custom_flavor(self, ram=512, vcpus=1):
        """Create a flavor with custom RAM size

        :param ram: RAM in MB to be set for the flavor.
        :returns: A flavor id
        """

        flavor_id = self.flavors_client.create_flavor(
            name=data_utils.rand_name('watcher_flavor'),
            ram=ram,
            vcpus=vcpus,
            disk=1,
            ephemeral=0,
            swap=0,
            rxtx_factor=1.0,
            is_public=True)['flavor']['id']
        self.addCleanup(test_utils.call_and_ignore_notfound_exc,
                        self.flavors_client.delete_flavor, flavor_id)
        return flavor_id

    def _create_one_instance_per_host(self, flavor=None, run_command=None):
        """Create one instance per compute node

        This goes up to the min_compute_nodes threshold so that things don't
        get crazy if you have 1000 compute nodes but set min to 3.

        :param flavor: Flavor Name or ID
        :param run_command: the command you want to run in the new instances
        :returns: A list of instance UUIDs.
        """

        compute_nodes = self.get_compute_nodes_setup()
        instances = self.mgr.servers_client.list_servers(
            detail=True)['servers']
        if instances:
            return instances

        hypervisors = self.get_hypervisors_setup()
        created_instances = []

        for node in compute_nodes[:CONF.compute.min_compute_nodes]:
            hyp_id = [
                hyp['id'] for hyp in hypervisors
                if hyp['hypervisor_hostname'] == node['host']]
            # Placement may fail to update trait because of Conflict
            # the trait may be updated by the Nova compute
            # update_available_resource periodic task.
            # We need node status is enabled, so we check the node
            # trait and delay if it is not the correct status.
            # the max delay time is 10 minutes.
            node_trait = os_traits.COMPUTE_STATUS_DISABLED
            retry = 20
            trait_status = True
            while trait_status and retry:
                trait_status = self.check_node_trait(hyp_id[0], node_trait)
                if not trait_status:
                    break
                time.sleep(30)
                retry -= 1
            self.assertNotEqual(0, retry)
            # by getting to active state here, this means this has
            # landed on the host in question.
            instance = self._create_instance(node['host'], flavor, run_command)
            created_instances.append(instance)
        return created_instances

    def _create_instance(self, host=None, flavor=None, run_command=None):
        # We enforce the compute node where we create the instance to
        # make sure we have one node on each compute.
        # This requires Nova API version 2.74 or higher.
        kwargs_server = {'host': host} if host else {}
        validatable = False
        validation_resources = None
        if run_command:
            # In case we want to run commands we will be injecting it via
            # user_data which requires to setup the instance as validatable
            # in tempest.common.compute.create_test_server
            validation_resources = self.get_test_validation_resources(
                self.os_admin)
            validatable = True
            script = '''
                    #!/bin/sh
                    {run_command}
                    '''.format(run_command=run_command)
            script_clean = textwrap.dedent(script).lstrip().encode('utf8')
            script_b64 = base64.b64encode(script_clean)
            kwargs_server['user_data'] = script_b64
        flavor = flavor if flavor else CONF.compute.flavor_ref
        instance = self.create_server(
            image_id=CONF.compute.image_ref, wait_until='ACTIVE',
            flavor=flavor, clients=self.os_admin, validatable=validatable,
            validation_resources=validation_resources,
            **kwargs_server)
        # get instance object again as admin
        instance = self.mgr.servers_client.show_server(
            instance['id'])['server']
        return instance

    def _pack_all_created_instances_on_one_host(self, instances):
        hypervisors = [
            hyp['hypervisor_hostname'] for hyp in self.get_hypervisors_setup()
            if hyp['state'] == 'up']
        node = hypervisors[0]
        for instance in instances:
            if self.get_host_for_server(instance['id']) != node:
                self._live_migrate(instance['id'], node, 'ACTIVE')
        return node

    # ### GNOCCHI ### #

    def create_resource(self, **kwargs):
        """Wrapper utility for creating a test resource

        :return: A tuple with The HTTP response and its body
        """
        try:
            resp, body = self.gnocchi.create_resource(**kwargs)
        except exceptions.Conflict:
            # if resource already exists we just request it
            search_body = {"=": {"original_resource_id": kwargs['id']}}
            resp, body = self.gnocchi.search_resource(**search_body)
            body = body[0]
            for metric_name in kwargs['metrics'].keys():
                default_metric = {
                    "archive_policy_name": "low",
                    "resource_id": body['id'],
                    "name": metric_name,
                    "unit": "ns"
                }
                metric_body = {**default_metric,
                               **kwargs['metrics'][metric_name]}
                if body['metrics'].get(metric_name, None):
                    self.gnocchi.delete_metric(body['metrics'][metric_name])
                self.gnocchi.create_metric(**metric_body)
            resp, body = self.gnocchi.search_resource(**search_body)
            body = body[0]
        return resp, body

    def _make_measures_host(self, measures_count, time_step, min=10, max=20):
        measures_body = []
        now = datetime.now(timezone.utc)
        for i in range(0, measures_count):
            dt = now - timedelta(minutes=i * time_step)
            measures_body.append(
                dict(value=random.randint(int(min), int(max)),
                     timestamp=dt.replace(microsecond=0).isoformat())
            )
        return measures_body

    def _make_measures_instance(self, measures_count, time_step,
                                min=80, max=90, metric_type='cpu'):
        measures_body = []
        now = datetime.now(timezone.utc)

        if metric_type == "cpu":
            final_cpu = (measures_count + 1) * 60 * time_step * 1e9
            for i in range(0, measures_count):
                dt = now - timedelta(minutes=i * time_step)
                cpu = final_cpu - ((i - 1) * 60 * time_step * 1e9
                                   * random.randint(int(min), int(max)) / 100)
                measures_body.append(
                    dict(value=cpu,
                         timestamp=dt.replace(microsecond=0).isoformat())
                )
        elif metric_type == "ram":
            for i in range(0, measures_count):
                dt = now - timedelta(minutes=i * time_step)
                ram = random.randint(int(min), int(max))
                measures_body.append(
                    dict(value=ram,
                         timestamp=dt.replace(microsecond=0).isoformat())
                )
        else:
            raise ValueError(f"Unsupported metric_type: {metric_type}")

        return measures_body

    def clean_injected_metrics(self):
        """Delete all injected metrics from datastore.

        This is useful to ensure that the tests are not affected by
        previously injected metrics.
        """
        LOG.debug("Deleting injected metrics from Datastore")
        if CONF.optimize.datasource == "gnocchi":
            # TODO(morenod): Add function for deleting injected metrics
            # from Gnocchi.
            pass
        elif CONF.optimize.datasource == "prometheus":
            self.prometheus_client.delete_series()

    def make_host_statistic(self, metrics=dict(), loaded_hosts=[]):
        """Add host metrics to the datasource

        :param metrics: Metrics that should be created
          in the configured datasource.
        :param loaded_hosts: list of hosts that we want to inject data
          representing high usage of resource.
        """
        # data sources that support fake metrics
        if CONF.optimize.datasource == "gnocchi":
            self.make_host_statistic_gnocchi(metrics,
                                             loaded_hosts=loaded_hosts)
        elif CONF.optimize.datasource == "prometheus":
            self.make_host_statistic_prometheus(loaded_hosts=loaded_hosts)

    def make_host_statistic_gnocchi(self, metrics=dict(), loaded_hosts=[]):
        """Create host resource and its measures

        :param metrics: Metrics that should be created
          in the Gnocchi datasource.
        :param loaded_hosts: list of hosts that we want to inject data
          representing high usage of resource.
        """
        hypervisors_client = self.mgr.hypervisor_client
        hypervisors = hypervisors_client.list_hypervisors(
            detail=True)['hypervisors']
        if metrics == dict():
            metrics = {
                self.GNOCCHI_METRIC_MAP['host_cpu_usage']: {
                    'archive_policy_name': 'low'
                },
                self.GNOCCHI_METRIC_MAP['host_ram_usage']: {
                    'archive_policy_name': 'low'
                }
            }
        else:
            metrics = {
                self.GNOCCHI_METRIC_MAP[m]: {
                    'archive_policy_name': 'low'
                } for m in metrics.keys()
            }
        for h in hypervisors:
            host_name = "%s_%s" % (h['hypervisor_hostname'],
                                   h['hypervisor_hostname'])
            resource_params = {
                'type': 'host',
                'metrics': metrics,
                'host_name': host_name,
                'id': host_name
            }
            _, res = self.create_resource(**resource_params)

            # Generate host_cpu_usage fake metrics
            cpu_metric_uuid = res['metrics'][
                self.GNOCCHI_METRIC_MAP['host_cpu_usage']
            ]
            if h['hypervisor_hostname'] in loaded_hosts:
                cpu_measures = self._make_measures_host(10, 1, min=80, max=90)
            else:
                cpu_measures = self._make_measures_host(10, 1)
            self.gnocchi.add_measures(cpu_metric_uuid, cpu_measures)

            # Generate host_ram_usage fake metrics. Metric is based on
            # hardware.memory.used which is in KB.
            ram_metric_uuid = res['metrics'][
                self.GNOCCHI_METRIC_MAP['host_ram_usage']
            ]
            if h['hypervisor_hostname'] in loaded_hosts:
                mem_measures = self._make_measures_host(
                    10, 1,
                    min=int(h['memory_mb']) * 0.8 * 1024,
                    max=int(h['memory_mb']) * 0.9 * 1024)
            else:
                mem_measures = self._make_measures_host(
                    10, 1,
                    min=int(h['memory_mb']) * 0.1 * 1024,
                    max=int(h['memory_mb']) * 0.2 * 1024)
            self.gnocchi.add_measures(ram_metric_uuid, mem_measures)

    def _show_measures(self, metric_uuid):
        try:
            _, res = self.gnocchi.show_measures(metric_uuid)
        except Exception:
            return False
        if len(res) > 0:
            return True

    def make_instance_statistic(self, instance, metrics=dict()):
        """Add instance resources and its measures to the datasource

        :param instance: Instance response body
        :param metrics: Metrics that should be created
          in the configured datasource.
        """
        # data sources that support fake metrics
        if CONF.optimize.datasource == "gnocchi":
            self.make_instance_statistic_gnocchi(instance, metrics)
        elif CONF.optimize.datasource == "prometheus":
            self.make_instance_statistic_prometheus(instance)

    def make_instance_statistic_gnocchi(self, instance, metrics=dict()):
        """Create instance resource and its measures in Gnocchi DB

        :param instance: Instance response body
        :param metrics: The metrics add to resource when using Gnocchi
        """
        all_flavors = self.flavors_client.list_flavors(detail=True)['flavors']
        flavor_name = instance['flavor']['original_name']
        flavor = [f for f in all_flavors if f['name'] == flavor_name]
        if metrics == dict():
            metrics = {
                self.GNOCCHI_METRIC_MAP["instance_cpu_usage"]: {
                    'archive_policy_name': 'ceilometer-low-rate',
                    'unit': 'ns'
                },
                self.GNOCCHI_METRIC_MAP["instance_ram_usage"]: {
                    'archive_policy_name': 'ceilometer-low',
                    'unit': 'MB'
                }
            }
        else:
            metrics = {
                self.GNOCCHI_METRIC_MAP[m]: {
                    'archive_policy_name': 'low'
                } for m in metrics.keys()
            }
        resource_params = {
            'type': 'instance',
            'metrics': metrics,
            'host': self.get_host_for_server(instance['id']),
            'display_name': instance.get('OS-EXT-SRV-ATTR:instance_name'),
            'image_ref': instance['image']['id'],
            'flavor_id': flavor[0]['id'],
            'flavor_name': flavor_name,
            'id': instance['id']
        }
        _, res = self.create_resource(**resource_params)

        # Generates cpu fake metrics
        cpu_metric_uuid = res['metrics'][
            self.GNOCCHI_METRIC_MAP["instance_cpu_usage"]
        ]
        cpu_measures = self._make_measures_instance(5, 5, metric_type='cpu')
        self.gnocchi.add_measures(cpu_metric_uuid, cpu_measures)

        # Generate ram fake metrics

        ram_metric_uuid = res['metrics'][
            self.GNOCCHI_METRIC_MAP["instance_ram_usage"]
        ]
        ram_measures = self._make_measures_instance(
            5, 5,
            min=int(flavor[0]['ram']) * 0.8,
            max=int(flavor[0]['ram']) * 0.9,
            metric_type='ram')
        self.gnocchi.add_measures(ram_metric_uuid, ram_measures)

        for metric_uuid in [cpu_metric_uuid, ram_metric_uuid]:
            self.assertTrue(test_utils.call_until_true(
                func=functools.partial(
                    self._show_measures, metric_uuid),
                duration=600,
                sleep_for=2
            ))

    # ### PROMETHEUS ### #
    def _generate_prometheus_metrics(self,
                                     metric_name,
                                     metric_type="counter",
                                     labels={},
                                     count=10,
                                     interval_secs=30,
                                     add_unique_label=True,
                                     inc_factor=0.8,
                                     start_value=1.0,
                                     timestamp=None):
        """Generates multiple samples for a given metric.


        Samples are generate for a time interval defined by
        the provided 'interval_secs' and the number of
        samples (count) parameters, where the most recent
        sample will be datetime.now().

        :param metric_name: the name of the metric.
        :param metric_type: type of the metric.
        :param labels: labels to be added to each sample.
        :param count: number of samples to be generated.
        :param interval_sec: seconds between each generated
          sample.
        :param add_unique_label: when set to True, a unique
          label pair will be added to all samples, thus
          creating a new series in prometheus. Providing a
          different label pair for every metric generation
          will have the same effect.
        :param inc_factor: factor used when calculating the
          value of a sample, which is a factor of the sample's
          interval.
        :param start_value: value of the first sample to be
          generated.
        :param timestamp: timestamp in ms for the most recent

        :return: String with all samples for a given metric.
        """
        ts_now_ms = timestamp or int(datetime.now().timestamp()*1000)

        # NOTE(dviroel): by including a unique label value, we avoid the
        #  'out of order sample' error when pushing multiple
        #  samples to prometheus that overlap the timestamp
        if add_unique_label:
            labels.update({"orig_timestamp": str(ts_now_ms)})

        # convert labels to exposition format
        str_labels = ""
        if labels:
            str_labels = json.dumps(labels, separators=(',', '='))

        # timestamp is in ms
        step_ms = interval_secs * 1000
        data = '# TYPE %s %s\n' % (metric_name, metric_type)
        # generate 'count' samples with incremental values
        # samples need to be ordered by their timestamp
        value = start_value
        for i in range(count, 0, -1):
            value += inc_factor * interval_secs
            data += '%s%s %s %s\n' % (
                metric_name, str_labels,
                value, ts_now_ms - step_ms * i
            )
        return data

    def make_instance_statistic_prometheus(self, instance):
        """Create Prometheus metrics for a instance

        :param instance: Instance response body
        """
        instance_labels = {
            "resource": instance['id'],
        }
        # Generate cpu usage data for a instance
        # unit is ns, so for a 80%, inc_factor is 0.8 * 1e+9
        cpu_data = self._generate_prometheus_metrics(
            self.PROMETHEUS_METRIC_MAP['instance_cpu_usage'],
            labels=instance_labels,
            start_value=1.0,
            inc_factor=8e+8)

        # Generate memory usage data for a instance consuming 80%
        # unit is megabytes, total is obtained from flavor
        # no inc_factor as memory is saved as gauge
        mem_usage_mb = int(instance['flavor']['ram'] * 0.8)
        ram_data = self._generate_prometheus_metrics(
            self.PROMETHEUS_METRIC_MAP['instance_ram_usage'],
            metric_type='gauge',
            labels=instance_labels,
            start_value=mem_usage_mb,
            inc_factor=0)

        self.prometheus_client.add_measures(cpu_data)
        self.prometheus_client.add_measures(ram_data)

    def make_host_statistic_prometheus(self, loaded_hosts=[]):
        """Create host resource and its measures in Prometheus.

        :param loaded_hosts: list of hosts that we want to inject data
          representing high usage of resource.
        """

        hypervisors = self.get_hypervisors_setup()

        for h in hypervisors:
            # When doing maths with prometheus, we need to
            # have all metrics with the same timestamp.
            timestamp = int(datetime.now().timestamp()*1000)
            instance = self.prometheus_client.prometheus_instances.get(
                h['hypervisor_hostname'], None)
            if not instance:
                LOG.info(f"Hostname {h['hypervisor_hostname']} does not "
                         "map to any prometheus instance.")
            else:
                # cpu metrics in prometheus are by cpu so we need to create
                # a set of metrics for each one.
                vcpus = h['vcpus']
                for cpu in range(vcpus):
                    host_labels = {
                        "instance": instance,
                        "fqdn": h['hypervisor_hostname'],
                        "mode": "idle",
                        "cpu": str(cpu),
                    }
                    # Generate host usage data
                    # unit is seconds, that represent cpu in idle
                    if h['hypervisor_hostname'] in loaded_hosts:
                        cpu_data = self._generate_prometheus_metrics(
                            self.PROMETHEUS_METRIC_MAP['host_cpu_usage'],
                            labels=host_labels,
                            start_value=1.0,
                            inc_factor=0.0,
                            timestamp=timestamp)
                    else:
                        cpu_data = self._generate_prometheus_metrics(
                            self.PROMETHEUS_METRIC_MAP['host_cpu_usage'],
                            labels=host_labels,
                            start_value=1.0,
                            inc_factor=1.0,
                            timestamp=timestamp)
                    self.prometheus_client.add_measures(cpu_data)

                host_labels_ram = {
                    "instance": instance,
                    "fqdn": h['hypervisor_hostname'],
                }

                # Generate memory usage data for a hypervisor
                # simulate 80% of memory usage on loaded_hosts
                # simulate 10% of memory load on others
                # unit is megabytes, total is obtained from hypervisor
                # no inc_factor as memory is saved as gauge

                load = 0.8 if h['hypervisor_hostname'] in loaded_hosts else 0.1
                mem_available_mb = int(h['memory_mb'] * (1 - load))
                # metric is node_memory_MemAvailable_bytes which is in bytes
                mem_available_bytes = mem_available_mb * 1024 * 1024
                ram_data = self._generate_prometheus_metrics(
                    self.PROMETHEUS_METRIC_MAP['host_ram_usage'],
                    metric_type='gauge',
                    labels=host_labels_ram,
                    start_value=mem_available_bytes,
                    inc_factor=0,
                    timestamp=timestamp)
                self.prometheus_client.add_measures(ram_data)

                # Generate host total memory data for a hypervisor
                # unit is megabytes, total is obtained from hypervisor
                # no inc_factor as memory is saved as gauge
                mem_total_mb = int(h['memory_mb'])
                # metric is node_memory_MemTotal_bytes which is in bytes
                mem_total_bytes = mem_total_mb * 1024 * 1024
                ram_total_data = self._generate_prometheus_metrics(
                    self.PROMETHEUS_METRIC_MAP['host_ram_total'],
                    metric_type='gauge',
                    labels=host_labels_ram,
                    start_value=mem_total_bytes,
                    inc_factor=0,
                    timestamp=timestamp)
                self.prometheus_client.add_measures(ram_total_data)

    def has_audit_succeeded(self, audit_uuid):
        _, audit = self.client.show_audit(audit_uuid)
        if audit.get('state') in ('FAILED', 'CANCELLED'):
            raise ValueError()

        return audit.get('state') == 'SUCCEEDED'

    @classmethod
    def has_audit_finished(cls, audit_uuid):
        _, audit = cls.client.show_audit(audit_uuid)
        return audit.get('state') in cls.AUDIT_FINISHED_STATES.values()

    @classmethod
    def is_audit_ongoing(cls, audit_uuid):
        _, audit = cls.client.show_audit(audit_uuid)
        return audit.get('state') == 'ONGOING'

    # ### ACTION PLANS ### #

    def has_action_plan_finished(self, action_plan_uuid):
        _, action_plan = self.client.show_action_plan(action_plan_uuid)
        return action_plan.get(
            'state') in self.ACTIONPLAN_FINISHED_STATES.values()

    def has_action_plans_finished(self):
        _, action_plans = self.client.list_action_plans()
        for ap in action_plans['action_plans']:
            _, action_plan = self.client.show_action_plan(ap['uuid'])
            if (action_plan.get('state') not in
                    self.ACTIONPLAN_FINISHED_STATES.values()):
                return False
        return True

    def has_action_plans_recommended(self, audit_uuid=None):
        _, action_plans = self.client.list_action_plans(audit_uuid=audit_uuid)
        for ap in action_plans['action_plans']:
            _, action_plan = self.client.show_action_plan(ap['uuid'])
            if action_plan.get('state') == 'RECOMMENDED':
                return True
        return False

    def create_audit_template_for_strategy(self, goal_name=None,
                                           strategy_name=None):
        """Create and return an audit template for the given goal/strategy."""

        goal_name = goal_name or self.GOAL
        strategy_name = strategy_name or self.STRATEGY

        _, goal = self.client.show_goal(goal_name)
        _, strategy = self.client.show_strategy(strategy_name)
        _, audit_template = self.create_audit_template(
            goal['uuid'], strategy=strategy['uuid'])

        return audit_template

    def create_audit_and_wait(self, audit_template_uuid, **audit_kwargs):
        """Create an audit from template and wait until it finishes.

        Returns the finished audit object (dict).
        Raises on FAILED/CANCELLED states.
        """
        # Ensure previous action plans are finished before
        # creating a new audit
        self.assertTrue(test_utils.call_until_true(
            func=functools.partial(
                self.has_action_plans_finished),
            duration=600,
            sleep_for=2
        ))

        audit_type = audit_kwargs.pop('audit_type', 'ONESHOT')
        state = audit_kwargs.pop('state', None)
        interval = audit_kwargs.pop('interval', None)
        parameters = audit_kwargs.pop('parameters', None)

        _, audit = self.create_audit(
            audit_template_uuid,
            audit_type=audit_type,
            state=state,
            interval=interval,
            parameters=parameters)

        try:
            self.assertTrue(test_utils.call_until_true(
                func=functools.partial(
                    self.has_audit_finished,
                    audit['uuid']),
                duration=600,
                sleep_for=2
            ))
        except ValueError:
            self.fail("The audit has failed!")

        _, finished_audit = self.client.show_audit(audit['uuid'])
        if finished_audit.get('state') in (
                'FAILED', 'CANCELLED', 'SUSPENDED'):
            self.fail("The audit ended in unexpected state: %s!"
                      % finished_audit.get('state'))

        return finished_audit

    def get_action_plan_and_validate_actions(
            self, audit_uuid, expected_action_types=None):
        """Fetch action plan for a finished audit and validate actions.

        Returns the action_plan object (dict).
        """
        _, action_plans = self.client.list_action_plans(
            audit_uuid=audit_uuid)
        action_plan_ref = action_plans['action_plans'][0]
        _, action_plan = self.client.show_action_plan(
            action_plan_ref['uuid'])

        actions = self.client.list_actions(
            action_plan_uuid=action_plan["uuid"])[1]['actions']

        if expected_action_types:
            action_types = {a['action_type'] for a in actions}
            if set(expected_action_types) != action_types:
                self.fail(
                    "The audit has found action types %s when expecting %s"
                    % (action_types, expected_action_types)
                )

        return action_plan, actions

    def execute_action_plan_and_validate_states(
            self, action_plan_uuid,
            expected_action_plan_states=['SUCCEEDED'],
            expected_action_states=['SUCCEEDED']):
        """Start the action plan execution and wait for completion.

        Returns the finished action plan object (dict).
        """
        # Execute the action by changing its state to PENDING
        _, updated_ap = self.client.start_action_plan(action_plan_uuid)

        self.assertTrue(test_utils.call_until_true(
            func=functools.partial(
                self.has_action_plan_finished,
                action_plan_uuid
            ),
            duration=600,
            sleep_for=2
        ))
        _, finished_ap = self.client.show_action_plan(action_plan_uuid)
        _, finished_actions = self.client.list_actions(
            action_plan_uuid=finished_ap["uuid"])
        self.assertIn(updated_ap['state'], ('PENDING', 'ONGOING'))
        self.assertIn(finished_ap['state'], expected_action_plan_states)

        for action in finished_actions['actions']:
            self.assertIn(action.get('state'), expected_action_states)

        return finished_ap, finished_actions

    def check_node_trait(self, node_id, trait):
        """Check if trait is in node traits

        :param node_id: The unique identifier of the node.
        :param trait: node trait
        :return: True if node has the trait else False
        """
        traits = self.placement_client.list_provider_traits(node_id)
        if trait in traits.get('traits'):
            return True
        else:
            return False

    def wait_for_instances_in_model(self, instances, timeout=300):
        """Waits until all instance ids are mapped to a model.

        Get the model and save instance ids and hypervisor hostname
        Get instances details and save instance ids and current hypervisora
        hostname
        (hypervisor hostname from argument is the one where it was created)
        Compare the two lists and wait until they are equal.
        """

        timeout_end = time.time() + timeout

        _, body = self.client.list_data_models(data_model_type="compute")
        model_pairs = [(s['server_uuid'], s['node_hostname'])
                       for s in body.get('context', [])
                       if 'server_uuid' in s and 'node_hostname' in s]
        instance_pairs = []
        for instance in instances:
            s = self.mgr.servers_client.show_server(instance['id'])['server']
            instance_pairs.append((s['id'], s['OS-EXT-SRV-ATTR:host']))

        # If no instances were created, we should not wait for them
        if not instance_pairs:
            raise Exception("No instances were created.")

        # Check all instances are in the model and model is not empty
        while (not set(instance_pairs) <= set(model_pairs) or not model_pairs):
            time.sleep(15)
            if time.time() >= timeout_end:
                raise Exception("Instances are not mapped to compute model.")

            _, body = self.client.list_data_models(data_model_type="compute")
            model_pairs = [(s['server_uuid'], s['node_hostname'])
                           for s in body.get('context', [])
                           if 'server_uuid' in s and 'node_hostname' in s]

    def wait_for_instances_attributes_in_model(self, instances, attributes_map,
                                               timeout=300):
        """Waits until all instances have attributes updated in the model.

        Builds a list of instances that have all attributes updated in the
        model. Compare with the list of instances provided as argument.

        :param instances: List of instances to wait for.
        :param attributes_map: Map withattributes and expected values.
        :param timeout: Timeout in seconds.

        :raises: Exception if attributes were not updated in the model.
        """

        timeout_end = time.time() + timeout

        _, body = self.client.list_data_models(data_model_type="compute")

        # Get all server_uuid from model that have all attributes updated
        # according to the attributes_map.
        model_ids = [
            s['server_uuid']
            for s in body.get('context', [])
            if 'server_uuid' in s and all(
                s.get(k) == v for k, v in attributes_map.items())]

        instance_ids = [i['id'] for i in instances]

        # Check all instances are in the model list built.
        while (not set(instance_ids) <= set(model_ids) or not model_ids):
            time.sleep(15)
            if time.time() >= timeout_end:
                raise Exception("Attributes were not updated in the model.")

            _, body = self.client.list_data_models(data_model_type="compute")
            model_ids = [
                s['server_uuid']
                for s in body.get('context', [])
                if 'server_uuid' in s and all(
                    s.get(k) == v for k, v in attributes_map.items())]

    def wait_delete_instances_from_model(self, timeout=300):
        """Waits until all deleted instaces be removed from model."""
        timeout_end = time.time() + timeout

        _, body = self.client.list_data_models(data_model_type="compute")
        model_uuids = [s["server_uuid"]
                       for s in body.get("context", []) if "server_uuid" in s]
        instances = self.mgr.servers_client.list_servers(
            detail=True)['servers']

        ids = [instance['id'] for instance in instances]

        while not set(model_uuids) <= set(ids):
            time.sleep(15)
            if time.time() >= timeout_end:
                raise Exception("Compute model still contains instances "
                                "that were already deleted. Failing...")

            _, body = self.client.list_data_models(data_model_type="compute")
            model_uuids = [
                s["server_uuid"]
                for s in body.get("context", []) if "server_uuid" in s]
