# coding=utf-8

# Copyright 2013 Hewlett-Packard Development Company, L.P.
# Copyright 2013 International Business Machines Corporation
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Test class for Ironic ManagerService."""

from collections import namedtuple
import datetime
import queue
import re
import time
from unittest import mock

import futurist
from futurist import waiters
from oslo_config import cfg
import oslo_messaging as messaging
from oslo_utils import uuidutils
from oslo_versionedobjects import base as ovo_base
from oslo_versionedobjects import fields
import tenacity

from ironic.common import boot_devices
from ironic.common import boot_modes
from ironic.common import components
from ironic.common import driver_factory
from ironic.common import exception
from ironic.common import faults
from ironic.common import health_states
from ironic.common import images
from ironic.common import indicator_states
from ironic.common import metrics as ironic_metrics
from ironic.common import nova
from ironic.common import states
from ironic.conductor import cleaning
from ironic.conductor import deployments
from ironic.conductor import inspection
from ironic.conductor import manager
from ironic.conductor import notification_utils
from ironic.conductor import servicing
from ironic.conductor import steps as conductor_steps
from ironic.conductor import task_manager
from ironic.conductor import utils as conductor_utils
from ironic.conductor import verify
from ironic.db import api as dbapi
from ironic.drivers import base as drivers_base
from ironic.drivers.modules import fake
from ironic.drivers.modules import image_utils
from ironic.drivers.modules import inspect_utils
from ironic.drivers.modules.network import common as n_common
from ironic.drivers.modules.network import flat as n_flat
from ironic.drivers.modules.redfish import management as redfish_mgmt
from ironic import objects
from ironic.objects import base as obj_base
from ironic.objects import fields as obj_fields
from ironic.tests.unit.conductor import mgr_utils
from ironic.tests.unit.db import base as db_base
from ironic.tests.unit.db import utils as db_utils
from ironic.tests.unit.objects import utils as obj_utils

CONF = cfg.CONF

INFO_DICT = db_utils.get_test_redfish_info()


@mgr_utils.mock_record_keepalive
class ChangeNodePowerStateTestCase(mgr_utils.ServiceSetUpMixin,
                                   db_base.DbTestCase):

    @mock.patch.object(fake.FakePower, 'get_power_state', autospec=True)
    def test_change_node_power_state_power_on(self, get_power_mock):
        # Test change_node_power_state including integration with
        # conductor.utils.node_power_action and lower.
        get_power_mock.return_value = states.POWER_OFF
        node = obj_utils.create_test_node(self.context,
                                          driver='fake-hardware',
                                          power_state=states.POWER_OFF)
        self._start_service()

        self.service.change_node_power_state(self.context,
                                             node.uuid,
                                             states.POWER_ON)

        get_power_mock.assert_called_once_with(mock.ANY, mock.ANY)
        node.refresh()
        self.assertEqual(states.POWER_ON, node.power_state)
        self.assertIsNone(node.target_power_state)
        self.assertIsNone(node.last_error)
        # Verify the reservation has been cleared by
        # background task's link callback.
        self.assertIsNone(node.reservation)

    @mock.patch.object(fake.FakePower, 'get_power_state', autospec=True)
    def test_change_node_power_state_soft_power_off_timeout(self,
                                                            get_power_mock):
        # Test change_node_power_state with timeout optional parameter
        # including integration with conductor.utils.node_power_action and
        # lower.
        get_power_mock.return_value = states.POWER_ON
        node = obj_utils.create_test_node(self.context,
                                          driver='fake-hardware',
                                          power_state=states.POWER_ON)
        self._start_service()

        self.service.change_node_power_state(self.context,
                                             node.uuid,
                                             states.SOFT_POWER_OFF,
                                             timeout=2)

        get_power_mock.assert_called_once_with(mock.ANY, mock.ANY)
        node.refresh()
        self.assertEqual(states.POWER_OFF, node.power_state)
        self.assertIsNone(node.target_power_state)
        self.assertIsNone(node.last_error)
        # Verify the reservation has been cleared by
        # background task's link callback.
        self.assertIsNone(node.reservation)

    @mock.patch.object(conductor_utils, 'node_power_action', autospec=True)
    def test_change_node_power_state_node_already_locked(self,
                                                         pwr_act_mock):
        # Test change_node_power_state with mocked
        # conductor.utils.node_power_action.
        fake_reservation = 'fake-reserv'
        pwr_state = states.POWER_ON
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          power_state=pwr_state,
                                          reservation=fake_reservation)
        self._start_service()

        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.change_node_power_state,
                                self.context,
                                node.uuid,
                                states.POWER_ON)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NodeLocked, exc.exc_info[0])

        self.assertFalse(pwr_act_mock.called, 'node_power_action has been '
                                              'unexpectedly called.')
        # Verify existing reservation wasn't broken.
        node.refresh()
        self.assertEqual(fake_reservation, node.reservation)

    def test_change_node_power_state_worker_pool_full(self):
        # Test change_node_power_state including integration with
        # conductor.utils.node_power_action and lower.
        initial_state = states.POWER_OFF
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          power_state=initial_state)
        self._start_service()

        with mock.patch.object(self.service,
                               '_spawn_worker', autospec=True) as spawn_mock:
            spawn_mock.side_effect = exception.NoFreeConductorWorker()

            exc = self.assertRaises(messaging.rpc.ExpectedException,
                                    self.service.change_node_power_state,
                                    self.context,
                                    node.uuid,
                                    states.POWER_ON)
            # Compare true exception hidden by @messaging.expected_exceptions
            self.assertEqual(exception.NoFreeConductorWorker, exc.exc_info[0])

            spawn_mock.assert_called_once_with(mock.ANY, mock.ANY,
                                               mock.ANY, timeout=mock.ANY)
            node.refresh()
            self.assertEqual(initial_state, node.power_state)
            self.assertIsNone(node.target_power_state)
            self.assertIsNotNone(node.last_error)
            # Verify the picked reservation has been cleared due to full pool.
            self.assertIsNone(node.reservation)

    @mock.patch.object(fake.FakePower, 'set_power_state', autospec=True)
    @mock.patch.object(fake.FakePower, 'get_power_state', autospec=True)
    def test_change_node_power_state_exception_in_background_task(
            self, get_power_mock, set_power_mock):
        # Test change_node_power_state including integration with
        # conductor.utils.node_power_action and lower.
        initial_state = states.POWER_OFF
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          power_state=initial_state)
        self._start_service()

        get_power_mock.return_value = states.POWER_OFF
        new_state = states.POWER_ON
        set_power_mock.side_effect = exception.PowerStateFailure(
            pstate=new_state
        )

        self.service.change_node_power_state(self.context,
                                             node.uuid,
                                             new_state)

        get_power_mock.assert_called_once_with(mock.ANY, mock.ANY)
        set_power_mock.assert_called_once_with(mock.ANY, mock.ANY,
                                               new_state, timeout=None)
        node.refresh()
        self.assertEqual(initial_state, node.power_state)
        self.assertIsNone(node.target_power_state)
        self.assertIsNotNone(node.last_error)
        # Verify the reservation has been cleared by background task's
        # link callback despite exception in background task.
        self.assertIsNone(node.reservation)

    @mock.patch.object(fake.FakePower, 'validate', autospec=True)
    def test_change_node_power_state_validate_fail(self, validate_mock):
        # Test change_node_power_state where task.driver.power.validate
        # fails and raises an exception
        initial_state = states.POWER_ON
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          power_state=initial_state)
        self._start_service()

        validate_mock.side_effect = exception.InvalidParameterValue(
            'wrong power driver info')

        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.change_node_power_state,
                                self.context,
                                node.uuid,
                                states.POWER_ON)

        self.assertEqual(exception.InvalidParameterValue, exc.exc_info[0])

        node.refresh()
        validate_mock.assert_called_once_with(mock.ANY, mock.ANY)
        self.assertEqual(states.POWER_ON, node.power_state)
        self.assertIsNone(node.target_power_state)
        self.assertIsNone(node.last_error)

    @mock.patch('ironic.objects.node.NodeSetPowerStateNotification',
                autospec=True)
    def test_node_set_power_state_notif_success(self, mock_notif):
        # Test that successfully changing a node's power state sends the
        # correct .start and .end notifications
        self.config(notification_level='info')
        self.config(host='my-host')
        # Required for exception handling
        mock_notif.__name__ = 'NodeSetPowerStateNotification'
        node = obj_utils.create_test_node(self.context,
                                          driver='fake-hardware',
                                          power_state=states.POWER_OFF)

        self._start_service()
        self.service.change_node_power_state(self.context,
                                             node.uuid,
                                             states.POWER_ON)

        # 2 notifications should be sent: 1 .start and 1 .end
        self.assertEqual(2, mock_notif.call_count)
        self.assertEqual(2, mock_notif.return_value.emit.call_count)

        first_notif_args = mock_notif.call_args_list[0][1]
        second_notif_args = mock_notif.call_args_list[1][1]

        self.assertNotificationEqual(first_notif_args,
                                     'ironic-conductor', CONF.host,
                                     'baremetal.node.power_set.start',
                                     obj_fields.NotificationLevel.INFO)
        self.assertNotificationEqual(second_notif_args,
                                     'ironic-conductor', CONF.host,
                                     'baremetal.node.power_set.end',
                                     obj_fields.NotificationLevel.INFO)

    @mock.patch.object(fake.FakePower, 'get_power_state', autospec=True)
    @mock.patch('ironic.objects.node.NodeSetPowerStateNotification',
                autospec=True)
    def test_node_set_power_state_notif_get_power_fail(self, mock_notif,
                                                       get_power_mock):
        # Test that correct notifications are sent when changing node power
        # state and retrieving the node's current power state fails
        self.config(notification_level='info')
        self.config(host='my-host')
        # Required for exception handling
        mock_notif.__name__ = 'NodeSetPowerStateNotification'
        node = obj_utils.create_test_node(self.context,
                                          driver='fake-hardware',
                                          power_state=states.POWER_OFF)
        self._start_service()

        get_power_mock.side_effect = Exception('I have failed')
        self.service.change_node_power_state(self.context,
                                             node.uuid,
                                             states.POWER_ON)

        get_power_mock.assert_called_once_with(mock.ANY, mock.ANY)

        # 2 notifications should be sent: 1 .start and 1 .error
        self.assertEqual(2, mock_notif.call_count)
        self.assertEqual(2, mock_notif.return_value.emit.call_count)

        first_notif_args = mock_notif.call_args_list[0][1]
        second_notif_args = mock_notif.call_args_list[1][1]

        self.assertNotificationEqual(first_notif_args,
                                     'ironic-conductor', CONF.host,
                                     'baremetal.node.power_set.start',
                                     obj_fields.NotificationLevel.INFO)
        self.assertNotificationEqual(second_notif_args,
                                     'ironic-conductor', CONF.host,
                                     'baremetal.node.power_set.error',
                                     obj_fields.NotificationLevel.ERROR)

    @mock.patch.object(fake.FakePower, 'set_power_state', autospec=True)
    @mock.patch('ironic.objects.node.NodeSetPowerStateNotification',
                autospec=True)
    def test_node_set_power_state_notif_set_power_fail(self, mock_notif,
                                                       set_power_mock):
        # Test that correct notifications are sent when changing node power
        # state and setting the node's power state fails
        self.config(notification_level='info')
        self.config(host='my-host')
        # Required for exception handling
        mock_notif.__name__ = 'NodeSetPowerStateNotification'
        node = obj_utils.create_test_node(self.context,
                                          driver='fake-hardware',
                                          power_state=states.POWER_OFF)
        self._start_service()

        set_power_mock.side_effect = Exception('I have failed')
        self.service.change_node_power_state(self.context,
                                             node.uuid,
                                             states.POWER_ON)

        set_power_mock.assert_called_once_with(mock.ANY, mock.ANY,
                                               states.POWER_ON, timeout=None)

        # 2 notifications should be sent: 1 .start and 1 .error
        self.assertEqual(2, mock_notif.call_count)
        self.assertEqual(2, mock_notif.return_value.emit.call_count)

        first_notif_args = mock_notif.call_args_list[0][1]
        second_notif_args = mock_notif.call_args_list[1][1]

        self.assertNotificationEqual(first_notif_args,
                                     'ironic-conductor', CONF.host,
                                     'baremetal.node.power_set.start',
                                     obj_fields.NotificationLevel.INFO)
        self.assertNotificationEqual(second_notif_args,
                                     'ironic-conductor', CONF.host,
                                     'baremetal.node.power_set.error',
                                     obj_fields.NotificationLevel.ERROR)

    @mock.patch('ironic.objects.node.NodeSetPowerStateNotification',
                autospec=True)
    def test_node_set_power_state_notif_spawn_fail(self, mock_notif):
        # Test that failure notification is not sent when spawning the
        # background conductor worker fails
        self.config(notification_level='info')
        self.config(host='my-host')
        # Required for exception handling
        mock_notif.__name__ = 'NodeSetPowerStateNotification'
        node = obj_utils.create_test_node(self.context,
                                          driver='fake-hardware',
                                          power_state=states.POWER_OFF)

        self._start_service()
        with mock.patch.object(self.service,
                               '_spawn_worker', autospec=True) as spawn_mock:
            spawn_mock.side_effect = exception.NoFreeConductorWorker()
            self.assertRaises(messaging.rpc.ExpectedException,
                              self.service.change_node_power_state,
                              self.context,
                              node.uuid,
                              states.POWER_ON)

            spawn_mock.assert_called_once_with(
                conductor_utils.node_power_action, mock.ANY, states.POWER_ON,
                timeout=None)
            self.assertFalse(mock_notif.called)

    @mock.patch('ironic.objects.node.NodeSetPowerStateNotification',
                autospec=True)
    def test_node_set_power_state_notif_no_state_change(self, mock_notif):
        # Test that correct notifications are sent when changing node power
        # state and no state change is necessary
        self.config(notification_level='info')
        self.config(host='my-host')
        # Required for exception handling
        mock_notif.__name__ = 'NodeSetPowerStateNotification'
        node = obj_utils.create_test_node(self.context,
                                          driver='fake-hardware',
                                          power_state=states.POWER_OFF)

        self._start_service()
        self.service.change_node_power_state(self.context,
                                             node.uuid,
                                             states.POWER_OFF)

        # 2 notifications should be sent: 1 .start and 1 .end
        self.assertEqual(2, mock_notif.call_count)
        self.assertEqual(2, mock_notif.return_value.emit.call_count)

        first_notif_args = mock_notif.call_args_list[0][1]
        second_notif_args = mock_notif.call_args_list[1][1]

        self.assertNotificationEqual(first_notif_args,
                                     'ironic-conductor', CONF.host,
                                     'baremetal.node.power_set.start',
                                     obj_fields.NotificationLevel.INFO)
        self.assertNotificationEqual(second_notif_args,
                                     'ironic-conductor', CONF.host,
                                     'baremetal.node.power_set.end',
                                     obj_fields.NotificationLevel.INFO)

    @mock.patch.object(fake.FakePower, 'get_supported_power_states',
                       autospec=True)
    def test_change_node_power_state_unsupported_state(self, supported_mock):
        # Test change_node_power_state where unsupported power state raises
        # an exception
        initial_state = states.POWER_ON
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          power_state=initial_state)
        self._start_service()

        supported_mock.return_value = [
            states.POWER_ON, states.POWER_OFF, states.REBOOT]

        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.change_node_power_state,
                                self.context,
                                node.uuid,
                                states.SOFT_POWER_OFF)

        self.assertEqual(exception.InvalidParameterValue, exc.exc_info[0])

        node.refresh()
        supported_mock.assert_called_once_with(mock.ANY, mock.ANY)
        self.assertEqual(states.POWER_ON, node.power_state)
        self.assertIsNone(node.target_power_state)
        self.assertIsNone(node.last_error)


@mgr_utils.mock_record_keepalive
class ChangeNodeBootModeTestCase(mgr_utils.ServiceSetUpMixin,
                                 db_base.DbTestCase):
    @mock.patch.object(fake.FakeManagement, 'set_boot_mode', autospec=True)
    @mock.patch.object(fake.FakeManagement, 'get_boot_mode', autospec=True)
    def test_change_node_boot_mode_valid(self, get_boot_mock, set_boot_mock):
        # Test change_node_boot_mode including integration with
        # conductor.utils.node_change_boot_mode
        get_boot_mock.side_effect = [boot_modes.LEGACY_BIOS,  # before setting
                                     boot_modes.UEFI]         # after setting
        node = obj_utils.create_test_node(self.context,
                                          driver='fake-hardware',
                                          boot_mode=boot_modes.LEGACY_BIOS)
        self._start_service()

        self.service.change_node_boot_mode(self.context,
                                           node.uuid,
                                           boot_modes.UEFI)

        set_boot_mock.assert_called_once_with(mock.ANY, mock.ANY,
                                              mode=boot_modes.UEFI)
        self.assertEqual(get_boot_mock.call_count, 1)
        # Call once before setting to see if it's required

        node.refresh()
        self.assertEqual(boot_modes.UEFI, node.boot_mode)
        self.assertIsNone(node.last_error)
        # Verify the reservation has been cleared by
        # background task's link callback.
        self.assertIsNone(node.reservation)

    @mock.patch.object(fake.FakeManagement, 'set_boot_mode', autospec=True)
    @mock.patch.object(fake.FakeManagement, 'get_boot_mode', autospec=True)
    def test_change_node_boot_mode_existing(self, get_boot_mock,
                                            set_boot_mock):
        # Test change_node_boot_mode including integration with
        # conductor.utils.node_change_boot_mode when target==current
        get_boot_mock.return_value = boot_modes.LEGACY_BIOS
        node = obj_utils.create_test_node(self.context,
                                          driver='fake-hardware',
                                          boot_mode=boot_modes.LEGACY_BIOS)
        self._start_service()

        self.service.change_node_boot_mode(self.context,
                                           node.uuid,
                                           boot_modes.LEGACY_BIOS)

        set_boot_mock.assert_not_called()
        self.assertEqual(get_boot_mock.call_count, 1)
        # Called once before setting to see if it's even required

        node.refresh()
        self.assertEqual(boot_modes.LEGACY_BIOS, node.boot_mode)
        self.assertIsNone(node.last_error)
        # Verify the reservation has been cleared by
        # background task's link callback.
        self.assertIsNone(node.reservation)

    @mock.patch.object(conductor_utils, 'node_change_boot_mode',
                       autospec=True)
    def test_change_node_boot_mode_node_already_locked(self, ncbm_mock):
        # Test change_node_boot_mode with mocked
        # conductor.utils.node_change_boot_mode.
        fake_reservation = 'fake-reserv'
        node = obj_utils.create_test_node(self.context,
                                          driver='fake-hardware',
                                          boot_mode=boot_modes.LEGACY_BIOS,
                                          reservation=fake_reservation)
        self._start_service()

        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.change_node_boot_mode,
                                self.context,
                                node.uuid,
                                boot_modes.LEGACY_BIOS)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NodeLocked, exc.exc_info[0])

        self.assertFalse(ncbm_mock.called, 'node_change_boot_mode has been '
                         'unexpectedly called.')
        # Verify existing reservation wasn't broken.
        node.refresh()
        self.assertEqual(fake_reservation, node.reservation)

    def test_change_node_boot_mode_worker_pool_full(self):
        # Test change_node_boot_mode including integration with
        # conductor.utils.change_node_boot_mode.
        initial_state = boot_modes.LEGACY_BIOS
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          boot_mode=initial_state)
        self._start_service()

        with mock.patch.object(self.service,
                               '_spawn_worker', autospec=True) as spawn_mock:
            spawn_mock.side_effect = exception.NoFreeConductorWorker()

            exc = self.assertRaises(messaging.rpc.ExpectedException,
                                    self.service.change_node_boot_mode,
                                    self.context,
                                    node.uuid,
                                    boot_modes.UEFI)
            # Compare true exception hidden by @messaging.expected_exceptions
            self.assertEqual(exception.NoFreeConductorWorker, exc.exc_info[0])

            spawn_mock.assert_called_once_with(mock.ANY, mock.ANY,
                                               mock.ANY)
            node.refresh()
            self.assertEqual(initial_state, node.boot_mode)
            self.assertIsNotNone(node.last_error)
            # Verify the picked reservation has been cleared due to full pool.
            self.assertIsNone(node.reservation)

    @mock.patch.object(fake.FakeManagement, 'set_boot_mode', autospec=True)
    @mock.patch.object(fake.FakeManagement, 'get_boot_mode', autospec=True)
    def test_change_node_boot_mode_exception_in_background_task(
            self, get_boot_mock, set_boot_mock):
        # Test change_node_boot_mode including integration with
        # conductor.utils.node_change_boot_mode.
        initial_state = boot_modes.LEGACY_BIOS
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          boot_mode=initial_state)
        self._start_service()

        get_boot_mock.return_value = boot_modes.LEGACY_BIOS
        new_state = boot_modes.UEFI
        set_boot_mock.side_effect = exception.UnsupportedDriverExtension(
            driver=fake, extension='set_boot_mode')

        self.service.change_node_boot_mode(self.context,
                                           node.uuid,
                                           new_state)

        # Call once before setting to see if it was required
        self.assertEqual(get_boot_mock.call_count, 1)
        set_boot_mock.assert_called_once_with(mock.ANY, mock.ANY,
                                              new_state)
        node.refresh()
        self.assertEqual(initial_state, node.boot_mode)
        self.assertIsNotNone(node.last_error)
        # Verify the reservation has been cleared by background task's
        # link callback despite exception in background task.
        self.assertIsNone(node.reservation)

    @mock.patch.object(fake.FakeManagement, 'validate', autospec=True)
    def test_change_node_boot_mode_validate_fail(self, validate_mock):
        # Test change_node_power_state where task.driver.management.validate
        # fails
        initial_state = boot_modes.UEFI
        node = obj_utils.create_test_node(self.context,
                                          driver='fake-hardware',
                                          boot_mode=initial_state)
        self._start_service()

        validate_mock.side_effect = exception.InvalidParameterValue(
            'wrong management driver info')
        new_state = boot_modes.LEGACY_BIOS
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.change_node_boot_mode,
                                self.context,
                                node.uuid,
                                new_state)

        self.assertEqual(exception.InvalidParameterValue, exc.exc_info[0])

        node.refresh()
        validate_mock.assert_called_once_with(mock.ANY, mock.ANY)
        self.assertEqual(initial_state, node.boot_mode)
        self.assertIsNone(node.last_error)

    @mock.patch.object(fake.FakeManagement, 'set_boot_mode', autospec=True)
    @mock.patch.object(fake.FakeManagement, 'get_boot_mode', autospec=True)
    def test_change_node_boot_mode_exception_getting_current(self,
                                                             get_boot_mock,
                                                             set_boot_mock):
        # Test change_node_boot_mode smooth operation when get_boot_mode mode
        # raises an exception
        initial_state = boot_modes.LEGACY_BIOS
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          boot_mode=initial_state)
        self._start_service()

        get_boot_mock.side_effect = exception.UnsupportedDriverExtension(
            driver=fake, extension='get_boot_mode')
        new_state = boot_modes.UEFI

        self.service.change_node_boot_mode(self.context,
                                           node.uuid,
                                           new_state)

        # Call once before setting to see if it is required
        self.assertEqual(get_boot_mock.call_count, 1)
        set_boot_mock.assert_called_once_with(mock.ANY, mock.ANY,
                                              new_state)
        node.refresh()
        self.assertEqual(new_state, node.boot_mode)
        self.assertIsNone(node.last_error)
        # Verify the reservation has been cleared by
        # background task's link callback.
        self.assertIsNone(node.reservation)


@mgr_utils.mock_record_keepalive
class ChangeNodeSecureBootTestCase(mgr_utils.ServiceSetUpMixin,
                                   db_base.DbTestCase):
    @mock.patch.object(fake.FakeManagement, 'set_secure_boot_state',
                       autospec=True)
    @mock.patch.object(fake.FakeManagement, 'get_secure_boot_state',
                       autospec=True)
    def test_change_node_secure_boot_valid(self, get_boot_mock, set_boot_mock):
        # Test change_node_secure_boot including integration with
        # conductor.utils.node_change_secure_boot
        get_boot_mock.side_effect = [False,  # before setting
                                     True]   # after setting
        initial_state = False
        node = obj_utils.create_test_node(self.context,
                                          driver='fake-hardware',
                                          secure_boot=initial_state)
        self._start_service()
        target_state = True
        self.service.change_node_secure_boot(self.context,
                                             node.uuid,
                                             target_state)

        set_boot_mock.assert_called_once_with(mock.ANY, mock.ANY,
                                              target_state)
        self.assertEqual(get_boot_mock.call_count, 1)
        # Call once before setting to see if it's required

        node.refresh()
        self.assertEqual(target_state, node.secure_boot)
        self.assertIsNone(node.last_error)
        # Verify the reservation has been cleared by
        # background task's link callback.
        self.assertIsNone(node.reservation)

    @mock.patch.object(fake.FakeManagement, 'set_secure_boot_state',
                       autospec=True)
    @mock.patch.object(fake.FakeManagement, 'get_secure_boot_state',
                       autospec=True)
    def test_change_node_secure_boot_existing(self, get_boot_mock,
                                              set_boot_mock):
        # Test change_node_secure_boot including integration with
        # conductor.utils.node_change_secure_boot when target==current
        get_boot_mock.return_value = False
        node = obj_utils.create_test_node(self.context,
                                          driver='fake-hardware',
                                          secure_boot=False)
        self._start_service()

        self.service.change_node_secure_boot(self.context,
                                             node.uuid,
                                             False)

        set_boot_mock.assert_not_called()
        self.assertEqual(get_boot_mock.call_count, 1)
        # Called once before setting to see if it's even required

        node.refresh()
        self.assertEqual(False, node.secure_boot)
        self.assertIsNone(node.last_error)
        # Verify the reservation has been cleared by
        # background task's link callback.
        self.assertIsNone(node.reservation)

    @mock.patch.object(conductor_utils, 'node_change_secure_boot',
                       autospec=True)
    def test_change_node_secure_boot_node_already_locked(self, ncbm_mock):
        # Test change_node_secure_boot with mocked
        # conductor.utils.node_change_secure_boot.
        fake_reservation = 'fake-reserv'
        node = obj_utils.create_test_node(self.context,
                                          driver='fake-hardware',
                                          secure_boot=False,
                                          reservation=fake_reservation)
        self._start_service()

        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.change_node_secure_boot,
                                self.context,
                                node.uuid,
                                False)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NodeLocked, exc.exc_info[0])

        self.assertFalse(ncbm_mock.called, 'node_change_secure_boot has been '
                         'unexpectedly called.')
        # Verify existing reservation wasn't broken.
        node.refresh()
        self.assertEqual(fake_reservation, node.reservation)

    def test_change_node_secure_boot_worker_pool_full(self):
        # Test change_node_secure_boot including integration with
        # conductor.utils.change_node_secure_boot.
        initial_state = False
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          secure_boot=initial_state)
        self._start_service()

        with mock.patch.object(self.service,
                               '_spawn_worker', autospec=True) as spawn_mock:
            spawn_mock.side_effect = exception.NoFreeConductorWorker()

            exc = self.assertRaises(messaging.rpc.ExpectedException,
                                    self.service.change_node_secure_boot,
                                    self.context,
                                    node.uuid,
                                    True)
            # Compare true exception hidden by @messaging.expected_exceptions
            self.assertEqual(exception.NoFreeConductorWorker, exc.exc_info[0])

            spawn_mock.assert_called_once_with(mock.ANY, mock.ANY,
                                               mock.ANY)
            node.refresh()
            self.assertEqual(initial_state, node.secure_boot)
            self.assertIsNotNone(node.last_error)
            # Verify the picked reservation has been cleared due to full pool.
            self.assertIsNone(node.reservation)

    @mock.patch.object(fake.FakeManagement, 'set_secure_boot_state',
                       autospec=True)
    @mock.patch.object(fake.FakeManagement, 'get_secure_boot_state',
                       autospec=True)
    def test_change_node_secure_boot_exception_in_background_task(
            self, get_boot_mock, set_boot_mock):
        # Test change_node_secure_boot including integration with
        # conductor.utils.node_change_secure_boot.
        initial_state = False
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          secure_boot=initial_state)
        self._start_service()

        get_boot_mock.return_value = False
        new_state = True
        set_boot_mock.side_effect = exception.UnsupportedDriverExtension(
            driver=fake, extension='set_secure_boot_state')

        self.service.change_node_secure_boot(self.context,
                                             node.uuid,
                                             new_state)

        # Call once before setting to see if it was required
        self.assertEqual(get_boot_mock.call_count, 1)
        set_boot_mock.assert_called_once_with(mock.ANY, mock.ANY,
                                              new_state)
        node.refresh()
        self.assertEqual(initial_state, node.secure_boot)
        self.assertIsNotNone(node.last_error)
        # Verify the reservation has been cleared by background task's
        # link callback despite exception in background task.
        self.assertIsNone(node.reservation)

    @mock.patch.object(fake.FakeManagement, 'validate', autospec=True)
    def test_change_node_secure_boot_validate_fail(self, validate_mock):
        # Test change_node_power_state where task.driver.management.validate
        # fails
        initial_state = True
        node = obj_utils.create_test_node(self.context,
                                          driver='fake-hardware',
                                          secure_boot=initial_state)
        self._start_service()

        validate_mock.side_effect = exception.InvalidParameterValue(
            'wrong management driver info')
        new_state = False
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.change_node_secure_boot,
                                self.context,
                                node.uuid,
                                new_state)

        self.assertEqual(exception.InvalidParameterValue, exc.exc_info[0])

        node.refresh()
        validate_mock.assert_called_once_with(mock.ANY, mock.ANY)
        self.assertEqual(initial_state, node.secure_boot)
        self.assertIsNone(node.last_error)

    @mock.patch.object(fake.FakeManagement, 'set_secure_boot_state',
                       autospec=True)
    @mock.patch.object(fake.FakeManagement, 'get_secure_boot_state',
                       autospec=True)
    def test_change_node_secure_boot_exception_getting_current(self,
                                                               get_boot_mock,
                                                               set_boot_mock):
        # Test change_node_secure_boot smooth operation when
        # get_secure_boot_state raises an exception
        initial_state = False
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          secure_boot=initial_state)
        self._start_service()

        get_boot_mock.side_effect = exception.UnsupportedDriverExtension(
            driver=fake, extension='get_secure_boot_state')
        new_state = True

        self.service.change_node_secure_boot(self.context,
                                             node.uuid,
                                             new_state)

        # Call once before setting to see if it is required
        self.assertEqual(get_boot_mock.call_count, 1)
        set_boot_mock.assert_called_once_with(mock.ANY, mock.ANY,
                                              new_state)
        node.refresh()
        self.assertEqual(new_state, node.secure_boot)
        self.assertIsNone(node.last_error)
        # Verify the reservation has been cleared by
        # background task's link callback.
        self.assertIsNone(node.reservation)


@mgr_utils.mock_record_keepalive
class CreateNodeTestCase(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase):
    def test_create_node(self):
        node = obj_utils.get_test_node(self.context, driver='fake-hardware',
                                       extra={'test': 'one'})

        res = self.service.create_node(self.context, node)

        self.assertEqual({'test': 'one'}, res['extra'])
        res = objects.Node.get_by_uuid(self.context, node['uuid'])
        self.assertEqual({'test': 'one'}, res['extra'])

    @mock.patch.object(driver_factory, 'check_and_update_node_interfaces',
                       autospec=True)
    def test_create_node_validation_fails(self, mock_validate):
        node = obj_utils.get_test_node(self.context, driver='fake-hardware',
                                       extra={'test': 'one'})
        mock_validate.side_effect = exception.InterfaceNotFoundInEntrypoint(
            'boom')

        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.create_node,
                                self.context, node)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.InterfaceNotFoundInEntrypoint,
                         exc.exc_info[0])

        self.assertRaises(exception.NotFound,
                          objects.Node.get_by_uuid, self.context, node['uuid'])


@mgr_utils.mock_record_keepalive
class UpdateNodeTestCase(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase):
    def test_update_node(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          extra={'test': 'one'})

        # check that ManagerService.update_node actually updates the node
        node.extra = {'test': 'two'}
        res = self.service.update_node(self.context, node)
        self.assertEqual({'test': 'two'}, res['extra'])

    def test_update_node_maintenance_set_false(self):
        node = obj_utils.create_test_node(self.context,
                                          driver='fake-hardware',
                                          maintenance=True,
                                          fault='clean failure',
                                          maintenance_reason='reason')

        # check that ManagerService.update_node actually updates the node
        node.maintenance = False
        res = self.service.update_node(self.context, node)
        self.assertFalse(res['maintenance'])
        self.assertIsNone(res['maintenance_reason'])
        self.assertIsNone(res['fault'])

    def test_update_node_protected_set(self):
        for state in ('active', 'rescue'):
            node = obj_utils.create_test_node(self.context,
                                              uuid=uuidutils.generate_uuid(),
                                              provision_state=state)

            node.protected = True
            res = self.service.update_node(self.context, node)
            self.assertTrue(res['protected'])
            self.assertIsNone(res['protected_reason'])

    def test_update_node_protected_unset(self):
        # NOTE(dtantsur): we allow unsetting protected in any state to make
        # sure a node cannot get stuck in it.
        for state in ('active', 'rescue', 'rescue failed'):
            node = obj_utils.create_test_node(self.context,
                                              uuid=uuidutils.generate_uuid(),
                                              provision_state=state,
                                              protected=True,
                                              protected_reason='reason')

            # check that ManagerService.update_node actually updates the node
            node.protected = False
            res = self.service.update_node(self.context, node)
            self.assertFalse(res['protected'])
            self.assertIsNone(res['protected_reason'])

    def test_update_node_protected_invalid_state(self):
        node = obj_utils.create_test_node(self.context,
                                          provision_state='available')

        node.protected = True
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.update_node,
                                self.context,
                                node)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.InvalidState, exc.exc_info[0])

        res = objects.Node.get_by_uuid(self.context, node['uuid'])
        self.assertFalse(res['protected'])
        self.assertIsNone(res['protected_reason'])

    def test_update_node_protected_reason_without_protected(self):
        node = obj_utils.create_test_node(self.context,
                                          provision_state='active')

        node.protected_reason = 'reason!'
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.update_node,
                                self.context,
                                node)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.InvalidParameterValue, exc.exc_info[0])

        res = objects.Node.get_by_uuid(self.context, node['uuid'])
        self.assertFalse(res['protected'])
        self.assertIsNone(res['protected_reason'])

    def test_update_node_retired_set(self):
        for state in ('active', 'rescue', 'manageable'):
            node = obj_utils.create_test_node(self.context,
                                              uuid=uuidutils.generate_uuid(),
                                              provision_state=state)

            node.retired = True
            res = self.service.update_node(self.context, node)
            self.assertTrue(res['retired'])
            self.assertIsNone(res['retired_reason'])

    def test_update_node_retired_invalid_state(self):
        # NOTE(arne_wiebalck): nodes in available cannot be 'retired'.
        # This is to ensure backwards compatibility.
        node = obj_utils.create_test_node(self.context,
                                          provision_state='available')

        node.retired = True
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.update_node,
                                self.context,
                                node)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.InvalidState, exc.exc_info[0])

        res = objects.Node.get_by_uuid(self.context, node['uuid'])
        self.assertFalse(res['retired'])
        self.assertIsNone(res['retired_reason'])

    def test_update_node_retired_unset(self):
        for state in ('active', 'manageable', 'rescue', 'rescue failed'):
            node = obj_utils.create_test_node(self.context,
                                              uuid=uuidutils.generate_uuid(),
                                              provision_state=state,
                                              retired=True,
                                              retired_reason='EOL')

            # check that ManagerService.update_node actually updates the node
            node.retired = False
            res = self.service.update_node(self.context, node)
            self.assertFalse(res['retired'])
            self.assertIsNone(res['retired_reason'])

    def test_update_node_retired_reason_without_retired(self):
        node = obj_utils.create_test_node(self.context,
                                          provision_state='active')

        node.retired_reason = 'warranty expired'
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.update_node,
                                self.context,
                                node)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.InvalidParameterValue, exc.exc_info[0])

        res = objects.Node.get_by_uuid(self.context, node['uuid'])
        self.assertFalse(res['retired'])
        self.assertIsNone(res['retired_reason'])

    def test_update_node_already_locked(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          extra={'test': 'one'})

        # check that it fails if something else has locked it already
        with task_manager.acquire(self.context, node['id'], shared=False):
            node.extra = {'test': 'two'}
            exc = self.assertRaises(messaging.rpc.ExpectedException,
                                    self.service.update_node,
                                    self.context,
                                    node)
            # Compare true exception hidden by @messaging.expected_exceptions
            self.assertEqual(exception.NodeLocked, exc.exc_info[0])

        # verify change did not happen
        res = objects.Node.get_by_uuid(self.context, node['uuid'])
        self.assertEqual({'test': 'one'}, res['extra'])

    def test_update_node_already_associated(self):
        old_instance = uuidutils.generate_uuid()
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          instance_uuid=old_instance)
        node.instance_uuid = uuidutils.generate_uuid()
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.update_node,
                                self.context,
                                node)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NodeAssociated, exc.exc_info[0])

        # verify change did not happen
        res = objects.Node.get_by_uuid(self.context, node['uuid'])
        self.assertEqual(old_instance, res['instance_uuid'])

    @mock.patch('ironic.drivers.modules.fake.FakePower.get_power_state',
                autospec=True)
    def _test_associate_node(self, power_state, mock_get_power_state):
        mock_get_power_state.return_value = power_state
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          instance_uuid=None,
                                          power_state=states.NOSTATE)
        uuid1 = uuidutils.generate_uuid()
        uuid2 = uuidutils.generate_uuid()
        node.instance_uuid = uuid1
        self.service.update_node(self.context, node)

        # Check if the change was applied
        node.instance_uuid = uuid2
        node.refresh()
        self.assertEqual(uuid1, node.instance_uuid)

    def test_associate_node_powered_off(self):
        self._test_associate_node(states.POWER_OFF)

    def test_associate_node_powered_on(self):
        self._test_associate_node(states.POWER_ON)

    def test_update_node_invalid_driver(self):
        existing_driver = 'fake-hardware'
        wrong_driver = 'wrong-driver'
        node = obj_utils.create_test_node(self.context,
                                          driver=existing_driver,
                                          extra={'test': 'one'},
                                          instance_uuid=None)
        # check that it fails because driver not found
        node.driver = wrong_driver
        node.driver_info = {}
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.update_node,
                                self.context, node)
        self.assertEqual(exception.DriverNotFound, exc.exc_info[0])

        # verify change did not happen
        node.refresh()
        self.assertEqual(existing_driver, node.driver)

    def test_update_node_from_invalid_driver(self):
        existing_driver = 'fake-hardware'
        wrong_driver = 'wrong-driver'
        node = obj_utils.create_test_node(self.context, driver=wrong_driver)
        node.driver = existing_driver
        result = self.service.update_node(self.context, node)
        self.assertEqual(existing_driver, result.driver)
        node.refresh()
        self.assertEqual(existing_driver, node.driver)

    UpdateInterfaces = namedtuple('UpdateInterfaces', ('old', 'new'))
    # NOTE(dtantsur): "old" interfaces here do not match the defaults, so that
    # we can test resetting them.
    IFACE_UPDATE_DICT = {
        'boot_interface': UpdateInterfaces('pxe', 'fake'),
        'console_interface': UpdateInterfaces('no-console', 'fake'),
        'deploy_interface': UpdateInterfaces('direct', 'fake'),
        'inspect_interface': UpdateInterfaces('no-inspect', 'fake'),
        'management_interface': UpdateInterfaces(None, 'fake'),
        'network_interface': UpdateInterfaces('noop', 'flat'),
        'power_interface': UpdateInterfaces(None, 'fake'),
        'raid_interface': UpdateInterfaces('no-raid', 'fake'),
        'rescue_interface': UpdateInterfaces('no-rescue', 'fake'),
        'storage_interface': UpdateInterfaces('fake', 'noop'),
    }

    def _create_node_with_interfaces(self, prov_state, maintenance=False):
        old_ifaces = {}
        for iface_name, ifaces in self.IFACE_UPDATE_DICT.items():
            old_ifaces[iface_name] = ifaces.old
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          uuid=uuidutils.generate_uuid(),
                                          provision_state=prov_state,
                                          maintenance=maintenance,
                                          **old_ifaces)
        return node

    def _test_update_node_interface_allowed(self, node, iface_name, new_iface):
        setattr(node, iface_name, new_iface)
        self.service.update_node(self.context, node)
        node.refresh()
        self.assertEqual(new_iface, getattr(node, iface_name))

    def _test_update_node_interface_in_allowed_state(self, prov_state,
                                                     maintenance=False):
        node = self._create_node_with_interfaces(prov_state,
                                                 maintenance=maintenance)
        for iface_name, ifaces in self.IFACE_UPDATE_DICT.items():
            self._test_update_node_interface_allowed(node, iface_name,
                                                     ifaces.new)
        node.destroy()

    def test_update_node_interface_in_allowed_state(self):
        for state in [states.ENROLL, states.MANAGEABLE, states.INSPECTING,
                      states.INSPECTWAIT, states.AVAILABLE]:
            self._test_update_node_interface_in_allowed_state(state)

    def test_update_node_interface_in_maintenance(self):
        self._test_update_node_interface_in_allowed_state(states.ACTIVE,
                                                          maintenance=True)

    def _test_update_node_interface_not_allowed(self, node, iface_name,
                                                new_iface):
        old_iface = getattr(node, iface_name)
        setattr(node, iface_name, new_iface)
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.update_node,
                                self.context, node)
        self.assertEqual(exception.InvalidState, exc.exc_info[0])
        node.refresh()
        self.assertEqual(old_iface, getattr(node, iface_name))

    def _test_update_node_interface_in_not_allowed_state(self, prov_state):
        node = self._create_node_with_interfaces(prov_state)
        for iface_name, ifaces in self.IFACE_UPDATE_DICT.items():
            self._test_update_node_interface_not_allowed(node, iface_name,
                                                         ifaces.new)
        node.destroy()

    def test_update_node_interface_in_not_allowed_state(self):
        for state in [states.ACTIVE, states.DELETING]:
            self._test_update_node_interface_in_not_allowed_state(state)

    def _test_update_node_interface_invalid(self, node, iface_name):
        old_iface = getattr(node, iface_name)
        setattr(node, iface_name, 'invalid')
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.update_node,
                                self.context, node)
        self.assertEqual(exception.InterfaceNotFoundInEntrypoint,
                         exc.exc_info[0])
        node.refresh()
        self.assertEqual(old_iface, getattr(node, iface_name))

    def test_update_node_interface_invalid(self):
        node = self._create_node_with_interfaces(states.MANAGEABLE)
        for iface_name in self.IFACE_UPDATE_DICT:
            self._test_update_node_interface_invalid(node, iface_name)

    def test_update_node_with_reset_interfaces(self):
        # Modify only one interface at a time
        for iface_name, ifaces in self.IFACE_UPDATE_DICT.items():
            node = self._create_node_with_interfaces(states.AVAILABLE)
            setattr(node, iface_name, ifaces.new)
            # Updating a driver is mandatory for reset_interfaces to work
            node.driver = 'fake-hardware'
            self.service.update_node(self.context, node,
                                     reset_interfaces=True)
            node.refresh()
            self.assertEqual(ifaces.new, getattr(node, iface_name))
            # Other interfaces must be reset to their defaults
            for other_iface_name, ifaces in self.IFACE_UPDATE_DICT.items():
                if other_iface_name == iface_name:
                    continue
                # For this to work, the "old" interfaces in IFACE_UPDATE_DICT
                # must not match the defaults.
                self.assertNotEqual(ifaces.old,
                                    getattr(node, other_iface_name),
                                    "%s does not match the default after "
                                    "reset with setting %s: %s" %
                                    (other_iface_name, iface_name,
                                     getattr(node, other_iface_name)))

    def _test_update_node_change_resource_class(self, state,
                                                resource_class=None,
                                                new_resource_class='new',
                                                expect_error=False,
                                                maintenance=False):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          uuid=uuidutils.generate_uuid(),
                                          provision_state=state,
                                          resource_class=resource_class,
                                          maintenance=maintenance)
        self.addCleanup(node.destroy)

        node.resource_class = new_resource_class
        if expect_error:
            exc = self.assertRaises(messaging.rpc.ExpectedException,
                                    self.service.update_node,
                                    self.context,
                                    node)
            # Compare true exception hidden by @messaging.expected_exceptions
            self.assertEqual(exception.InvalidState, exc.exc_info[0])

            expected_msg_regex = \
                (r'^Node {} can not have resource_class updated unless it is '
                 r'in one of allowed \(.*\) states. '
                 r'Current state is \'{}\'.$').format(
                    re.escape(node.uuid),
                    re.escape(node.provision_state))
            self.assertRegex(str(exc.exc_info[1]), expected_msg_regex)

            # verify change did not happen
            res = objects.Node.get_by_uuid(self.context, node['uuid'])
            self.assertEqual(resource_class, res['resource_class'])
        else:
            self.service.update_node(self.context, node)

            res = objects.Node.get_by_uuid(self.context, node['uuid'])
            self.assertEqual('new', res['resource_class'])

    def test_update_resource_class_allowed_state(self):
        for state in [states.ENROLL, states.MANAGEABLE, states.INSPECTING,
                      states.AVAILABLE]:
            self._test_update_node_change_resource_class(
                state, resource_class='old', expect_error=False)

    def test_update_resource_class_no_previous_value(self):
        for state in [states.ENROLL, states.MANAGEABLE, states.INSPECTING,
                      states.AVAILABLE, states.ACTIVE]:
            self._test_update_node_change_resource_class(
                state, resource_class=None, expect_error=False)

    def test_update_resource_class_not_allowed(self):
        self._test_update_node_change_resource_class(
            states.ACTIVE, resource_class='old', new_resource_class='new',
            expect_error=True)
        self._test_update_node_change_resource_class(
            states.ACTIVE, resource_class='old', new_resource_class=None,
            expect_error=True)
        self._test_update_node_change_resource_class(
            states.ACTIVE, resource_class='old', new_resource_class=None,
            expect_error=True, maintenance=True)

    def test_update_node_hardware_type(self):
        existing_hardware = 'fake-hardware'
        existing_interface = 'fake'
        new_hardware = 'manual-management'
        new_interface = 'pxe'
        node = obj_utils.create_test_node(self.context,
                                          driver=existing_hardware,
                                          boot_interface=existing_interface)
        node.driver = new_hardware
        node.boot_interface = new_interface
        self.service.update_node(self.context, node)
        node.refresh()
        self.assertEqual(new_hardware, node.driver)
        self.assertEqual(new_interface, node.boot_interface)

    def test_update_node_deleting_allocation(self):
        node = obj_utils.create_test_node(self.context)
        alloc = obj_utils.create_test_allocation(self.context)
        # Establish cross-linking between the node and the allocation
        alloc.node_id = node.id
        alloc.save()
        node.refresh()
        self.assertEqual(alloc.id, node.allocation_id)
        self.assertEqual(alloc.uuid, node.instance_uuid)

        node.instance_uuid = None
        res = self.service.update_node(self.context, node)
        self.assertRaises(exception.AllocationNotFound,
                          objects.Allocation.get_by_id,
                          self.context, alloc.id)
        self.assertIsNone(res['instance_uuid'])
        self.assertIsNone(res['allocation_id'])

        node.refresh()
        self.assertIsNone(node.instance_uuid)
        self.assertIsNone(node.allocation_id)

    def test_update_node_deleting_allocation_forbidden(self):
        node = obj_utils.create_test_node(self.context,
                                          provision_state='active',
                                          maintenance=False)
        alloc = obj_utils.create_test_allocation(self.context)
        # Establish cross-linking between the node and the allocation
        alloc.node_id = node.id
        alloc.save()
        node.refresh()
        self.assertEqual(alloc.id, node.allocation_id)
        self.assertEqual(alloc.uuid, node.instance_uuid)

        node.instance_uuid = None
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.update_node,
                                self.context, node)
        self.assertEqual(exception.InvalidState, exc.exc_info[0])

        node.refresh()
        self.assertEqual(alloc.id, node.allocation_id)
        self.assertEqual(alloc.uuid, node.instance_uuid)

    def test_update_node_deleting_allocation_in_maintenance(self):
        node = obj_utils.create_test_node(self.context,
                                          provision_state='active',
                                          maintenance=True)
        alloc = obj_utils.create_test_allocation(self.context)
        # Establish cross-linking between the node and the allocation
        alloc.node_id = node.id
        alloc.save()
        node.refresh()
        self.assertEqual(alloc.id, node.allocation_id)
        self.assertEqual(alloc.uuid, node.instance_uuid)

        node.instance_uuid = None
        res = self.service.update_node(self.context, node)
        self.assertRaises(exception.AllocationNotFound,
                          objects.Allocation.get_by_id,
                          self.context, alloc.id)
        self.assertIsNone(res['instance_uuid'])
        self.assertIsNone(res['allocation_id'])

        node.refresh()
        self.assertIsNone(node.instance_uuid)
        self.assertIsNone(node.allocation_id)

    def test_update_node_maintenance_with_broken_interface(self):
        # Updates of non-driver fields are possible with a broken driver
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          power_interface='foobar',
                                          extra={'test': 'one'})

        node.maintenance = True
        res = self.service.update_node(self.context, node)
        self.assertTrue(res.maintenance)

        node.refresh()
        self.assertTrue(node.maintenance)
        self.assertEqual('foobar', node.power_interface)

    def test_update_node_interface_field_with_broken_interface(self):
        # Updates of driver fields are NOT possible with a broken driver,
        # unless they're fixing the breakage.
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          power_interface='foobar',
                                          deploy_interface='fake',
                                          extra={'test': 'one'})

        node.deploy_interface = 'direct'
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.update_node,
                                self.context, node)
        self.assertEqual(exception.InterfaceNotFoundInEntrypoint,
                         exc.exc_info[0])

        node.refresh()
        self.assertEqual('foobar', node.power_interface)
        self.assertEqual('fake', node.deploy_interface)

    def test_update_node_fix_broken_interface(self):
        # Updates of non-driver fields are possible with a broken driver
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          power_interface='foobar',
                                          extra={'test': 'one'})

        node.power_interface = 'fake'
        self.service.update_node(self.context, node)

        node.refresh()
        self.assertEqual('fake', node.power_interface)


@mgr_utils.mock_record_keepalive
class VendorPassthruTestCase(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase):

    @mock.patch.object(task_manager.TaskManager, 'upgrade_lock', autospec=True)
    @mock.patch.object(task_manager.TaskManager, 'spawn_after', autospec=True)
    def test_vendor_passthru_async(self, mock_spawn,
                                   mock_upgrade):
        node = obj_utils.create_test_node(self.context,
                                          vendor_interface='fake')
        info = {'bar': 'baz'}
        self._start_service()

        response = self.service.vendor_passthru(self.context, node.uuid,
                                                'second_method', 'POST',
                                                info)

        # Assert spawn_after was called
        self.assertTrue(mock_spawn.called)
        self.assertIsNone(response['return'])
        self.assertTrue(response['async'])

        # Assert lock was upgraded to an exclusive one
        self.assertEqual(1, mock_upgrade.call_count)

        node.refresh()
        self.assertIsNone(node.last_error)
        # Verify reservation has been cleared.
        self.assertIsNone(node.reservation)

    @mock.patch.object(task_manager.TaskManager, 'upgrade_lock', autospec=True)
    @mock.patch.object(task_manager.TaskManager, 'spawn_after', autospec=True)
    def test_vendor_passthru_sync(self, mock_spawn, mock_upgrade):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        info = {'bar': 'meow'}
        self._start_service()

        response = self.service.vendor_passthru(self.context, node.uuid,
                                                'third_method_sync',
                                                'POST', info)

        # Assert no workers were used
        self.assertFalse(mock_spawn.called)
        self.assertTrue(response['return'])
        self.assertFalse(response['async'])

        # Assert lock was upgraded to an exclusive one
        self.assertEqual(1, mock_upgrade.call_count)

        node.refresh()
        self.assertIsNone(node.last_error)
        # Verify reservation has been cleared.
        self.assertIsNone(node.reservation)

    @mock.patch.object(task_manager.TaskManager, 'upgrade_lock', autospec=True)
    @mock.patch.object(task_manager.TaskManager, 'spawn_after', autospec=True)
    def test_vendor_passthru_shared_lock(self, mock_spawn, mock_upgrade):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        info = {'bar': 'woof'}
        self._start_service()

        response = self.service.vendor_passthru(self.context, node.uuid,
                                                'fourth_method_shared_lock',
                                                'POST', info)

        # Assert spawn_after was called
        self.assertTrue(mock_spawn.called)
        self.assertIsNone(response['return'])
        self.assertTrue(response['async'])

        # Assert lock was never upgraded to an exclusive one
        self.assertFalse(mock_upgrade.called)

        node.refresh()
        self.assertIsNone(node.last_error)
        # Verify there's no reservation on the node
        self.assertIsNone(node.reservation)

    def test_vendor_passthru_http_method_not_supported(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        self._start_service()

        # GET not supported by first_method
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.vendor_passthru,
                                self.context, node.uuid,
                                'second_method', 'GET', {})
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.InvalidParameterValue, exc.exc_info[0])

        node.refresh()
        self.assertIsNone(node.last_error)
        # Verify reservation has been cleared.
        self.assertIsNone(node.reservation)

    def test_vendor_passthru_node_already_locked(self):
        fake_reservation = 'test_reserv'
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          reservation=fake_reservation)
        info = {'bar': 'baz'}
        self._start_service()

        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.vendor_passthru,
                                self.context, node.uuid, 'second_method',
                                'POST', info)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NodeLocked, exc.exc_info[0])

        node.refresh()
        self.assertIsNone(node.last_error)
        # Verify the existing reservation is not broken.
        self.assertEqual(fake_reservation, node.reservation)

    def test_vendor_passthru_unsupported_method(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        info = {'bar': 'baz'}
        self._start_service()

        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.vendor_passthru,
                                self.context, node.uuid,
                                'unsupported_method', 'POST', info)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.InvalidParameterValue,
                         exc.exc_info[0])

        node.refresh()
        self.assertIsNone(node.last_error)
        # Verify reservation has been cleared.
        self.assertIsNone(node.reservation)

    def test_vendor_passthru_missing_method_parameters(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        info = {'invalid_param': 'whatever'}
        self._start_service()

        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.vendor_passthru,
                                self.context, node.uuid,
                                'second_method', 'POST', info)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.MissingParameterValue, exc.exc_info[0])

        node.refresh()
        self.assertIsNone(node.last_error)
        # Verify reservation has been cleared.
        self.assertIsNone(node.reservation)

    def test_vendor_passthru_worker_pool_full(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        info = {'bar': 'baz'}
        self._start_service()

        with mock.patch.object(self.service,
                               '_spawn_worker', autospec=True) as spawn_mock:
            spawn_mock.side_effect = exception.NoFreeConductorWorker()

            exc = self.assertRaises(messaging.rpc.ExpectedException,
                                    self.service.vendor_passthru,
                                    self.context, node.uuid,
                                    'second_method', 'POST', info)
            # Compare true exception hidden by @messaging.expected_exceptions
            self.assertEqual(exception.NoFreeConductorWorker, exc.exc_info[0])

            node.refresh()
            self.assertIsNone(node.last_error)
            # Verify reservation has been cleared.
            self.assertIsNone(node.reservation)

    @mock.patch.object(driver_factory, 'get_interface', autospec=True)
    def test_get_node_vendor_passthru_methods(self, mock_iface):
        fake_routes = {'test_method': {'async': True,
                                       'description': 'foo',
                                       'http_methods': ['POST'],
                                       'func': None}}
        mock_iface.return_value.vendor_routes = fake_routes
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        self._start_service()

        data = self.service.get_node_vendor_passthru_methods(self.context,
                                                             node.uuid)
        # The function reference should not be returned
        del fake_routes['test_method']['func']
        self.assertEqual(fake_routes, data)

    @mock.patch.object(driver_factory, 'get_interface', autospec=True)
    @mock.patch.object(manager.ConductorManager, '_spawn_worker',
                       autospec=True)
    def test_driver_vendor_passthru_sync(self, mock_spawn, mock_get_if):
        expected = {'foo': 'bar'}
        vendor_mock = mock.Mock(spec=drivers_base.VendorInterface)
        mock_get_if.return_value = vendor_mock
        driver_name = 'fake-hardware'
        test_method = mock.MagicMock(return_value=expected)
        vendor_mock.driver_routes = {
            'test_method': {'func': test_method,
                            'async': False,
                            'attach': False,
                            'http_methods': ['POST']}}
        self._start_service()
        # init_host() called _spawn_worker because of the heartbeat
        mock_spawn.reset_mock()
        # init_host() called get_interface during driver loading
        mock_get_if.reset_mock()

        vendor_args = {'test': 'arg'}
        response = self.service.driver_vendor_passthru(
            self.context, driver_name, 'test_method', 'POST', vendor_args)

        # Assert that the vendor interface has no custom
        # driver_vendor_passthru()
        self.assertFalse(hasattr(vendor_mock, 'driver_vendor_passthru'))
        self.assertEqual(expected, response['return'])
        self.assertFalse(response['async'])
        test_method.assert_called_once_with(self.context, **vendor_args)
        # No worker was spawned
        self.assertFalse(mock_spawn.called)
        mock_get_if.assert_called_once_with(mock.ANY, 'vendor', 'fake')

    @mock.patch.object(driver_factory, 'get_interface', autospec=True)
    @mock.patch.object(manager.ConductorManager, '_spawn_worker',
                       autospec=True)
    def test_driver_vendor_passthru_async(self, mock_spawn, mock_iface):
        test_method = mock.MagicMock()
        mock_iface.return_value.driver_routes = {
            'test_sync_method': {'func': test_method,
                                 'async': True,
                                 'attach': False,
                                 'http_methods': ['POST']}}
        self._start_service()
        # init_host() called _spawn_worker because of the heartbeat
        mock_spawn.reset_mock()

        vendor_args = {'test': 'arg'}
        response = self.service.driver_vendor_passthru(
            self.context, 'fake-hardware', 'test_sync_method', 'POST',
            vendor_args)

        self.assertIsNone(response['return'])
        self.assertTrue(response['async'])
        mock_spawn.assert_called_once_with(self.service, test_method,
                                           self.context, **vendor_args)

    @mock.patch.object(driver_factory, 'get_interface', autospec=True)
    def test_driver_vendor_passthru_http_method_not_supported(self,
                                                              mock_iface):
        mock_iface.return_value.driver_routes = {
            'test_method': {'func': mock.MagicMock(),
                            'async': True,
                            'http_methods': ['POST']}}
        self._start_service()
        # GET not supported by test_method
        exc = self.assertRaises(messaging.ExpectedException,
                                self.service.driver_vendor_passthru,
                                self.context, 'fake-hardware', 'test_method',
                                'GET', {})
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.InvalidParameterValue,
                         exc.exc_info[0])

    def test_driver_vendor_passthru_method_not_supported(self):
        # Test for when the vendor interface is set, but hasn't passed a
        # driver_passthru_mapping to MixinVendorInterface
        self._start_service()
        exc = self.assertRaises(messaging.ExpectedException,
                                self.service.driver_vendor_passthru,
                                self.context, 'fake-hardware', 'test_method',
                                'POST', {})
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.InvalidParameterValue,
                         exc.exc_info[0])

    def test_driver_vendor_passthru_driver_not_found(self):
        self._start_service()
        self.assertRaises(messaging.ExpectedException,
                          self.service.driver_vendor_passthru,
                          self.context, 'does_not_exist', 'test_method',
                          'POST', {})

    @mock.patch.object(driver_factory, 'default_interface', autospec=True)
    def test_driver_vendor_passthru_no_default_interface(self,
                                                         mock_def_iface):
        self._start_service()
        # NOTE(rloo): service.init_host() will call
        #             driver_factory.default_interface() and we want these to
        #             succeed, so we set the side effect *after* that call.
        mock_def_iface.reset_mock()
        mock_def_iface.side_effect = exception.NoValidDefaultForInterface('no')
        exc = self.assertRaises(messaging.ExpectedException,
                                self.service.driver_vendor_passthru,
                                self.context, 'fake-hardware', 'test_method',
                                'POST', {})
        mock_def_iface.assert_called_once_with(mock.ANY, 'vendor',
                                               driver_name='fake-hardware')
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NoValidDefaultForInterface,
                         exc.exc_info[0])

    @mock.patch.object(driver_factory, 'get_interface', autospec=True)
    def test_get_driver_vendor_passthru_methods(self, mock_get_if):
        vendor_mock = mock.Mock(spec=drivers_base.VendorInterface)
        mock_get_if.return_value = vendor_mock
        driver_name = 'fake-hardware'
        fake_routes = {'test_method': {'async': True,
                                       'description': 'foo',
                                       'http_methods': ['POST'],
                                       'func': None}}
        vendor_mock.driver_routes = fake_routes
        self._start_service()

        # init_host() will call get_interface
        mock_get_if.reset_mock()

        data = self.service.get_driver_vendor_passthru_methods(self.context,
                                                               driver_name)
        # The function reference should not be returned
        del fake_routes['test_method']['func']
        self.assertEqual(fake_routes, data)

        mock_get_if.assert_called_once_with(mock.ANY, 'vendor', 'fake')

    @mock.patch.object(driver_factory, 'default_interface', autospec=True)
    def test_get_driver_vendor_passthru_methods_no_default_interface(
            self, mock_def_iface):
        self._start_service()
        # NOTE(rloo): service.init_host() will call
        #             driver_factory.default_interface() and we want these to
        #             succeed, so we set the side effect *after* that call.
        mock_def_iface.reset_mock()
        mock_def_iface.side_effect = exception.NoValidDefaultForInterface('no')
        exc = self.assertRaises(
            messaging.rpc.ExpectedException,
            self.service.get_driver_vendor_passthru_methods,
            self.context, 'fake-hardware')
        mock_def_iface.assert_called_once_with(mock.ANY, 'vendor',
                                               driver_name='fake-hardware')
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NoValidDefaultForInterface,
                         exc.exc_info[0])

    @mock.patch.object(driver_factory, 'get_interface', autospec=True)
    def test_driver_vendor_passthru_validation_failed(self, mock_iface):
        mock_iface.return_value.driver_validate.side_effect = (
            exception.MissingParameterValue('error'))
        test_method = mock.Mock()
        mock_iface.return_value.driver_routes = {
            'test_method': {'func': test_method,
                            'async': False,
                            'http_methods': ['POST']}}
        self._start_service()
        exc = self.assertRaises(messaging.ExpectedException,
                                self.service.driver_vendor_passthru,
                                self.context, 'fake-hardware', 'test_method',
                                'POST', {})
        self.assertEqual(exception.MissingParameterValue,
                         exc.exc_info[0])
        self.assertFalse(test_method.called)


@mgr_utils.mock_record_keepalive
@mock.patch.object(images, 'is_whole_disk_image', autospec=True)
class ServiceDoNodeDeployTestCase(mgr_utils.ServiceSetUpMixin,
                                  db_base.DbTestCase):

    class fake_conductor:
        id = 1

    def test_do_node_deploy_invalid_state(self, mock_iwdi):
        mock_iwdi.return_value = False
        self._start_service()
        # test that node deploy fails if the node is already provisioned
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.ACTIVE,
            target_provision_state=states.NOSTATE)
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.do_node_deploy,
                                self.context, node['uuid'])
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.InvalidStateRequested, exc.exc_info[0])
        # This is a sync operation last_error should be None.
        self.assertIsNone(node.last_error)
        # Verify reservation has been cleared.
        self.assertIsNone(node.reservation)
        self.assertFalse(mock_iwdi.called)
        self.assertNotIn('is_whole_disk_image', node.driver_internal_info)

    def test_do_node_deploy_maintenance(self, mock_iwdi):
        mock_iwdi.return_value = False
        self._start_service()
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          maintenance=True)
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.do_node_deploy,
                                self.context, node['uuid'])
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NodeInMaintenance, exc.exc_info[0])
        # This is a sync operation last_error should be None.
        self.assertIsNone(node.last_error)
        # Verify reservation has been cleared.
        self.assertIsNone(node.reservation)
        self.assertFalse(mock_iwdi.called)

    def _test_do_node_deploy_validate_fail(self, mock_validate, mock_iwdi):
        mock_iwdi.return_value = False
        # InvalidParameterValue should be re-raised as InstanceDeployFailure
        mock_validate.side_effect = exception.InvalidParameterValue('error')
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        with mock.patch.object(self.service, '_concurrent_action_limit',
                               autospec=True) as mock_ca_limit:

            exc = self.assertRaises(messaging.rpc.ExpectedException,
                                    self.service.do_node_deploy,
                                    self.context, node.uuid)
            mock_ca_limit.assert_called_once_with(action='provisioning')
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.InstanceDeployFailure, exc.exc_info[0])
        self.assertEqual(exc.exc_info[1].code, 400)
        # Check the message of InstanceDeployFailure. In a
        # messaging.rpc.ExpectedException sys.exc_info() is stored in exc_info
        # in the exception object. So InstanceDeployFailure will be in
        # exc_info[1]
        self.assertIn(r'node 1be26c0b-03f2-4d2e-ae87-c02d7f33c123',
                      str(exc.exc_info[1]))
        node.refresh()
        # This is a sync operation last_error should be None.
        self.assertIsNone(node.last_error)
        # Verify reservation has been cleared.
        self.assertIsNone(node.reservation)
        mock_iwdi.assert_called_once_with(self.context, node.instance_info)
        # The image type must be set for validation to actually work
        self.assertFalse(node.driver_internal_info['is_whole_disk_image'])
        self.assertEqual('partition', node.instance_info['image_type'])

    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.validate',
                autospec=True)
    def test_do_node_deploy_validate_fail(self, mock_validate, mock_iwdi):
        self._test_do_node_deploy_validate_fail(mock_validate, mock_iwdi)

    @mock.patch('ironic.drivers.modules.fake.FakePower.validate',
                autospec=True)
    def test_do_node_deploy_power_validate_fail(self, mock_validate,
                                                mock_iwdi):
        self._test_do_node_deploy_validate_fail(mock_validate, mock_iwdi)

    @mock.patch.object(conductor_utils, 'validate_instance_info_traits',
                       autospec=True)
    def test_do_node_deploy_traits_validate_fail(self, mock_validate,
                                                 mock_iwdi):
        self._test_do_node_deploy_validate_fail(mock_validate, mock_iwdi)

    @mock.patch.object(conductor_steps,
                       'validate_user_deploy_steps_and_templates',
                       autospec=True)
    def test_do_node_deploy_validate_template_fail(self, mock_validate,
                                                   mock_iwdi):
        self._test_do_node_deploy_validate_fail(mock_validate, mock_iwdi)

    def test_do_node_deploy_partial_ok(self, mock_iwdi):
        mock_iwdi.return_value = False
        self._start_service()
        thread = self.service._spawn_worker(lambda: None)
        with mock.patch.object(self.service, '_spawn_worker',
                               autospec=True) as mock_spawn:
            mock_spawn.return_value = thread

            node = obj_utils.create_test_node(
                self.context,
                driver='fake-hardware',
                provision_state=states.AVAILABLE,
                driver_internal_info={'agent_url': 'url'})

            self.service.do_node_deploy(self.context, node.uuid)
            node.refresh()
            self.assertEqual(states.DEPLOYING, node.provision_state)
            self.assertEqual(states.ACTIVE, node.target_provision_state)
            # This is a sync operation last_error should be None.
            self.assertIsNone(node.last_error)
            # Verify reservation has been cleared.
            self.assertIsNone(node.reservation)
            mock_spawn.assert_called_once_with(mock.ANY, mock.ANY,
                                               mock.ANY, None, None)
            mock_iwdi.assert_called_once_with(self.context, node.instance_info)
            self.assertFalse(node.driver_internal_info['is_whole_disk_image'])

    def test_do_node_deploy_rebuild_active_state_error(self, mock_iwdi):
        # Tests manager.do_node_deploy() & deployments.do_next_deploy_step(),
        # when getting an unexpected state returned from a deploy_step.
        mock_iwdi.return_value = True
        self._start_service()
        # NOTE(rloo): We have to mock this here as opposed to using a
        # decorator. With a decorator, when initialization is done, the
        # mocked deploy() method isn't considered a deploy step. So we defer
        # mock'ing until after the init is done.
        with mock.patch.object(fake.FakeDeploy,
                               'deploy', autospec=True) as mock_deploy:
            mock_deploy.return_value = states.DEPLOYING
            node = obj_utils.create_test_node(
                self.context, driver='fake-hardware',
                provision_state=states.ACTIVE,
                target_provision_state=states.NOSTATE,
                instance_info={'image_source': uuidutils.generate_uuid(),
                               'kernel': 'aaaa', 'ramdisk': 'bbbb'},
                driver_internal_info={'is_whole_disk_image': False})

            self.service.do_node_deploy(self.context, node.uuid, rebuild=True)
            node.refresh()
            self.assertEqual(states.DEPLOYFAIL, node.provision_state)
            self.assertEqual(states.ACTIVE, node.target_provision_state)
            self.assertIsNotNone(node.last_error)
            # Verify reservation has been cleared.
            self.assertIsNone(node.reservation)
            mock_deploy.assert_called_once_with(mock.ANY, mock.ANY)
            # Verify instance_info values have been cleared.
            self.assertNotIn('kernel', node.instance_info)
            self.assertNotIn('ramdisk', node.instance_info)
            mock_iwdi.assert_called_once_with(self.context, node.instance_info)
            # Verify is_whole_disk_image reflects correct value on rebuild.
            self.assertTrue(node.driver_internal_info['is_whole_disk_image'])
            self.assertIsNone(node.driver_internal_info['deploy_steps'])

    def test_do_node_deploy_rebuild_boot_iso_cleared(self, mock_iwdi):
        # Tests that boot_iso and original_image_source are cleared
        # on rebuild, so the driver re-resolves them from Glance.
        mock_iwdi.return_value = True
        self._start_service()
        with mock.patch.object(fake.FakeDeploy,
                               'deploy', autospec=True) as mock_deploy:
            mock_deploy.return_value = states.DEPLOYING
            node = obj_utils.create_test_node(
                self.context, driver='fake-hardware',
                provision_state=states.ACTIVE,
                target_provision_state=states.NOSTATE,
                instance_info={
                    'image_source': uuidutils.generate_uuid(),
                    'boot_iso': 'glance://boot-iso-uuid',
                    'original_image_source': 'glance://old-image',
                },
                driver_internal_info={'is_whole_disk_image': False})

            self.service.do_node_deploy(
                self.context, node.uuid, rebuild=True)
            node.refresh()
            # Verify boot_iso and original_image_source are cleared.
            self.assertNotIn('boot_iso', node.instance_info)
            self.assertNotIn(
                'original_image_source', node.instance_info)

    def test_do_node_deploy_rebuild_active_state_waiting(self, mock_iwdi):
        mock_iwdi.return_value = False
        self._start_service()
        # NOTE(rloo): We have to mock this here as opposed to using a
        # decorator. With a decorator, when initialization is done, the
        # mocked deploy() method isn't considered a deploy step. So we defer
        # mock'ing until after the init is done.
        with mock.patch.object(fake.FakeDeploy,
                               'deploy', autospec=True) as mock_deploy:
            mock_deploy.return_value = states.DEPLOYWAIT
            node = obj_utils.create_test_node(
                self.context, driver='fake-hardware',
                provision_state=states.ACTIVE,
                target_provision_state=states.NOSTATE,
                instance_info={'image_source': uuidutils.generate_uuid()})

            self.service.do_node_deploy(self.context, node.uuid, rebuild=True)
            node.refresh()
            self.assertEqual(states.DEPLOYWAIT, node.provision_state)
            self.assertEqual(states.ACTIVE, node.target_provision_state)
            # last_error should be None.
            self.assertIsNone(node.last_error)
            # Verify reservation has been cleared.
            self.assertIsNone(node.reservation)
            mock_deploy.assert_called_once_with(mock.ANY, mock.ANY)
            mock_iwdi.assert_called_once_with(self.context, node.instance_info)
            self.assertFalse(node.driver_internal_info['is_whole_disk_image'])
            self.assertEqual(1, len(node.driver_internal_info['deploy_steps']))

    def test_do_node_deploy_rebuild_active_state_done(self, mock_iwdi):
        mock_iwdi.return_value = False
        self._start_service()
        # NOTE(rloo): We have to mock this here as opposed to using a
        # decorator. With a decorator, when initialization is done, the
        # mocked deploy() method isn't considered a deploy step. So we defer
        # mock'ing until after the init is done.
        with mock.patch.object(fake.FakeDeploy,
                               'deploy', autospec=True) as mock_deploy:
            mock_deploy.return_value = None
            node = obj_utils.create_test_node(
                self.context, driver='fake-hardware',
                provision_state=states.ACTIVE,
                target_provision_state=states.NOSTATE)

            self.service.do_node_deploy(self.context, node.uuid, rebuild=True)
            node.refresh()
            self.assertEqual(states.ACTIVE, node.provision_state)
            self.assertEqual(states.NOSTATE, node.target_provision_state)
            # last_error should be None.
            self.assertIsNone(node.last_error)
            # Verify reservation has been cleared.
            self.assertIsNone(node.reservation)
            mock_deploy.assert_called_once_with(mock.ANY, mock.ANY)
            mock_iwdi.assert_called_once_with(self.context, node.instance_info)
            self.assertFalse(node.driver_internal_info['is_whole_disk_image'])
            self.assertIsNone(node.driver_internal_info['deploy_steps'])

    def test_do_node_deploy_rebuild_deployfail_state(self, mock_iwdi):
        mock_iwdi.return_value = False
        self._start_service()
        # NOTE(rloo): We have to mock this here as opposed to using a
        # decorator. With a decorator, when initialization is done, the
        # mocked deploy() method isn't considered a deploy step. So we defer
        # mock'ing until after the init is done.
        with mock.patch.object(fake.FakeDeploy,
                               'deploy', autospec=True) as mock_deploy:
            mock_deploy.return_value = None
            node = obj_utils.create_test_node(
                self.context, driver='fake-hardware',
                provision_state=states.DEPLOYFAIL,
                target_provision_state=states.NOSTATE)

            self.service.do_node_deploy(self.context, node.uuid, rebuild=True)
            node.refresh()
            self.assertEqual(states.ACTIVE, node.provision_state)
            self.assertEqual(states.NOSTATE, node.target_provision_state)
            # last_error should be None.
            self.assertIsNone(node.last_error)
            # Verify reservation has been cleared.
            self.assertIsNone(node.reservation)
            mock_deploy.assert_called_once_with(mock.ANY, mock.ANY)
            mock_iwdi.assert_called_once_with(self.context, node.instance_info)
            self.assertFalse(node.driver_internal_info['is_whole_disk_image'])
            self.assertIsNone(node.driver_internal_info['deploy_steps'])

    def test_do_node_deploy_rebuild_error_state(self, mock_iwdi):
        mock_iwdi.return_value = False
        self._start_service()
        # NOTE(rloo): We have to mock this here as opposed to using a
        # decorator. With a decorator, when initialization is done, the
        # mocked deploy() method isn't considered a deploy step. So we defer
        # mock'ing until after the init is done.
        with mock.patch.object(fake.FakeDeploy,
                               'deploy', autospec=True) as mock_deploy:
            mock_deploy.return_value = None
            node = obj_utils.create_test_node(
                self.context, driver='fake-hardware',
                provision_state=states.ERROR,
                target_provision_state=states.NOSTATE)

            self.service.do_node_deploy(self.context, node.uuid, rebuild=True)
            node.refresh()
            self.assertEqual(states.ACTIVE, node.provision_state)
            self.assertEqual(states.NOSTATE, node.target_provision_state)
            # last_error should be None.
            self.assertIsNone(node.last_error)
            # Verify reservation has been cleared.
            self.assertIsNone(node.reservation)
            mock_deploy.assert_called_once_with(mock.ANY, mock.ANY)
            mock_iwdi.assert_called_once_with(self.context, node.instance_info)
            self.assertFalse(node.driver_internal_info['is_whole_disk_image'])
            self.assertIsNone(node.driver_internal_info['deploy_steps'])

    def test_do_node_deploy_rebuild_from_available_state(self, mock_iwdi):
        mock_iwdi.return_value = False
        # test node will not rebuild if state is AVAILABLE
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          provision_state=states.AVAILABLE)
        with mock.patch.object(self.service, '_concurrent_action_limit',
                               autospec=True) as mock_ca_limit:
            exc = self.assertRaises(messaging.rpc.ExpectedException,
                                    self.service.do_node_deploy,
                                    self.context, node['uuid'], rebuild=True)
            mock_ca_limit.assert_called_once_with(action='provisioning')
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.InvalidStateRequested, exc.exc_info[0])
        # Last_error should be None.
        self.assertIsNone(node.last_error)
        # Verify reservation has been cleared.
        self.assertIsNone(node.reservation)
        self.assertFalse(mock_iwdi.called)
        self.assertNotIn('is_whole_disk_image', node.driver_internal_info)

    def test_do_node_deploy_rebuild_protected(self, mock_iwdi):
        mock_iwdi.return_value = False
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          provision_state=states.ACTIVE,
                                          protected=True)
        with mock.patch.object(self.service, '_concurrent_action_limit',
                               autospec=True) as mock_ca_limit:
            exc = self.assertRaises(messaging.rpc.ExpectedException,
                                    self.service.do_node_deploy,
                                    self.context, node['uuid'], rebuild=True)
            mock_ca_limit.assert_called_once_with(action='provisioning')
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NodeProtected, exc.exc_info[0])
        # Last_error should be None.
        self.assertIsNone(node.last_error)
        # Verify reservation has been cleared.
        self.assertIsNone(node.reservation)
        self.assertFalse(mock_iwdi.called)

    def test_do_node_deploy_worker_pool_full(self, mock_iwdi):
        mock_iwdi.return_value = False
        prv_state = states.AVAILABLE
        tgt_prv_state = states.NOSTATE
        node = obj_utils.create_test_node(self.context,
                                          provision_state=prv_state,
                                          target_provision_state=tgt_prv_state,
                                          last_error=None,
                                          driver='fake-hardware')
        self._start_service()

        with mock.patch.object(self.service, '_spawn_worker',
                               autospec=True) as mock_spawn:
            mock_spawn.side_effect = exception.NoFreeConductorWorker()

            with mock.patch.object(self.service, '_concurrent_action_limit',
                                   autospec=True) as mock_ca_limit:
                exc = self.assertRaises(messaging.rpc.ExpectedException,
                                        self.service.do_node_deploy,
                                        self.context, node.uuid)
                mock_ca_limit.assert_called_once_with(action='provisioning')
            # Compare true exception hidden by @messaging.expected_exceptions
            self.assertEqual(exception.NoFreeConductorWorker, exc.exc_info[0])
            node.refresh()
            # Make sure things were rolled back
            self.assertEqual(prv_state, node.provision_state)
            self.assertEqual(tgt_prv_state, node.target_provision_state)
            self.assertIsNotNone(node.last_error)
            # Verify reservation has been cleared.
            self.assertIsNone(node.reservation)
            mock_iwdi.assert_called_once_with(self.context, node.instance_info)
            self.assertFalse(node.driver_internal_info['is_whole_disk_image'])

    @mock.patch.object(deployments, 'start_deploy', autospec=True)
    def test_do_node_deploy_disallowed_step_raises(self, mock_start,
                                                   mock_iwdi):
        """Disallowed deploy step raises before state transition."""
        self.config(disallow_deploy_steps=['bios.factory_reset'], group='api')
        mock_iwdi.return_value = False
        self._start_service()
        deploy_steps = [{'interface': 'bios', 'step': 'factory_reset',
                         'priority': 95}]
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.AVAILABLE,
            target_provision_state=states.NOSTATE)
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.do_node_deploy,
                                self.context, node.uuid,
                                deploy_steps=deploy_steps)
        self.assertEqual(exception.StepNotAllowed, exc.exc_info[0])
        # start_deploy must NOT have been called
        self.assertFalse(mock_start.called)
        node.refresh()
        self.assertEqual(states.AVAILABLE, node.provision_state)

    @mock.patch.object(deployments, 'start_deploy', autospec=True)
    def test_do_node_deploy_disallowed_step_raises_send_raw(
            self, mock_start, mock_iwdi):
        mock_iwdi.return_value = False
        self._start_service()
        deploy_steps = [
            {'step': 'send_raw',
             'priority': 7,
             'interface': 'vendor'}
        ]
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.AVAILABLE,
            target_provision_state=states.NOSTATE)
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.do_node_deploy,
                                self.context, node.uuid,
                                deploy_steps=deploy_steps)
        self.assertEqual(exception.StepNotAllowed, exc.exc_info[0])
        # start_deploy must NOT have been called
        self.assertFalse(mock_start.called)
        node.refresh()
        self.assertEqual(states.AVAILABLE, node.provision_state)

    @mock.patch.object(deployments, 'start_deploy', autospec=True)
    def test_do_node_deploy_allowed_step_proceeds(self, mock_start,
                                                  mock_iwdi):
        """Allowed deploy step proceeds normally."""
        self.config(disallow_deploy_steps=['raid.create_configuration'],
                    group='api')
        mock_iwdi.return_value = False
        self._start_service()
        deploy_steps = [{'interface': 'bios', 'step': 'factory_reset',
                         'priority': 95}]
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.AVAILABLE,
            target_provision_state=states.NOSTATE)
        self.service.do_node_deploy(self.context, node.uuid,
                                    deploy_steps=deploy_steps)
        mock_start.assert_called_once_with(
            mock.ANY, mock.ANY, None, event='deploy',
            deploy_steps=deploy_steps)


@mgr_utils.mock_record_keepalive
class ContinueNodeDeployTestCase(mgr_utils.ServiceSetUpMixin,
                                 db_base.DbTestCase):
    def setUp(self):
        super(ContinueNodeDeployTestCase, self).setUp()
        self.deploy_start = {
            'step': 'deploy_start', 'priority': 50, 'interface': 'deploy'}
        self.deploy_end = {
            'step': 'deploy_end', 'priority': 20, 'interface': 'deploy'}
        self.deploy_steps = [self.deploy_start, self.deploy_end]

    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_continue_node_deploy_worker_pool_full(self, mock_spawn):
        # Test the appropriate exception is raised if the worker pool is full
        prv_state = states.DEPLOYWAIT
        tgt_prv_state = states.ACTIVE
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          provision_state=prv_state,
                                          target_provision_state=tgt_prv_state,
                                          last_error=None)
        self._start_service()

        mock_spawn.side_effect = exception.NoFreeConductorWorker()

        self.assertRaises(exception.NoFreeConductorWorker,
                          self.service.continue_node_deploy,
                          self.context, node.uuid)

    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_continue_node_deploy_wrong_state(self, mock_spawn):
        # Test the appropriate exception is raised if node isn't already
        # in DEPLOYWAIT state
        prv_state = states.DEPLOYFAIL
        tgt_prv_state = states.ACTIVE
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          provision_state=prv_state,
                                          target_provision_state=tgt_prv_state,
                                          last_error=None)
        self._start_service()

        self.assertRaises(exception.InvalidStateRequested,
                          self.service.continue_node_deploy,
                          self.context, node.uuid)

        node.refresh()
        # Make sure node wasn't modified
        self.assertEqual(prv_state, node.provision_state)
        self.assertEqual(tgt_prv_state, node.target_provision_state)
        # Verify reservation has been cleared.
        self.assertIsNone(node.reservation)

    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_continue_node_deploy(self, mock_spawn):
        # test a node can continue deploying via RPC
        prv_state = states.DEPLOYWAIT
        tgt_prv_state = states.ACTIVE
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          provision_state=prv_state,
                                          target_provision_state=tgt_prv_state,
                                          last_error=None,
                                          deploy_step=self.deploy_steps[0])
        self._start_service()
        self.service.continue_node_deploy(self.context, node.uuid)
        node.refresh()
        self.assertEqual(states.DEPLOYING, node.provision_state)
        self.assertEqual(tgt_prv_state, node.target_provision_state)
        mock_spawn.assert_called_with(mock.ANY,
                                      deployments.continue_node_deploy,
                                      mock.ANY)

    @mock.patch.object(tenacity, 'stop_after_attempt',
                       return_value=tenacity.stop_after_attempt(4),
                       autospec=True)
    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_continue_node_deploy_locked(self, mock_spawn, mock_stop):
        """Test that continuing a deploy via RPC cannot fail due to locks."""
        max_attempts = 3
        prv_state = states.DEPLOYWAIT
        tgt_prv_state = states.ACTIVE
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          provision_state=prv_state,
                                          target_provision_state=tgt_prv_state,
                                          last_error=None,
                                          deploy_step=self.deploy_steps[0])
        self._start_service()
        with mock.patch.object(objects.Node, 'reserve', autospec=True) as mck:
            mck.side_effect = (
                ([exception.NodeLocked(node='foo', host='foo')] * max_attempts)
                + [node])
            self.service.continue_node_deploy(self.context, node.uuid)

    @mock.patch.object(task_manager.TaskManager, 'process_event',
                       autospec=True)
    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_continue_node_deploy_deprecated(self, mock_spawn, mock_event):
        # TODO(rloo): delete this when we remove support for handling
        # deploy steps; node will always be in DEPLOYWAIT then.

        # test a node can continue deploying via RPC
        prv_state = states.DEPLOYING
        tgt_prv_state = states.ACTIVE
        driver_info = {'deploy_steps': self.deploy_steps,
                       'deploy_step_index': 0,
                       'steps_validated': True}
        self._start_service()
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          provision_state=prv_state,
                                          target_provision_state=tgt_prv_state,
                                          last_error=None,
                                          driver_internal_info=driver_info,
                                          deploy_step=self.deploy_steps[0])
        self.service.continue_node_deploy(self.context, node.uuid)
        node.refresh()
        self.assertEqual(states.DEPLOYING, node.provision_state)
        self.assertEqual(tgt_prv_state, node.target_provision_state)
        mock_spawn.assert_called_with(mock.ANY,
                                      deployments.continue_node_deploy,
                                      mock.ANY)
        self.assertFalse(mock_event.called)

    def _continue_node_deploy_abort(self):
        last_deploy_step = self.deploy_steps[0]
        last_deploy_step['abortable'] = False
        last_deploy_step['abort_after'] = True
        driver_info = {'deploy_steps': self.deploy_steps,
                       'deploy_step_index': 0,
                       'steps_validated': True}
        tgt_prov_state = states.ACTIVE
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.DEPLOYWAIT,
            target_provision_state=tgt_prov_state, last_error=None,
            driver_internal_info=driver_info, deploy_step=self.deploy_steps[0])

        self._start_service()
        self.service.continue_node_deploy(self.context, node.uuid)
        node.refresh()
        self.assertEqual(states.DEPLOYFAIL, node.provision_state)
        self.assertEqual(tgt_prov_state, node.target_provision_state)
        self.assertIsNotNone(node.last_error)
        # assert the deploy step name is in the last error message
        self.assertIn(self.deploy_steps[0]['step'], node.last_error)

    def test_continue_node_deploy_abort(self):
        self._continue_node_deploy_abort()

    def _continue_node_deploy_abort_last_deploy_step(self):
        last_deploy_step = self.deploy_steps[0]
        last_deploy_step['abortable'] = False
        last_deploy_step['abort_after'] = True
        driver_info = {'deploy_steps': [self.deploy_steps[0]],
                       'deploy_step_index': 0,
                       'steps_validated': True}
        tgt_prov_state = states.ACTIVE
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.DEPLOYWAIT,
            target_provision_state=tgt_prov_state, last_error=None,
            driver_internal_info=driver_info, deploy_step=self.deploy_steps[0])

        self._start_service()
        self.service.continue_node_deploy(self.context, node.uuid)
        node.refresh()
        self.assertEqual(states.DEPLOYFAIL, node.provision_state)
        self.assertEqual(tgt_prov_state, node.target_provision_state)
        self.assertIsNotNone(node.last_error)
        self.assertIn(self.deploy_steps[0]['step'], node.last_error)

    def test_continue_node_deploy_abort_last_deploy_step(self):
        self._continue_node_deploy_abort_last_deploy_step()


@mgr_utils.mock_record_keepalive
class CheckTimeoutsTestCase(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase):
    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.clean_up',
                autospec=True)
    def test__check_deploy_timeouts(self, mock_cleanup):
        self._start_service()
        CONF.set_override('deploy_callback_timeout', 1, group='conductor')
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.DEPLOYWAIT,
            target_provision_state=states.ACTIVE,
            provision_updated_at=datetime.datetime(2000, 1, 1, 0, 0))

        self.service._check_deploy_timeouts(self.context)
        node.refresh()
        self.assertEqual(states.DEPLOYFAIL, node.provision_state)
        self.assertEqual(states.ACTIVE, node.target_provision_state)
        self.assertIsNotNone(node.last_error)
        mock_cleanup.assert_called_once_with(mock.ANY, mock.ANY)

    def _check_cleanwait_timeouts(self, manual=False, with_step=True):
        self._start_service()
        CONF.set_override('clean_callback_timeout', 1, group='conductor')
        tgt_prov_state = states.MANAGEABLE if manual else states.AVAILABLE
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.CLEANWAIT,
            target_provision_state=tgt_prov_state,
            provision_updated_at=datetime.datetime(2000, 1, 1, 0, 0),
            clean_step={
                'interface': 'deploy',
                'step': 'erase_devices'} if with_step else {},
            driver_internal_info={
                'cleaning_reboot': manual,
                'clean_step_index': 0})

        self.service._check_cleanwait_timeouts(self.context)
        node.refresh()
        self.assertEqual(states.CLEANFAIL, node.provision_state)
        self.assertEqual(tgt_prov_state, node.target_provision_state)
        self.assertIsNotNone(node.last_error)
        self.assertEqual(with_step, node.maintenance)
        self.assertEqual(faults.CLEAN_FAILURE if with_step else None,
                         node.fault)
        # Test that cleaning parameters have been purged in order
        # to prevent looping of the cleaning sequence
        self.assertEqual({}, node.clean_step)
        self.assertNotIn('clean_step_index', node.driver_internal_info)
        self.assertNotIn('cleaning_reboot', node.driver_internal_info)

    def test__check_cleanwait_timeouts_automated_clean(self):
        self._check_cleanwait_timeouts()

    def test__check_cleanwait_timeouts_manual_clean(self):
        self._check_cleanwait_timeouts(manual=True)

    def test__check_cleanwait_timeouts_boot_timeout(self):
        self._check_cleanwait_timeouts(with_step=False)

    @mock.patch('ironic.drivers.modules.fake.FakeRescue.clean_up',
                autospec=True)
    @mock.patch.object(conductor_utils, 'node_power_action', autospec=True)
    def test_check_rescuewait_timeouts(self, node_power_mock,
                                       mock_clean_up):
        self._start_service()
        CONF.set_override('rescue_callback_timeout', 1, group='conductor')
        tgt_prov_state = states.RESCUE
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            rescue_interface='fake',
            network_interface='flat',
            provision_state=states.RESCUEWAIT,
            target_provision_state=tgt_prov_state,
            provision_updated_at=datetime.datetime(2000, 1, 1, 0, 0))

        self.service._check_rescuewait_timeouts(self.context)
        node.refresh()
        self.assertEqual(states.RESCUEFAIL, node.provision_state)
        self.assertEqual(tgt_prov_state, node.target_provision_state)
        self.assertIsNotNone(node.last_error)
        self.assertIn('Timeout reached while waiting for rescue ramdisk',
                      node.last_error)
        mock_clean_up.assert_called_once_with(mock.ANY, mock.ANY)
        node_power_mock.assert_called_once_with(mock.ANY, states.POWER_OFF)

    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.tear_down_service',
                autospec=True)
    @mock.patch.object(conductor_utils, 'node_power_action', autospec=True)
    def test_check_servicewait_timeouts(self, node_power_mock, mock_clean_up):
        self._start_service()
        CONF.set_override('service_callback_timeout', 1, group='conductor')
        tgt_prov_state = states.RESCUE
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            network_interface='flat',
            provision_state=states.SERVICEWAIT,
            target_provision_state=tgt_prov_state,
            provision_updated_at=datetime.datetime(2000, 1, 1, 0, 0))

        self.service._check_servicewait_timeouts(self.context)
        node.refresh()
        self.assertEqual(states.SERVICEFAIL, node.provision_state)
        self.assertEqual(tgt_prov_state, node.target_provision_state)
        self.assertIsNotNone(node.last_error)
        self.assertIn('Timeout reached while servicing the node',
                      node.last_error)
        mock_clean_up.assert_called_once_with(mock.ANY, mock.ANY)
        node_power_mock.assert_not_called()


@mgr_utils.mock_record_keepalive
class DoNodeTearDownTestCase(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase):
    def test_do_node_tear_down_invalid_state(self):
        self._start_service()
        # test node.provision_state is incorrect for tear_down
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          provision_state=states.AVAILABLE)
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.do_node_tear_down,
                                self.context, node['uuid'])
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.InvalidStateRequested, exc.exc_info[0])

    def test_do_node_tear_down_protected(self):
        self._start_service()
        # test node.provision_state is incorrect for tear_down
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          provision_state=states.ACTIVE,
                                          protected=True)
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.do_node_tear_down,
                                self.context, node['uuid'])
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NodeProtected, exc.exc_info[0])

    @mock.patch('ironic.drivers.modules.fake.FakePower.validate',
                autospec=True)
    def test_do_node_tear_down_validate_fail(self, mock_validate):
        self._start_service()
        # InvalidParameterValue should be re-raised as InstanceDeployFailure
        mock_validate.side_effect = exception.InvalidParameterValue('error')
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.ACTIVE,
            target_provision_state=states.NOSTATE)
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.do_node_tear_down,
                                self.context, node.uuid)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.InstanceDeployFailure, exc.exc_info[0])

    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.tear_down',
                autospec=True)
    def test_do_node_tear_down_driver_raises_error(self, mock_tear_down):
        # test when driver.deploy.tear_down raises exception
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.DELETING,
            target_provision_state=states.AVAILABLE,
            instance_info={'foo': 'bar'},
            driver_internal_info={'is_whole_disk_image': False,
                                  'is_source_a_path': None})

        task = task_manager.TaskManager(self.context, node.uuid)
        self._start_service()
        mock_tear_down.side_effect = exception.InstanceDeployFailure('test')
        self.assertRaises(exception.InstanceDeployFailure,
                          self.service._do_node_tear_down, task,
                          node.provision_state)
        node.refresh()
        self.assertEqual(states.ERROR, node.provision_state)
        self.assertEqual(states.NOSTATE, node.target_provision_state)
        self.assertIsNotNone(node.last_error)
        # Assert instance_info was erased
        self.assertEqual({}, node.instance_info)
        mock_tear_down.assert_called_once_with(mock.ANY, task)

    @mock.patch('ironic.drivers.modules.fake.FakeConsole.stop_console',
                autospec=True)
    def test_do_node_tear_down_console_raises_error(self, mock_console):
        # test when _set_console_mode raises exception
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.DELETING,
            target_provision_state=states.AVAILABLE,
            instance_info={'foo': 'bar'},
            console_enabled=True,
            driver_internal_info={'is_whole_disk_image': False})

        task = task_manager.TaskManager(self.context, node.uuid)
        self._start_service()
        mock_console.side_effect = exception.ConsoleError('test')
        self.assertRaises(exception.ConsoleError,
                          self.service._do_node_tear_down, task,
                          node.provision_state)
        node.refresh()
        self.assertEqual(states.ERROR, node.provision_state)
        self.assertEqual(states.NOSTATE, node.target_provision_state)
        self.assertIsNotNone(node.last_error)
        # Assert instance_info was erased
        self.assertEqual({}, node.instance_info)
        mock_console.assert_called_once_with(mock.ANY, task)

    # TODO(TheJulia): Since we're functionally bound to neutron support
    # by default, the fake drivers still invoke neutron.
    @mock.patch('ironic.drivers.modules.fake.FakeConsole.stop_console',
                autospec=True)
    @mock.patch('ironic.common.neutron.unbind_neutron_port', autospec=True)
    @mock.patch('ironic.conductor.cleaning.do_node_clean', autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.tear_down',
                autospec=True)
    def _test__do_node_tear_down_ok(self, mock_tear_down, mock_clean,
                                    mock_unbind, mock_console,
                                    enabled_console=False,
                                    with_allocation=False,
                                    source_a_path=False):
        # test when driver.deploy.tear_down succeeds
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.DELETING,
            target_provision_state=states.AVAILABLE,
            instance_uuid=(uuidutils.generate_uuid()
                           if not with_allocation else None),
            instance_info={'foo': 'bar'},
            console_enabled=enabled_console,
            driver_internal_info={'is_whole_disk_image': False,
                                  'deploy_steps': {},
                                  'root_uuid_or_disk_id': 'foo',
                                  'instance': {'ephemeral_gb': 10},
                                  'automatic_lessee': True},
            lessee='fooproject')
        port = obj_utils.create_test_port(
            self.context, node_id=node.id,
            internal_info={n_common.NetType.TENANT.vif_key: 'foo'})
        if with_allocation:
            alloc = obj_utils.create_test_allocation(self.context)
            # Establish cross-linking between the node and the allocation
            alloc.node_id = node.id
            alloc.save()
            node.refresh()
        if source_a_path:
            d_ii = node.driver_internal_info
            d_ii['is_source_a_path'] = True
            node.driver_internal_info = d_ii

        task = task_manager.TaskManager(self.context, node.uuid)
        self._start_service()
        self.service._do_node_tear_down(task, node.provision_state)
        node.refresh()
        port.refresh()
        # Node will be moved to AVAILABLE after cleaning, not tested here
        self.assertEqual(states.CLEANING, node.provision_state)
        self.assertEqual(states.AVAILABLE, node.target_provision_state)
        self.assertIsNone(node.last_error)
        self.assertIsNone(node.instance_uuid)
        self.assertIsNone(node.allocation_id)
        self.assertIsNone(node.lessee)
        self.assertIsNone(node.instance_name)
        self.assertEqual({}, node.instance_info)
        self.assertNotIn('instance', node.driver_internal_info)
        self.assertIsNone(node.driver_internal_info['deploy_steps'])
        self.assertNotIn('root_uuid_or_disk_id', node.driver_internal_info)
        self.assertNotIn('is_whole_disk_image', node.driver_internal_info)
        self.assertNotIn('automatic_lessee', node.driver_internal_info)
        self.assertNotIn('is_source_a_path', node.driver_internal_info)
        mock_tear_down.assert_called_once_with(task.driver.deploy, task)
        mock_clean.assert_called_once_with(task, mock.ANY, mock.ANY, mock.ANY)
        self.assertEqual({}, port.internal_info)
        mock_unbind.assert_called_once_with('foo', context=mock.ANY)
        if enabled_console:
            mock_console.assert_called_once_with(task.driver.console, task)
        else:
            self.assertFalse(mock_console.called)
        if with_allocation:
            self.assertRaises(exception.AllocationNotFound,
                              objects.Allocation.get_by_id,
                              self.context, alloc.id)

    def test__do_node_tear_down_ok_without_console(self):
        self._test__do_node_tear_down_ok(enabled_console=False)

    def test__do_node_tear_down_ok_with_console(self):
        self._test__do_node_tear_down_ok(enabled_console=True)

    def test__do_node_tear_down_with_allocation(self):
        self._test__do_node_tear_down_ok(with_allocation=True)

    def test__do_node_tear_down_with_source_path(self):
        self._test__do_node_tear_down_ok(source_a_path=True)

    @mock.patch('ironic.drivers.modules.fake.FakeRescue.clean_up',
                autospec=True)
    @mock.patch('ironic.conductor.cleaning.do_node_clean', autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.tear_down',
                autospec=True)
    def _test_do_node_tear_down_from_state(self, init_state, is_rescue_state,
                                           mock_tear_down, mock_clean,
                                           mock_rescue_clean):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            uuid=uuidutils.generate_uuid(),
            provision_state=init_state,
            target_provision_state=states.AVAILABLE,
            driver_internal_info={'is_whole_disk_image': False})

        self._start_service()
        self.service.do_node_tear_down(self.context, node.uuid)
        node.refresh()
        # Node will be moved to AVAILABLE after cleaning, not tested here
        self.assertEqual(states.CLEANING, node.provision_state)
        self.assertEqual(states.AVAILABLE, node.target_provision_state)
        self.assertIsNone(node.last_error)
        self.assertEqual({}, node.instance_info)
        mock_tear_down.assert_called_once_with(mock.ANY, mock.ANY)
        mock_clean.assert_called_once_with(mock.ANY, mock.ANY, mock.ANY,
                                           mock.ANY)
        if is_rescue_state:
            mock_rescue_clean.assert_called_once_with(mock.ANY, mock.ANY)
        else:
            self.assertFalse(mock_rescue_clean.called)

    def test__do_node_tear_down_from_valid_states(self):
        valid_states = [states.ACTIVE, states.DEPLOYWAIT, states.DEPLOYFAIL,
                        states.ERROR]
        for state in valid_states:
            self._test_do_node_tear_down_from_state(state, False)

        valid_rescue_states = [states.RESCUEWAIT, states.RESCUE,
                               states.UNRESCUEFAIL, states.RESCUEFAIL]
        for state in valid_rescue_states:
            self._test_do_node_tear_down_from_state(state, True)

    # NOTE(tenbrae): partial tear-down was broken. A node left in a state of
    #                DELETING could not have tear_down called on it a second
    #                time Thus, I have removed the unit test, which faultily
    #                asserted only that a node could be left in a state of
    #                incomplete deletion -- not that such a node's deletion
    #                could later be completed.

    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_do_node_tear_down_worker_pool_full(self, mock_spawn):
        prv_state = states.ACTIVE
        tgt_prv_state = states.NOSTATE
        fake_instance_info = {'foo': 'bar'}
        driver_internal_info = {'is_whole_disk_image': False}
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware', provision_state=prv_state,
            target_provision_state=tgt_prv_state,
            instance_info=fake_instance_info,
            driver_internal_info=driver_internal_info, last_error=None)
        self._start_service()

        mock_spawn.side_effect = exception.NoFreeConductorWorker()

        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.do_node_tear_down,
                                self.context, node.uuid)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NoFreeConductorWorker, exc.exc_info[0])
        node.refresh()
        # Assert instance_info/driver_internal_info was not touched
        self.assertEqual(fake_instance_info, node.instance_info)
        self.assertEqual(driver_internal_info, node.driver_internal_info)
        # Make sure things were rolled back
        self.assertEqual(prv_state, node.provision_state)
        self.assertEqual(tgt_prv_state, node.target_provision_state)
        self.assertIsNotNone(node.last_error)
        # Verify reservation has been cleared.
        self.assertIsNone(node.reservation)

    @mock.patch('ironic.common.network.remove_vifs_from_node',
                autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.tear_down',
                autospec=True)
    def test__do_node_tear_remove_vif_failed(self, mock_tear_down,
                                             mock_remove):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.DELETING,
            target_provision_state=states.AVAILABLE,
            instance_uuid=uuidutils.generate_uuid(),
            instance_info={'foo': 'bar'},
            driver_internal_info={'is_whole_disk_image': False,
                                  'deploy_steps': {},
                                  'instance': {'ephemeral_gb': 10}})
        port = obj_utils.create_test_port(
            self.context, node_id=node.id,
            internal_info={n_common.NetType.TENANT.vif_key: 'foo'})

        mock_remove.side_effect = exception.IronicException('Oops')
        task = task_manager.TaskManager(self.context, node.uuid)
        self._start_service()
        self.assertRaises(exception.IronicException,
                          self.service._do_node_tear_down,
                          task, node.provision_state)
        node.refresh()
        port.refresh()
        self.assertEqual(states.ERROR, node.provision_state)
        mock_tear_down.assert_called_once_with(task.driver.deploy, task)

    @mock.patch('ironic.conductor.cleaning.do_node_clean', autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.tear_down',
                autospec=True)
    @mock.patch.object(
        manager.ConductorManager, '_get_steps_for_automated_cleaning',
        autospec=True)
    def test__do_node_tear_down_filters_disallowed_automated_steps(
            self, mock_get_steps, mock_tear_down, mock_clean):
        """Automated cleaning silently filters disallowed steps."""
        self.config(disallow_clean_steps=['deploy.erase_disks'], group='api')
        runbook_steps = [
            {'interface': 'deploy', 'step': 'erase_disks',
             'priority': 20, 'args': {}},
            {'interface': 'power', 'step': 'update_firmware',
             'priority': 10, 'args': {}},
        ]
        mock_get_steps.return_value = (runbook_steps, False)
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.DELETING,
            target_provision_state=states.AVAILABLE,
            instance_info={'foo': 'bar'},
            driver_internal_info={'is_whole_disk_image': False,
                                  'deploy_steps': {},
                                  'instance': {'ephemeral_gb': 10}})
        task = task_manager.TaskManager(self.context, node.uuid)
        self._start_service()
        self.service._do_node_tear_down(task, node.provision_state)
        # clean should be called with the filtered steps
        mock_clean.assert_called_once_with(
            task,
            clean_steps=[runbook_steps[1]],
            disable_ramdisk=False,
            automated_with_steps=True)


@mgr_utils.mock_record_keepalive
class DoProvisioningActionTestCase(mgr_utils.ServiceSetUpMixin,
                                   db_base.DbTestCase):
    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_do_provisioning_action_worker_pool_full(self, mock_spawn):
        prv_state = states.MANAGEABLE
        tgt_prv_state = states.NOSTATE
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          provision_state=prv_state,
                                          target_provision_state=tgt_prv_state,
                                          last_error=None)
        self._start_service()

        mock_spawn.side_effect = exception.NoFreeConductorWorker()

        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.do_provisioning_action,
                                self.context, node.uuid, 'provide')
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NoFreeConductorWorker, exc.exc_info[0])
        node.refresh()
        # Make sure things were rolled back
        self.assertEqual(prv_state, node.provision_state)
        self.assertEqual(tgt_prv_state, node.target_provision_state)
        self.assertIsNotNone(node.last_error)
        # Verify reservation has been cleared.
        self.assertIsNone(node.reservation)

    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_do_provision_action_provide(self, mock_spawn):
        # test when a node is cleaned going from manageable to available
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.MANAGEABLE,
            target_provision_state=states.AVAILABLE)

        self._start_service()
        self.service.do_provisioning_action(self.context, node.uuid, 'provide')
        node.refresh()
        # Node will be moved to AVAILABLE after cleaning, not tested here
        self.assertEqual(states.CLEANING, node.provision_state)
        self.assertEqual(states.AVAILABLE, node.target_provision_state)
        self.assertIsNone(node.last_error)
        mock_spawn.assert_called_with(self.service,
                                      cleaning.do_node_clean,
                                      mock.ANY, mock.ANY, mock.ANY, mock.ANY)

    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_do_provision_action_provide_in_maintenance(self, mock_spawn):
        CONF.set_override('allow_provisioning_in_maintenance', False,
                          group='conductor')
        # test when a node is cleaned going from manageable to available
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.MANAGEABLE,
            target_provision_state=None,
            maintenance=True)

        self._start_service()
        mock_spawn.reset_mock()
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.do_provisioning_action,
                                self.context, node.uuid, 'provide')
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NodeInMaintenance, exc.exc_info[0])
        node.refresh()
        self.assertEqual(states.MANAGEABLE, node.provision_state)
        self.assertIsNone(node.target_provision_state)
        self.assertIsNone(node.last_error)
        self.assertFalse(mock_spawn.called)

    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_do_provision_action_manage(self, mock_spawn):
        # test when a node is verified going from enroll to manageable
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.ENROLL,
            target_provision_state=states.MANAGEABLE)

        self._start_service()
        self.service.do_provisioning_action(self.context, node.uuid, 'manage')
        node.refresh()
        # Node will be moved to MANAGEABLE after verification, not tested here
        self.assertEqual(states.VERIFYING, node.provision_state)
        self.assertEqual(states.MANAGEABLE, node.target_provision_state)
        self.assertIsNone(node.last_error)
        mock_spawn.assert_called_with(self.service,
                                      verify.do_node_verify, mock.ANY)

    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def _do_provision_action_abort(self, mock_spawn, manual=False):
        tgt_prov_state = states.MANAGEABLE if manual else states.AVAILABLE
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.CLEANWAIT,
            target_provision_state=tgt_prov_state)

        self._start_service()
        self.service.do_provisioning_action(self.context, node.uuid, 'abort')
        node.refresh()
        # Node will be moved to tgt_prov_state after cleaning, not tested here
        self.assertEqual(states.CLEANFAIL, node.provision_state)
        self.assertEqual(tgt_prov_state, node.target_provision_state)
        self.assertEqual('By request, the clean operation was aborted',
                         node.last_error)
        mock_spawn.assert_called_with(
            self.service, cleaning.do_node_clean_abort, mock.ANY)

    def test_do_provision_action_abort_automated_clean(self):
        self._do_provision_action_abort()

    def test_do_provision_action_abort_manual_clean(self):
        self._do_provision_action_abort(manual=True)

    def test_do_provision_action_abort_clean_step_not_abortable(self):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.CLEANWAIT,
            target_provision_state=states.AVAILABLE,
            clean_step={'step': 'foo', 'abortable': False})

        self._start_service()
        self.service.do_provisioning_action(self.context, node.uuid, 'abort')
        node.refresh()
        # Assert the current clean step was marked to be aborted later
        self.assertIn('abort_after', node.clean_step)
        self.assertTrue(node.clean_step['abort_after'])
        # Make sure things stays as it was before
        self.assertEqual(states.CLEANWAIT, node.provision_state)
        self.assertEqual(states.AVAILABLE, node.target_provision_state)

    def test_do_provision_action_abort_service_step_not_abortable_wait(self):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.SERVICEWAIT,
            target_provision_state=states.ACTIVE,
            service_step={'step': 'foo', 'abortable': False})

        self._start_service()
        self.service.do_provisioning_action(self.context, node.uuid, 'abort')
        node.refresh()
        # Assert the current service step was marked to be aborted later
        self.assertIn('abort_after', node.service_step)
        self.assertTrue(node.service_step['abort_after'])
        # Make sure things stays as it was before
        self.assertEqual(states.SERVICEWAIT, node.provision_state)
        self.assertEqual(states.ACTIVE, node.target_provision_state)

    def test_do_provision_action_abort_service_step_not_abortable_hold(self):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.SERVICEHOLD,
            target_provision_state=states.ACTIVE,
            service_step={'step': 'foo', 'abortable': False})

        self._start_service()
        self.service.do_provisioning_action(self.context, node.uuid, 'abort')
        node.refresh()
        # Assert the current service step was marked to be aborted later
        self.assertIn('abort_after', node.service_step)
        self.assertTrue(node.service_step['abort_after'])
        # Make sure things stays as it was before
        self.assertEqual(states.SERVICEHOLD, node.provision_state)
        self.assertEqual(states.ACTIVE, node.target_provision_state)

    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_do_provision_action_abort_service_step_abortable_wait(
            self, mock_spawn):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.SERVICEWAIT,
            target_provision_state=states.ACTIVE,
            service_step={'step': 'foo', 'abortable': True})

        self._start_service()
        self.service.do_provisioning_action(self.context, node.uuid, 'abort')
        node.refresh()
        # Assert the abort was initiated immediately
        mock_spawn.assert_called_with(
            self.service, servicing.do_node_service_abort, mock.ANY)

    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_do_provision_action_abort_service_step_abortable_hold(
            self, mock_spawn):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.SERVICEHOLD,
            target_provision_state=states.ACTIVE,
            service_step={'step': 'foo', 'abortable': True})

        self._start_service()
        self.service.do_provisioning_action(self.context, node.uuid, 'abort')
        node.refresh()
        # Assert the abort was initiated immediately
        mock_spawn.assert_called_with(
            self.service, servicing.do_node_service_abort, mock.ANY)

    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def _do_provision_action_abort_from_cleanhold(self, mock_spawn,
                                                  manual=False):
        tgt_prov_state = states.MANAGEABLE if manual else states.AVAILABLE
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.CLEANHOLD,
            target_provision_state=tgt_prov_state)

        self._start_service()
        self.service.do_provisioning_action(self.context, node.uuid, 'abort')
        node.refresh()
        # Node will be moved to tgt_prov_state after cleaning, not tested here
        self.assertEqual(states.CLEANFAIL, node.provision_state)
        self.assertEqual(tgt_prov_state, node.target_provision_state)
        self.assertEqual('By request, the clean operation was aborted',
                         node.last_error)
        mock_spawn.assert_called_with(
            self.service, cleaning.do_node_clean_abort, mock.ANY)

    def test_do_provision_action_abort_cleanhold_automated_clean(self):
        self._do_provision_action_abort_from_cleanhold()

    def test_do_provision_action_abort_cleanhold_manual_clean(self):
        self._do_provision_action_abort_from_cleanhold(manual=True)

    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_do_provision_action_abort_from_deployhold(self, mock_spawn):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.DEPLOYHOLD,
            driver_internal_info={
                'agent_url': 'https://foo.bar/'
            })

        self._start_service()
        self.service.do_provisioning_action(self.context, node.uuid, 'abort')
        node.refresh()
        mock_spawn.assert_called_with(
            self.service,
            self.service._do_node_tear_down,
            mock.ANY,
            states.DEPLOYHOLD)
        self.assertNotIn('agent_url', node.driver_internal_info)

    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_do_provision_action_abort_from_deploywait(self, mock_spawn):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.DEPLOYWAIT,
            driver_internal_info={
                'agent_url': 'https://foo.bar/'
            })

        self._start_service()
        self.service.do_provisioning_action(self.context, node.uuid, 'abort')
        node.refresh()
        mock_spawn.assert_called_with(
            self.service,
            deployments.do_node_deploy_abort,
            mock.ANY)
        self.assertNotIn('agent_url', node.driver_internal_info)

    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_do_provision_action_abort_from_deploywait_step_not_abortable(
            self, mock_spawn):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.DEPLOYWAIT,
            target_provision_state=states.ACTIVE,
            deploy_step={'step': 'foo', 'abortable': False})

        self._start_service(start_allocations=False)
        self.service.do_provisioning_action(self.context, node.uuid, 'abort')
        node.refresh()
        # Assert the current deploy step was marked to be aborted later
        self.assertIn('abort_after', node.deploy_step)
        self.assertTrue(node.deploy_step['abort_after'])
        # Make sure things stays as it was before
        self.assertEqual(states.DEPLOYWAIT, node.provision_state)
        self.assertEqual(states.ACTIVE, node.target_provision_state)
        # Make sure spawn worker was not called
        mock_spawn.assert_not_called()

    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_do_provision_action_abort_from_servicefail(self, mock_spawn):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.SERVICEFAIL,
            target_provision_state=states.ACTIVE)

        self._start_service()
        self.service.do_provisioning_action(self.context, node.uuid, 'abort')
        mock_spawn.assert_called_with(
            self.service,
            servicing.do_node_service_abort,
            mock.ANY)

    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_do_provision_action_abort_from_servicefail_maintenance(
            self, mock_spawn):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.SERVICEFAIL,
            target_provision_state=states.ACTIVE,
            maintenance=True)

        self._start_service()
        self.assertRaises(
            exception.InvalidState,
            self.service.do_provisioning_action,
            self.context, node.uuid, 'abort')


@mgr_utils.mock_record_keepalive
class DoNodeCleanTestCase(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase):
    def setUp(self):
        super(DoNodeCleanTestCase, self).setUp()
        self.config(automated_clean=True, group='conductor')
        self.power_update = {
            'step': 'update_firmware', 'priority': 10, 'interface': 'power'}
        self.deploy_update = {
            'step': 'update_firmware', 'priority': 10, 'interface': 'deploy'}
        self.deploy_erase = {
            'step': 'erase_disks', 'priority': 20, 'interface': 'deploy'}
        # Automated cleaning should be executed in this order
        self.clean_steps = [self.deploy_erase, self.power_update,
                            self.deploy_update]
        self.next_clean_step_index = 1
        # Manual clean step
        self.deploy_raid = {
            'step': 'build_raid', 'priority': 0, 'interface': 'deploy'}

    @mock.patch('ironic.drivers.modules.fake.FakePower.validate',
                autospec=True)
    def test_do_node_clean_maintenance(self, mock_validate):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.MANAGEABLE,
            target_provision_state=states.NOSTATE,
            maintenance=True, maintenance_reason='reason')
        self._start_service()
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.do_node_clean,
                                self.context, node.uuid, [])
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NodeInMaintenance, exc.exc_info[0])
        self.assertFalse(mock_validate.called)

    @mock.patch('ironic.conductor.task_manager.TaskManager.process_event',
                autospec=True)
    def _test_do_node_clean_validate_fail(self, mock_validate, mock_process):
        mock_validate.side_effect = exception.InvalidParameterValue('error')
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.MANAGEABLE,
            target_provision_state=states.NOSTATE)
        self._start_service()
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.do_node_clean,
                                self.context, node.uuid, [])
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.InvalidParameterValue, exc.exc_info[0])
        mock_validate.assert_called_once_with(mock.ANY, mock.ANY)
        self.assertFalse(mock_process.called)

    @mock.patch('ironic.drivers.modules.fake.FakePower.validate',
                autospec=True)
    def test_do_node_clean_power_validate_fail(self, mock_validate):
        self._test_do_node_clean_validate_fail(mock_validate)

    @mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
                autospec=True)
    def test_do_node_clean_network_validate_fail(self, mock_validate):
        self._test_do_node_clean_validate_fail(mock_validate)

    @mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
                autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakePower.validate',
                autospec=True)
    def test_do_node_clean_invalid_state(self, mock_power_valid,
                                         mock_network_valid):
        # test node.provision_state is incorrect for clean
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.ENROLL,
            target_provision_state=states.NOSTATE)
        self._start_service()
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.do_node_clean,
                                self.context, node.uuid, [])
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.InvalidStateRequested, exc.exc_info[0])
        mock_power_valid.assert_called_once_with(mock.ANY, mock.ANY)
        mock_network_valid.assert_called_once_with(mock.ANY, mock.ANY)
        node.refresh()
        self.assertNotIn('clean_steps', node.driver_internal_info)

    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    @mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
                autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakePower.validate',
                autospec=True)
    def test_do_node_clean_ok(self, mock_power_valid, mock_network_valid,
                              mock_spawn):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.MANAGEABLE,
            target_provision_state=states.NOSTATE, last_error='old error')
        self._start_service()
        clean_steps = [self.deploy_raid]
        self.service.do_node_clean(self.context, node.uuid, clean_steps)
        mock_power_valid.assert_called_once_with(mock.ANY, mock.ANY)
        mock_network_valid.assert_called_once_with(mock.ANY, mock.ANY)
        mock_spawn.assert_called_with(
            self.service, cleaning.do_node_clean, mock.ANY, clean_steps,
            False, False)
        node.refresh()
        # Node will be moved to CLEANING
        self.assertEqual(states.CLEANING, node.provision_state)
        self.assertEqual(states.MANAGEABLE, node.target_provision_state)
        self.assertNotIn('clean_steps', node.driver_internal_info)
        self.assertIsNone(node.last_error)

    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    @mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
                autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakePower.validate',
                autospec=True)
    def test_do_node_clean_worker_pool_full(self, mock_power_valid,
                                            mock_network_valid, mock_spawn):
        prv_state = states.MANAGEABLE
        tgt_prv_state = states.NOSTATE
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware', provision_state=prv_state,
            target_provision_state=tgt_prv_state)
        self._start_service()
        clean_steps = [self.deploy_raid]
        mock_spawn.side_effect = exception.NoFreeConductorWorker()
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.do_node_clean,
                                self.context, node.uuid, clean_steps)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NoFreeConductorWorker, exc.exc_info[0])
        mock_power_valid.assert_called_once_with(mock.ANY, mock.ANY)
        mock_network_valid.assert_called_once_with(mock.ANY, mock.ANY)
        mock_spawn.assert_called_with(
            self.service, cleaning.do_node_clean, mock.ANY, clean_steps,
            False, False)
        node.refresh()
        # Make sure states were rolled back
        self.assertEqual(prv_state, node.provision_state)
        self.assertEqual(tgt_prv_state, node.target_provision_state)

        self.assertIsNotNone(node.last_error)
        self.assertIsNone(node.reservation)

    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_continue_node_clean_worker_pool_full(self, mock_spawn):
        # Test the appropriate exception is raised if the worker pool is full
        prv_state = states.CLEANWAIT
        tgt_prv_state = states.AVAILABLE
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          provision_state=prv_state,
                                          target_provision_state=tgt_prv_state,
                                          last_error=None)
        self._start_service()

        mock_spawn.side_effect = exception.NoFreeConductorWorker()

        self.assertRaises(exception.NoFreeConductorWorker,
                          self.service.continue_node_clean,
                          self.context, node.uuid)

    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_continue_node_clean_wrong_state(self, mock_spawn):
        # Test the appropriate exception is raised if node isn't already
        # in CLEANWAIT state
        prv_state = states.ACTIVE
        tgt_prv_state = states.AVAILABLE
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          provision_state=prv_state,
                                          target_provision_state=tgt_prv_state,
                                          last_error=None)
        self._start_service()

        self.assertRaises(exception.InvalidStateRequested,
                          self.service.continue_node_clean,
                          self.context, node.uuid)

        node.refresh()
        # Make sure things were rolled back
        self.assertEqual(prv_state, node.provision_state)
        self.assertEqual(tgt_prv_state, node.target_provision_state)
        # Verify reservation has been cleared.
        self.assertIsNone(node.reservation)

    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def _continue_node_clean(self, return_state, mock_spawn, manual=False):
        # test a node can continue cleaning via RPC
        prv_state = return_state
        tgt_prv_state = states.MANAGEABLE if manual else states.AVAILABLE
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          provision_state=prv_state,
                                          target_provision_state=tgt_prv_state,
                                          last_error=None,
                                          clean_step=self.clean_steps[0])
        self._start_service()
        self.service.continue_node_clean(self.context, node.uuid)
        node.refresh()
        self.assertEqual(states.CLEANING, node.provision_state)
        self.assertEqual(tgt_prv_state, node.target_provision_state)
        mock_spawn.assert_called_with(
            self.service, cleaning.continue_node_clean, mock.ANY)

    def test_continue_node_clean_automated(self):
        self._continue_node_clean(states.CLEANWAIT)

    def test_continue_node_clean_manual(self):
        self._continue_node_clean(states.CLEANWAIT, manual=True)

    def _continue_node_clean_abort(self, manual=False):
        last_clean_step = self.clean_steps[0]
        last_clean_step['abortable'] = False
        last_clean_step['abort_after'] = True
        driver_info = {'clean_steps': self.clean_steps,
                       'clean_step_index': 0}
        tgt_prov_state = states.MANAGEABLE if manual else states.AVAILABLE
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.CLEANWAIT,
            target_provision_state=tgt_prov_state, last_error=None,
            driver_internal_info=driver_info, clean_step=self.clean_steps[0])

        self._start_service()
        self.service.continue_node_clean(self.context, node.uuid)
        node.refresh()
        self.assertEqual(states.CLEANFAIL, node.provision_state)
        self.assertEqual(tgt_prov_state, node.target_provision_state)
        self.assertIsNotNone(node.last_error)
        # assert the clean step name is in the last error message
        self.assertIn(self.clean_steps[0]['step'], node.last_error)

    def test_continue_node_clean_automated_abort(self):
        self._continue_node_clean_abort()

    def test_continue_node_clean_manual_abort(self):
        self._continue_node_clean_abort(manual=True)

    def _continue_node_clean_abort_last_clean_step(self, manual=False):
        last_clean_step = self.clean_steps[0]
        last_clean_step['abortable'] = False
        last_clean_step['abort_after'] = True
        driver_info = {'clean_steps': [self.clean_steps[0]],
                       'clean_step_index': 0}
        tgt_prov_state = states.MANAGEABLE if manual else states.AVAILABLE
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.CLEANWAIT,
            target_provision_state=tgt_prov_state, last_error=None,
            driver_internal_info=driver_info, clean_step=self.clean_steps[0])

        self._start_service()
        self.service.continue_node_clean(self.context, node.uuid)
        node.refresh()
        self.assertEqual(states.CLEANFAIL, node.provision_state)
        self.assertEqual(tgt_prov_state, node.target_provision_state)
        self.assertIsNotNone(node.last_error)
        self.assertIn(self.clean_steps[0]['step'], node.last_error)

    def test_continue_node_clean_automated_abort_last_clean_step(self):
        self._continue_node_clean_abort_last_clean_step()

    def test_continue_node_clean_manual_abort_last_clean_step(self):
        self._continue_node_clean_abort_last_clean_step(manual=True)

    @mock.patch.object(tenacity, 'stop_after_attempt',
                       return_value=tenacity.stop_after_attempt(4),
                       autospec=True)
    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_continue_node_clean_locked(self, mock_spawn, mock_stop):
        """Test that continuing a clean via RPC cannot fail due to locks."""
        max_attempts = 3
        self.config(node_locked_retry_attempts=max_attempts, group='conductor')
        driver_info = {'clean_steps': [self.clean_steps[0]],
                       'clean_step_index': 0}
        tgt_prov_state = states.AVAILABLE
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.CLEANWAIT,
            target_provision_state=tgt_prov_state, last_error=None,
            driver_internal_info=driver_info, clean_step=self.clean_steps[0])
        self._start_service()
        self.service._correct_stuck_states()
        with mock.patch.object(objects.Node, 'reserve', autospec=True) as mck:
            mck.side_effect = (
                ([exception.NodeLocked(node='foo', host='foo')] * max_attempts)
                + [node])
            self.service.continue_node_clean(self.context, node.uuid)

    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_do_provision_action_unlocks_cleaning_manual(self, mock_spawn):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.CLEANHOLD,
            target_provision_state=states.MANAGEABLE,
            driver_internal_info={
                'clean_steps': [
                    {'step': 'hold', 'priority': 9, 'interface': 'power'},
                    {'step': 'update_firmware', 'priority': 10,
                     'interface': 'power'}
                ],
                'clean_step_index': 1
            }
        )
        self._start_service()
        self.service.do_provisioning_action(self.context, node.uuid, 'unhold')
        node.refresh()
        self.assertIsNone(node.last_error)
        self.assertEqual(states.CLEANWAIT, node.provision_state)
        self.service.continue_node_clean(self.context, node.uuid)
        node.refresh()
        self.assertIsNone(node.last_error)
        self.assertEqual(states.CLEANING, node.provision_state)

    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_do_provision_action_unlocks_cleaning_automated(self, mock_spawn):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.CLEANHOLD,
            # NOTE(TheJulia): This actually tests should this occur with
            # automated cleaning. While not an explicit feature, we don't
            # want things to go sideways in this case.
            target_provision_state=states.AVAILABLE,
            driver_internal_info={
                'clean_steps': [
                    {'step': 'hold', 'priority': 9, 'interface': 'power'},
                    {'step': 'update_firmware', 'priority': 10,
                     'interface': 'power'}
                ],
                'clean_step_index': 1
            }
        )
        self._start_service()
        self.service.do_provisioning_action(self.context, node.uuid, 'unhold')
        node.refresh()
        self.assertIsNone(node.last_error)
        self.assertEqual(states.CLEANWAIT, node.provision_state)
        self.service.continue_node_clean(self.context, node.uuid)
        node.refresh()
        self.assertIsNone(node.last_error)
        self.assertEqual(states.CLEANING, node.provision_state)

    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_do_provision_action_unlocks_deploying(self, mock_spawn):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.DEPLOYHOLD,
            target_provision_state=states.ACTIVE,
            driver_internal_info={
                'clean_steps': [
                    {'step': 'hold', 'priority': 9, 'interface': 'power'},
                    {'step': 'update_firmware', 'priority': 10,
                     'interface': 'power'}
                ],
                'deploy_step_index': 1
            }
        )
        self._start_service()
        self.service.do_provisioning_action(self.context, node.uuid, 'unhold')
        node.refresh()
        self.assertIsNone(node.last_error)
        self.assertEqual(states.DEPLOYWAIT, node.provision_state)
        self.service.continue_node_deploy(self.context, node.uuid)
        node.refresh()
        self.assertIsNone(node.last_error)
        self.assertEqual(states.DEPLOYING, node.provision_state)

    @mock.patch('ironic.conductor.task_manager.TaskManager.process_event',
                autospec=True)
    @mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
                autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakePower.validate',
                autospec=True)
    def test_do_node_clean_disallowed_step_raises(self, mock_power_valid,
                                                  mock_network_valid,
                                                  mock_process):
        """Disallowed manual clean step raises before state transition."""
        self.config(disallow_clean_steps=['deploy.build_raid'], group='api')
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.MANAGEABLE,
            target_provision_state=states.NOSTATE)
        self._start_service()
        clean_steps = [self.deploy_raid]
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.do_node_clean,
                                self.context, node.uuid, clean_steps)
        self.assertEqual(exception.StepNotAllowed, exc.exc_info[0])
        # process_event must NOT have been called
        self.assertFalse(mock_process.called)
        node.refresh()
        # Node stays in original state
        self.assertEqual(states.MANAGEABLE, node.provision_state)

    @mock.patch('ironic.conductor.task_manager.TaskManager.process_event',
                autospec=True)
    @mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
                autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakePower.validate',
                autospec=True)
    def test_do_node_clean_disallowed_step_raises_send_raw(
            self, mock_power_valid,
            mock_network_valid,
            mock_process):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.MANAGEABLE,
            target_provision_state=states.NOSTATE)
        self._start_service()
        clean_steps = [
            {'step': 'send_raw',
             'priority': 7,
             'interface': 'vendor'}
        ]
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.do_node_clean,
                                self.context, node.uuid, clean_steps)
        self.assertEqual(exception.StepNotAllowed, exc.exc_info[0])
        # process_event must NOT have been called
        self.assertFalse(mock_process.called)
        node.refresh()
        # Node stays in original state
        self.assertEqual(states.MANAGEABLE, node.provision_state)

    @mock.patch('ironic.conductor.task_manager.TaskManager.process_event',
                autospec=True)
    @mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
                autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakePower.validate',
                autospec=True)
    def test_do_node_clean_allowed_step_proceeds(self, mock_power_valid,
                                                 mock_network_valid,
                                                 mock_process):
        """Allowed clean step proceeds normally."""
        self.config(disallow_clean_steps=['raid.create_configuration'],
                    group='api')
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.MANAGEABLE,
            target_provision_state=states.NOSTATE)
        self._start_service()
        clean_steps = [self.deploy_raid]
        self.service.do_node_clean(self.context, node.uuid, clean_steps)
        mock_process.assert_called_once_with(
            mock.ANY,
            'clean',
            callback=mock.ANY,
            call_args=(cleaning.do_node_clean, mock.ANY,
                       clean_steps, False, False),
            err_handler=mock.ANY, target_state='manageable')


@mgr_utils.mock_record_keepalive
class DoNodeServiceTestCase(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase):
    def setUp(self):
        super(DoNodeServiceTestCase, self).setUp()
        self.config(group='conductor')
        self.power_update = {
            'step': 'update_firmware', 'priority': 10, 'interface': 'power'}
        self.deploy_update = {
            'step': 'update_firmware', 'priority': 10, 'interface': 'deploy'}
        self.deploy_magic = {
            'step': 'magic_firmware', 'priority': 20, 'interface': 'deploy'}
        self.next_clean_step_index = 1
        self.deploy_raid = {
            'step': 'build_raid', 'priority': 0, 'interface': 'deploy'}
        self.service_steps = [self.deploy_update,
                              self.power_update,
                              self.deploy_magic]

    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_continue_node_service_worker_pool_full(self, mock_spawn):
        # Test the appropriate exception is raised if the worker pool is full
        prv_state = states.SERVICEWAIT
        tgt_prv_state = states.AVAILABLE
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          provision_state=prv_state,
                                          target_provision_state=tgt_prv_state,
                                          last_error=None)
        self._start_service()

        mock_spawn.side_effect = exception.NoFreeConductorWorker()

        self.assertRaises(exception.NoFreeConductorWorker,
                          self.service.continue_node_service,
                          self.context, node.uuid)

    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_continue_node_service_wrong_state(self, mock_spawn):
        # Test the appropriate exception is raised if node isn't already
        # in SERVICEWAIT state
        prv_state = states.ACTIVE
        tgt_prv_state = states.AVAILABLE
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          provision_state=prv_state,
                                          target_provision_state=tgt_prv_state,
                                          last_error=None)
        self._start_service()

        self.assertRaises(exception.InvalidStateRequested,
                          self.service.continue_node_service,
                          self.context, node.uuid)

        node.refresh()
        # Make sure things were rolled back
        self.assertEqual(prv_state, node.provision_state)
        self.assertEqual(tgt_prv_state, node.target_provision_state)
        # Verify reservation has been cleared.
        self.assertIsNone(node.reservation)

    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def _continue_node_service(self, return_state, mock_spawn):
        # test a node can continue servicing via RPC
        prv_state = return_state
        tgt_prv_state = states.ACTIVE
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          provision_state=prv_state,
                                          target_provision_state=tgt_prv_state,
                                          last_error=None,
                                          service_step=self.service_steps[0])
        self._start_service()
        self.service.continue_node_service(self.context, node.uuid)
        node.refresh()
        self.assertEqual(states.SERVICING, node.provision_state)
        self.assertEqual(tgt_prv_state, node.target_provision_state)
        mock_spawn.assert_called_with(
            self.service, servicing.continue_node_service, mock.ANY)

    def test_continue_node_service(self):
        self._continue_node_service(states.SERVICEWAIT)

    def _continue_node_service_abort(self):
        last_service_step = self.service_steps[0]
        last_service_step['abortable'] = False
        last_service_step['abort_after'] = True
        driver_info = {'service_steps': self.service_steps,
                       'service_step_index': 0}
        tgt_prov_state = states.ACTIVE
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.SERVICEWAIT,
            target_provision_state=tgt_prov_state, last_error=None,
            driver_internal_info=driver_info,
            service_step=self.service_steps[0])

        self._start_service()
        self.service.continue_node_service(self.context, node.uuid)
        node.refresh()
        self.assertEqual(states.SERVICEFAIL, node.provision_state)
        self.assertEqual(tgt_prov_state, node.target_provision_state)
        self.assertIsNotNone(node.last_error)
        # assert the clean step name is in the last error message
        self.assertIn(self.service_steps[0]['step'], node.last_error)

    def test_continue_node_service_abort(self):
        self._continue_node_service_abort()

    def _continue_node_service_abort_last_service_step(self):
        last_service_step = self.service_steps[0]
        last_service_step['abortable'] = False
        last_service_step['abort_after'] = True
        driver_info = {'service_steps': [self.service_steps[0]],
                       'service_step_index': 0}
        tgt_prov_state = states.ACTIVE
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.SERVICEWAIT,
            target_provision_state=tgt_prov_state, last_error=None,
            driver_internal_info=driver_info,
            service_step=self.service_steps[0])

        self._start_service()
        self.service.continue_node_service(self.context, node.uuid)
        node.refresh()
        self.assertEqual(states.SERVICEFAIL, node.provision_state)
        self.assertEqual(tgt_prov_state, node.target_provision_state)
        self.assertIsNotNone(node.last_error)
        self.assertIn(self.service_steps[0]['step'], node.last_error)

    def test_continue_node_service_abort_last_service_step(self):
        self._continue_node_service_abort_last_service_step()

    @mock.patch.object(tenacity, 'stop_after_attempt',
                       return_value=tenacity.stop_after_attempt(4),
                       autospec=True)
    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_continue_node_service_locked(self, mock_spawn, mock_stop):
        """Test that continuing a service via RPC cannot fail due to locks."""
        max_attempts = 3
        self.config(node_locked_retry_attempts=max_attempts, group='conductor')
        driver_info = {'service_steps': [self.service_steps[0]],
                       'service_step_index': 0}
        tgt_prov_state = states.ACTIVE
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.SERVICEWAIT,
            target_provision_state=tgt_prov_state, last_error=None,
            driver_internal_info=driver_info,
            service_step=self.service_steps[0])
        self._start_service()
        with mock.patch.object(objects.Node, 'reserve', autospec=True) as mck:
            mck.side_effect = (
                ([exception.NodeLocked(node='foo', host='foo')] * max_attempts)
                + [node])
            self.service.continue_node_service(self.context, node.uuid)

    @mock.patch('ironic.drivers.modules.fake.FakePower.validate',
                autospec=True)
    def test_do_node_service_maintenance(self, mock_validate):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.ACTIVE,
            target_provision_state=states.NOSTATE,
            maintenance=True, maintenance_reason='reason')
        self._start_service()
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.do_node_service,
                                self.context, node.uuid, {'foo': 'bar'})
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NodeInMaintenance, exc.exc_info[0])
        self.assertFalse(mock_validate.called)

    @mock.patch.object(task_manager.TaskManager, 'process_event',
                       autospec=True)
    @mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
                autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakePower.validate',
                autospec=True)
    def test_do_node_service(self, mock_pv, mock_nv, mock_event):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.ACTIVE,
            target_provision_state=states.NOSTATE)
        self._start_service()
        self.service.do_node_service(self.context,
                                     node.uuid,
                                     [{'step': 'foo',
                                       'priority': 7,
                                       'interface': 'management'}])
        self.assertTrue(mock_pv.called)
        self.assertTrue(mock_nv.called)
        mock_event.assert_called_once_with(
            mock.ANY,
            'service',
            callback=mock.ANY,
            call_args=(servicing.do_node_service, mock.ANY,
                       [{'step': 'foo', 'priority': 7,
                         'interface': 'management'}], False),
            err_handler=mock.ANY, target_state='active')

    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_do_node_provision_action_unhold_service(self, mock_spawn):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.SERVICEHOLD,
            target_provision_state=states.ACTIVE,
            driver_internal_info={
                'service_steps': [
                    {'step': 'hold', 'priority': 9, 'interface': 'power'},
                ],
                'clean_step_index': 1
            }
        )
        self._start_service()
        self.service.do_provisioning_action(self.context, node.uuid, 'unhold')
        mock_spawn.assert_called_with(
            self.service, servicing.continue_node_service, mock.ANY)

    @mock.patch('ironic.conductor.task_manager.TaskManager.process_event',
                autospec=True)
    @mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
                autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakePower.validate',
                autospec=True)
    def test_do_node_service_disallowed_step_raises(self, mock_pv, mock_nv,
                                                    mock_event):
        """Disallowed service step raises before state transition."""
        self.config(disallow_service_steps=['deploy.update_firmware'],
                    group='api')
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.ACTIVE,
            target_provision_state=states.NOSTATE)
        self._start_service()
        service_steps = [self.deploy_update]
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.do_node_service,
                                self.context, node.uuid, service_steps)
        self.assertEqual(exception.StepNotAllowed, exc.exc_info[0])
        self.assertFalse(mock_event.called)
        node.refresh()
        self.assertEqual(states.ACTIVE, node.provision_state)

    @mock.patch('ironic.conductor.task_manager.TaskManager.process_event',
                autospec=True)
    @mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
                autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakePower.validate',
                autospec=True)
    def test_do_node_service_disallowed_step_raises_on_send_raw(
            self, mock_pv, mock_nv,
            mock_event):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.ACTIVE,
            target_provision_state=states.NOSTATE)
        self._start_service()
        service_steps = [
            {'step': 'send_raw',
             'priority': 7,
             'interface': 'vendor'}
        ]
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.do_node_service,
                                self.context, node.uuid, service_steps)
        self.assertEqual(exception.StepNotAllowed, exc.exc_info[0])
        self.assertFalse(mock_event.called)
        node.refresh()
        self.assertEqual(states.ACTIVE, node.provision_state)

    @mock.patch('ironic.conductor.task_manager.TaskManager.process_event',
                autospec=True)
    @mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
                autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakePower.validate',
                autospec=True)
    def test_do_node_service_allowed_step_proceeds(self, mock_pv, mock_nv,
                                                   mock_event):
        """Allowed service step proceeds normally."""
        self.config(disallow_service_steps=['raid.create_configuration'],
                    group='api')
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.ACTIVE,
            target_provision_state=states.NOSTATE)
        self._start_service()
        service_steps = [self.deploy_update]
        self.service.do_node_service(self.context, node.uuid, service_steps)
        mock_event.assert_called_once_with(
            mock.ANY,
            'service',
            callback=mock.ANY,
            call_args=(servicing.do_node_service, mock.ANY,
                       service_steps, False),
            err_handler=mock.ANY, target_state='active')


class DoNodeRescueTestCase(mgr_utils.CommonMixIn, mgr_utils.ServiceSetUpMixin,
                           db_base.DbTestCase):
    @mock.patch('ironic.conductor.task_manager.acquire', autospec=True)
    def test_do_node_rescue(self, mock_acquire):
        self._start_service()
        dii = {'agent_secret_token': 'token',
               'agent_url': 'http://url',
               'other field': 'value'}
        task = self._create_task(
            node_attrs=dict(driver='fake-hardware',
                            provision_state=states.ACTIVE,
                            instance_info={},
                            driver_internal_info=dii))
        mock_acquire.side_effect = self._get_acquire_side_effect(task)
        self.service.do_node_rescue(self.context, task.node.uuid,
                                    "password")
        task.process_event.assert_called_once_with(
            'rescue',
            callback=self.service._spawn_worker,
            call_args=(self.service._do_node_rescue, task),
            err_handler=conductor_utils.spawn_rescue_error_handler)
        self.assertIn('rescue_password', task.node.instance_info)
        self.assertIn('hashed_rescue_password', task.node.instance_info)
        task.node.del_driver_internal_info.assert_has_calls([
            mock.call('agent_secret_token'),
            mock.call('agent_secret_token_pregenerated'),
            mock.call('agent_url'),
            mock.call('agent_verify_ca')
        ])

    def test_do_node_rescue_invalid_state(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          network_interface='noop',
                                          provision_state=states.AVAILABLE,
                                          instance_info={})
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.do_node_rescue,
                                self.context, node.uuid, "password")
        node.refresh()
        self.assertNotIn('rescue_password', node.instance_info)
        self.assertNotIn('hashed_rescue_password', node.instance_info)
        self.assertEqual(exception.InvalidStateRequested, exc.exc_info[0])

    def _test_do_node_rescue_when_validate_fail(self, mock_validate):
        # InvalidParameterValue should be re-raised as InstanceRescueFailure
        mock_validate.side_effect = exception.InvalidParameterValue('error')
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.ACTIVE,
            target_provision_state=states.NOSTATE,
            instance_info={})
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.do_node_rescue,
                                self.context, node.uuid, "password")
        node.refresh()
        self.assertNotIn('hashed_rescue_password', node.instance_info)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.InstanceRescueFailure, exc.exc_info[0])

    @mock.patch('ironic.drivers.modules.fake.FakeRescue.validate',
                autospec=True)
    def test_do_node_rescue_when_rescue_validate_fail(self, mock_validate):
        self._test_do_node_rescue_when_validate_fail(mock_validate)

    @mock.patch('ironic.drivers.modules.fake.FakePower.validate',
                autospec=True)
    def test_do_node_rescue_when_power_validate_fail(self, mock_validate):
        self._test_do_node_rescue_when_validate_fail(mock_validate)

    @mock.patch('ironic.drivers.modules.network.flat.FlatNetwork.validate',
                autospec=True)
    def test_do_node_rescue_when_network_validate_fail(self, mock_validate):
        self._test_do_node_rescue_when_validate_fail(mock_validate)

    def test_do_node_rescue_maintenance(self):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            network_interface='noop',
            provision_state=states.ACTIVE,
            maintenance=True,
            target_provision_state=states.NOSTATE,
            instance_info={})
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.do_node_rescue,
                                self.context, node['uuid'], "password")
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NodeInMaintenance, exc.exc_info[0])
        # This is a sync operation last_error should be None.
        self.assertIsNone(node.last_error)

    @mock.patch('ironic.drivers.modules.fake.FakeRescue.rescue', autospec=True)
    def test__do_node_rescue_returns_rescuewait(self, mock_rescue):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.RESCUING,
            instance_info={'rescue_password': 'password',
                           'hashed_rescue_password': '1234'})
        with task_manager.TaskManager(self.context, node.uuid) as task:
            mock_rescue.return_value = states.RESCUEWAIT
            self.service._do_node_rescue(task)
            node.refresh()
            self.assertEqual(states.RESCUEWAIT, node.provision_state)
            self.assertEqual(states.RESCUE, node.target_provision_state)
            self.assertIn('rescue_password', node.instance_info)
            self.assertIn('hashed_rescue_password', node.instance_info)

    @mock.patch('ironic.drivers.modules.fake.FakeRescue.rescue', autospec=True)
    def test__do_node_rescue_returns_rescue(self, mock_rescue):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.RESCUING,
            instance_info={
                'rescue_password': 'password',
                'hashed_rescue_password': '1234'})
        with task_manager.TaskManager(self.context, node.uuid) as task:
            mock_rescue.return_value = states.RESCUE
            self.service._do_node_rescue(task)
            node.refresh()
            self.assertEqual(states.RESCUE, node.provision_state)
            self.assertEqual(states.NOSTATE, node.target_provision_state)
            self.assertIn('rescue_password', node.instance_info)
            self.assertIn('hashed_rescue_password', node.instance_info)

    @mock.patch.object(manager, 'LOG', autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakeRescue.rescue', autospec=True)
    def test__do_node_rescue_errors(self, mock_rescue, mock_log):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.RESCUING,
            instance_info={
                'rescue_password': 'password',
                'hashed_rescue_password': '1234'})
        mock_rescue.side_effect = exception.InstanceRescueFailure(
            'failed to rescue')
        with task_manager.TaskManager(self.context, node.uuid) as task:
            self.assertRaises(exception.InstanceRescueFailure,
                              self.service._do_node_rescue, task)
            node.refresh()
            self.assertEqual(states.RESCUEFAIL, node.provision_state)
            self.assertEqual(states.RESCUE, node.target_provision_state)
            self.assertNotIn('rescue_password', node.instance_info)
            self.assertNotIn('hashed_rescue_password', node.instance_info)
            self.assertTrue(node.last_error.startswith('Failed to rescue'))
            self.assertTrue(mock_log.error.called)

    @mock.patch.object(manager, 'LOG', autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakeRescue.rescue', autospec=True)
    def test__do_node_rescue_bad_state(self, mock_rescue, mock_log):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.RESCUING,
            instance_info={
                'rescue_password': 'password',
                'hashed_rescue_password': '1234'})
        mock_rescue.return_value = states.ACTIVE
        with task_manager.TaskManager(self.context, node.uuid) as task:
            self.service._do_node_rescue(task)
            node.refresh()
            self.assertEqual(states.RESCUEFAIL, node.provision_state)
            self.assertEqual(states.RESCUE, node.target_provision_state)
            self.assertNotIn('rescue_password', node.instance_info)
            self.assertNotIn('hashed_rescue_password', node.instance_info)
            self.assertTrue(node.last_error.startswith('Failed to rescue'))
            self.assertTrue(mock_log.error.called)

    @mock.patch('ironic.conductor.task_manager.acquire', autospec=True)
    def test_do_node_unrescue(self, mock_acquire):
        task = self._create_task(
            node_attrs=dict(driver='fake-hardware',
                            provision_state=states.RESCUE,
                            driver_internal_info={'agent_url': 'url'}))
        mock_acquire.side_effect = self._get_acquire_side_effect(task)
        self.service.do_node_unrescue(self.context, task.node.uuid)
        task.node.refresh()
        task.node.del_driver_internal_info.assert_has_calls([
            mock.call('agent_url')
        ])
        task.process_event.assert_called_once_with(
            'unrescue',
            callback=self.service._spawn_worker,
            call_args=(self.service._do_node_unrescue, task),
            err_handler=conductor_utils.provisioning_error_handler)

    def test_do_node_unrescue_invalid_state(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          provision_state=states.AVAILABLE)
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.do_node_unrescue,
                                self.context, node.uuid)
        self.assertEqual(exception.InvalidStateRequested, exc.exc_info[0])

    @mock.patch('ironic.drivers.modules.fake.FakePower.validate',
                autospec=True)
    def test_do_node_unrescue_validate_fail(self, mock_validate):
        # InvalidParameterValue should be re-raised as InstanceUnrescueFailure
        mock_validate.side_effect = exception.InvalidParameterValue('error')
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.RESCUE,
            target_provision_state=states.NOSTATE)
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.do_node_unrescue,
                                self.context, node.uuid)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.InstanceUnrescueFailure, exc.exc_info[0])

    def test_do_node_unrescue_maintenance(self):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.RESCUE,
            maintenance=True,
            target_provision_state=states.NOSTATE,
            instance_info={})
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.do_node_unrescue,
                                self.context, node.uuid)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NodeInMaintenance, exc.exc_info[0])
        # This is a sync operation last_error should be None.
        node.refresh()
        self.assertIsNone(node.last_error)

    @mock.patch('ironic.drivers.modules.fake.FakeRescue.unrescue',
                autospec=True)
    def test__do_node_unrescue(self, mock_unrescue):
        dii = {'agent_url': 'http://url',
               'agent_secret_token': 'token',
               'other field': 'value'}
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          provision_state=states.UNRESCUING,
                                          target_provision_state=states.ACTIVE,
                                          instance_info={},
                                          driver_internal_info=dii)
        with task_manager.TaskManager(self.context, node.uuid) as task:
            mock_unrescue.return_value = states.ACTIVE
            self.service._do_node_unrescue(task)
            node.refresh()
            self.assertEqual(states.ACTIVE, node.provision_state)
            self.assertEqual(states.NOSTATE, node.target_provision_state)
            self.assertEqual({'other field': 'value'},
                             node.driver_internal_info)

    @mock.patch.object(manager, 'LOG', autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakeRescue.unrescue',
                autospec=True)
    def test__do_node_unrescue_ironic_error(self, mock_unrescue, mock_log):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          provision_state=states.UNRESCUING,
                                          target_provision_state=states.ACTIVE,
                                          instance_info={})
        mock_unrescue.side_effect = exception.InstanceUnrescueFailure(
            'Unable to unrescue')
        with task_manager.TaskManager(self.context, node.uuid) as task:
            self.assertRaises(exception.InstanceUnrescueFailure,
                              self.service._do_node_unrescue, task)
            node.refresh()
            self.assertEqual(states.UNRESCUEFAIL, node.provision_state)
            self.assertEqual(states.ACTIVE, node.target_provision_state)
            self.assertIn('Unable to unrescue', node.last_error)
            self.assertTrue(mock_log.error.called)

    @mock.patch.object(manager, 'LOG', autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakeRescue.unrescue',
                autospec=True)
    def test__do_node_unrescue_other_error(self, mock_unrescue, mock_log):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          provision_state=states.UNRESCUING,
                                          target_provision_state=states.ACTIVE,
                                          instance_info={})
        mock_unrescue.side_effect = RuntimeError('Some failure')
        with task_manager.TaskManager(self.context, node.uuid) as task:
            self.assertRaises(RuntimeError,
                              self.service._do_node_unrescue, task)
            node.refresh()
            self.assertEqual(states.UNRESCUEFAIL, node.provision_state)
            self.assertEqual(states.ACTIVE, node.target_provision_state)
            self.assertIn('Some failure', node.last_error)
            self.assertTrue(mock_log.exception.called)

    @mock.patch('ironic.drivers.modules.fake.FakeRescue.unrescue',
                autospec=True)
    def test__do_node_unrescue_bad_state(self, mock_unrescue):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          provision_state=states.UNRESCUING,
                                          instance_info={})
        mock_unrescue.return_value = states.RESCUEWAIT
        with task_manager.TaskManager(self.context, node.uuid) as task:
            self.service._do_node_unrescue(task)
            node.refresh()
            self.assertEqual(states.UNRESCUEFAIL, node.provision_state)
            self.assertEqual(states.ACTIVE, node.target_provision_state)
            self.assertIn('Driver returned unexpected state', node.last_error)

    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_provision_rescue_abort(self, mock_spawn):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.RESCUEWAIT,
            target_provision_state=states.RESCUE,
            instance_info={'rescue_password': 'password'})
        self.service.do_provisioning_action(self.context, node.uuid, 'abort')
        node.refresh()
        self.assertEqual(states.RESCUEFAIL, node.provision_state)
        self.assertIsNone(node.last_error)
        self.assertNotIn('rescue_password', node.instance_info)
        mock_spawn.assert_called_with(
            self.service, self.service._do_node_rescue_abort, mock.ANY)

    @mock.patch.object(fake.FakeRescue, 'clean_up', autospec=True)
    def test__do_node_rescue_abort(self, clean_up_mock):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.RESCUEFAIL,
            target_provision_state=states.RESCUE,
            driver_internal_info={'agent_url': 'url'})
        with task_manager.acquire(self.context, node.uuid) as task:
            self.service._do_node_rescue_abort(task)
            clean_up_mock.assert_called_once_with(task.driver.rescue, task)
            self.assertIsNone(task.node.last_error)
            self.assertFalse(task.node.maintenance)
            self.assertNotIn('agent_url', task.node.driver_internal_info)

    @mock.patch.object(fake.FakeRescue, 'clean_up', autospec=True)
    def test__do_node_rescue_abort_clean_up_fail(self, clean_up_mock):
        clean_up_mock.side_effect = Exception('Surprise')
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.RESCUEFAIL)
        with task_manager.acquire(self.context, node.uuid) as task:
            self.service._do_node_rescue_abort(task)
            clean_up_mock.assert_called_once_with(task.driver.rescue, task)
            self.assertIsNotNone(task.node.last_error)
            self.assertIsNotNone(task.node.maintenance_reason)
            self.assertTrue(task.node.maintenance)
            self.assertEqual('rescue abort failure',
                             task.node.fault)


@mgr_utils.mock_record_keepalive
class MiscTestCase(mgr_utils.ServiceSetUpMixin, mgr_utils.CommonMixIn,
                   db_base.DbTestCase):
    def test__mapped_to_this_conductor(self):
        self._start_service()
        n = db_utils.get_test_node()
        self.assertTrue(self.service._mapped_to_this_conductor(
            n['uuid'], 'fake-hardware', ''))
        self.assertFalse(self.service._mapped_to_this_conductor(
            n['uuid'], 'fake-hardware', 'foogroup'))
        self.assertFalse(self.service._mapped_to_this_conductor(n['uuid'],
                                                                'otherdriver',
                                                                ''))

    @mock.patch.object(images, 'is_whole_disk_image', autospec=True)
    def test_validate_dynamic_driver_interfaces(self, mock_iwdi):
        mock_iwdi.return_value = False
        target_raid_config = {'logical_disks': [{'size_gb': 1,
                                                 'raid_level': '1'}]}
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            target_raid_config=target_raid_config,
            network_interface='noop')
        expected_info = dict(node.instance_info, image_type='partition')
        ret = self.service.validate_driver_interfaces(self.context,
                                                      node.uuid)
        expected = {'console': {'result': True},
                    'power': {'result': True},
                    'inspect': {'result': True},
                    'management': {'result': True},
                    'boot': {'result': True},
                    'raid': {'result': True},
                    'deploy': {'result': True},
                    'network': {'result': True},
                    'storage': {'result': True},
                    'rescue': {'result': True},
                    'bios': {'result': True},
                    'firmware': {'result': True}}
        self.assertEqual(expected, ret)
        mock_iwdi.assert_called_once_with(self.context, expected_info)

    @mock.patch.object(fake.FakeDeploy, 'validate', autospec=True)
    @mock.patch.object(images, 'is_whole_disk_image', autospec=True)
    def test_validate_driver_interfaces_validation_fail(self, mock_iwdi,
                                                        mock_val):
        mock_iwdi.return_value = False
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          network_interface='noop')
        reason = 'fake reason'
        mock_val.side_effect = exception.InvalidParameterValue(reason)
        expected_info = dict(node.instance_info, image_type='partition')
        ret = self.service.validate_driver_interfaces(self.context,
                                                      node.uuid)
        self.assertFalse(ret['deploy']['result'])
        self.assertEqual(reason, ret['deploy']['reason'])
        mock_iwdi.assert_called_once_with(self.context, expected_info)

    @mock.patch.object(fake.FakeDeploy, 'validate', autospec=True)
    @mock.patch.object(images, 'is_whole_disk_image', autospec=True)
    def test_validate_driver_interfaces_validation_fail_unexpected(
            self, mock_iwdi, mock_val):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        mock_val.side_effect = Exception('boom')
        expected_info = dict(node.instance_info, image_type='whole-disk')
        ret = self.service.validate_driver_interfaces(self.context,
                                                      node.uuid)
        reason = ('Unexpected exception, traceback saved '
                  'into log by ironic conductor service '
                  'that is running on test-host: boom')
        self.assertFalse(ret['deploy']['result'])
        self.assertEqual(reason, ret['deploy']['reason'])
        mock_iwdi.assert_called_once_with(self.context, expected_info)

    @mock.patch.object(images, 'is_whole_disk_image', autospec=True)
    def test_validate_driver_interfaces_validation_fail_instance_traits(
            self, mock_iwdi):
        mock_iwdi.return_value = False
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          network_interface='noop')
        expected_info = dict(node.instance_info, image_type='partition')
        with mock.patch(
                'ironic.conductor.utils.validate_instance_info_traits',
                autospec=True) as ii_traits:
            reason = 'fake reason'
            ii_traits.side_effect = exception.InvalidParameterValue(reason)
            ret = self.service.validate_driver_interfaces(self.context,
                                                          node.uuid)
            self.assertFalse(ret['deploy']['result'])
            self.assertEqual(reason, ret['deploy']['reason'])
        mock_iwdi.assert_called_once_with(self.context, expected_info)

    @mock.patch.object(images, 'is_whole_disk_image', autospec=True)
    def test_validate_driver_interfaces_validation_fail_deploy_templates(
            self, mock_iwdi):
        mock_iwdi.return_value = False
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          network_interface='noop')
        expected_info = dict(node.instance_info, image_type='partition')
        with mock.patch(
                'ironic.conductor.steps'
                '.validate_user_deploy_steps_and_templates',
                autospec=True) as mock_validate:
            reason = 'fake reason'
            mock_validate.side_effect = exception.InvalidParameterValue(reason)
            ret = self.service.validate_driver_interfaces(self.context,
                                                          node.uuid)
            self.assertFalse(ret['deploy']['result'])
            self.assertEqual(reason, ret['deploy']['reason'])
        mock_iwdi.assert_called_once_with(self.context, expected_info)

    @mock.patch.object(manager.ConductorManager, '_fail_if_in_state',
                       autospec=True)
    @mock.patch.object(manager.ConductorManager, '_mapped_to_this_conductor',
                       autospec=True)
    @mock.patch.object(dbapi.IMPL, 'get_nodeinfo_list', autospec=True)
    def test_iter_nodes(self, mock_nodeinfo_list, mock_mapped,
                        mock_fail_if_state):
        self._start_service()
        self.service._correct_stuck_states()
        # reset_mock() after startup to verify only test-invoked ones
        mock_nodeinfo_list.reset_mock()
        self.columns = ['uuid', 'driver', 'conductor_group', 'id']
        nodes = [self._create_node(id=i, driver='fake-hardware',
                                   conductor_group='')
                 for i in range(2)]
        mock_nodeinfo_list.return_value = self._get_nodeinfo_list_response(
            nodes)
        mock_mapped.side_effect = [True, False]

        result = list(self.service.iter_nodes(fields=['id'],
                                              filters=mock.sentinel.filters))
        self.assertEqual([(nodes[0].uuid, 'fake-hardware', '', 0)], result)
        mock_nodeinfo_list.assert_called_once_with(
            columns=self.columns, filters=mock.sentinel.filters)
        expected_calls = [mock.call(mock.ANY, mock.ANY,
                                    {'provision_state': 'deploying',
                                     'reserved': False},
                                    'deploying',
                                    'provision_updated_at',
                                    last_error=mock.ANY),
                          mock.call(mock.ANY, mock.ANY,
                                    {'provision_state': 'cleaning',
                                     'reserved': False},
                                    'cleaning',
                                    'provision_updated_at',
                                    last_error=mock.ANY)]
        mock_fail_if_state.assert_has_calls(expected_calls)

    @mock.patch.object(dbapi.IMPL, 'get_nodeinfo_list', autospec=True)
    def test_iter_nodes_shutdown(self, mock_nodeinfo_list):
        self._start_service()
        self.columns = ['uuid', 'driver', 'conductor_group', 'id']
        nodes = [self._create_node(driver='fake-hardware')]
        mock_nodeinfo_list.return_value = self._get_nodeinfo_list_response(
            nodes)
        self.service._shutdown.set()

        result = list(self.service.iter_nodes(fields=['id'],
                                              filters=mock.sentinel.filters))
        self.assertEqual([], result)
        self.service._shutdown.clear()

    def test_get_node_with_token(self):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            network_interface='noop')
        self.assertNotIn('agent_secret_token', node.driver_internal_info)
        res = self.service.get_node_with_token(self.context, node.id)
        self.assertIn('agent_secret_token', res.driver_internal_info)

    def test_node_with_token_already_set(self):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            network_interface='noop',
            driver_internal_info={'agent_secret_token': 'secret'})
        res = self.service.get_node_with_token(self.context, node.id)
        self.assertEqual('******',
                         res.driver_internal_info['agent_secret_token'])

    def test_node_with_token_already_locked(self):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            network_interface='noop',
            reservation='meow')
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.get_node_with_token,
                                self.context, node.id)
        self.assertEqual(exception.NodeLocked, exc.exc_info[0])


@mgr_utils.mock_record_keepalive
class ConsoleTestCase(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase):
    def test_set_console_mode_worker_pool_full(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        self._start_service()
        with mock.patch.object(self.service,
                               '_spawn_worker', autospec=True) as spawn_mock:
            spawn_mock.side_effect = exception.NoFreeConductorWorker()

            exc = self.assertRaises(messaging.rpc.ExpectedException,
                                    self.service.set_console_mode,
                                    self.context, node.uuid, True)
            # Compare true exception hidden by @messaging.expected_exceptions
            self.assertEqual(exception.NoFreeConductorWorker, exc.exc_info[0])
            spawn_mock.assert_called_once_with(mock.ANY, mock.ANY, mock.ANY)

    @mock.patch.object(notification_utils, 'emit_console_notification',
                       autospec=True)
    def test_set_console_mode_enabled(self, mock_notify):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        self.service._executor = futurist.SynchronousExecutor()
        self.service.set_console_mode(self.context, node.uuid, True)
        node.refresh()
        self.assertTrue(node.console_enabled)
        mock_notify.assert_has_calls(
            [mock.call(mock.ANY, 'console_set',
                       obj_fields.NotificationStatus.START),
             mock.call(mock.ANY, 'console_set',
                       obj_fields.NotificationStatus.END)])

    @mock.patch.object(notification_utils, 'emit_console_notification',
                       autospec=True)
    def test_set_console_mode_disabled(self, mock_notify):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          console_enabled=True)
        self.service._executor = futurist.SynchronousExecutor()
        self.service.set_console_mode(self.context, node.uuid, False)
        node.refresh()
        self.assertFalse(node.console_enabled)
        mock_notify.assert_has_calls(
            [mock.call(mock.ANY, 'console_set',
                       obj_fields.NotificationStatus.START),
             mock.call(mock.ANY, 'console_set',
                       obj_fields.NotificationStatus.END)])

    @mock.patch.object(fake.FakeConsole, 'validate', autospec=True)
    def test_set_console_mode_validation_fail(self, mock_val):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          last_error=None)
        mock_val.side_effect = exception.InvalidParameterValue('error')
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.set_console_mode,
                                self.context, node.uuid, True)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.InvalidParameterValue, exc.exc_info[0])

    @mock.patch.object(fake.FakeConsole, 'start_console', autospec=True)
    @mock.patch.object(notification_utils, 'emit_console_notification',
                       autospec=True)
    def test_set_console_mode_start_fail(self, mock_notify, mock_sc):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          last_error=None,
                                          console_enabled=False)
        self.service._executor = futurist.SynchronousExecutor()
        mock_sc.side_effect = exception.IronicException('test-error')
        self.service.set_console_mode(self.context, node.uuid, True)
        mock_sc.assert_called_once_with(mock.ANY, mock.ANY)
        node.refresh()
        self.assertIsNotNone(node.last_error)
        mock_notify.assert_has_calls(
            [mock.call(mock.ANY, 'console_set',
                       obj_fields.NotificationStatus.START),
             mock.call(mock.ANY, 'console_set',
                       obj_fields.NotificationStatus.ERROR)])

    @mock.patch.object(fake.FakeConsole, 'stop_console', autospec=True)
    @mock.patch.object(notification_utils, 'emit_console_notification',
                       autospec=True)
    def test_set_console_mode_stop_fail(self, mock_notify, mock_sc):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          last_error=None,
                                          console_enabled=True)
        self.service._executor = futurist.SynchronousExecutor()
        mock_sc.side_effect = exception.IronicException('test-error')
        self.service.set_console_mode(self.context, node.uuid, False)
        mock_sc.assert_called_once_with(mock.ANY, mock.ANY)
        node.refresh()
        self.assertIsNotNone(node.last_error)
        mock_notify.assert_has_calls(
            [mock.call(mock.ANY, 'console_set',
                       obj_fields.NotificationStatus.START),
             mock.call(mock.ANY, 'console_set',
                       obj_fields.NotificationStatus.ERROR)])

    @mock.patch.object(fake.FakeConsole, 'start_console', autospec=True)
    @mock.patch.object(notification_utils, 'emit_console_notification',
                       autospec=True)
    def test_enable_console_already_enabled(self, mock_notify, mock_sc):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          console_enabled=True)
        self.service._executor = futurist.SynchronousExecutor()
        self.service.set_console_mode(self.context, node.uuid, True)
        self.assertFalse(mock_sc.called)
        self.assertFalse(mock_notify.called)

    @mock.patch.object(fake.FakeConsole, 'stop_console', autospec=True)
    @mock.patch.object(notification_utils, 'emit_console_notification',
                       autospec=True)
    def test_disable_console_already_disabled(self, mock_notify, mock_sc):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          console_enabled=False)
        self.service.set_console_mode(self.context, node.uuid, False)
        self.assertFalse(mock_sc.called)
        self.assertFalse(mock_notify.called)

    @mock.patch.object(fake.FakeConsole, 'get_console', autospec=True)
    def test_get_console(self, mock_gc):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          console_enabled=True)
        console_info = {'test': 'test info'}
        mock_gc.return_value = console_info
        data = self.service.get_console_information(self.context,
                                                    node.uuid)
        self.assertEqual(console_info, data)

    def test_get_console_disabled(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          console_enabled=False)
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.get_console_information,
                                self.context, node.uuid)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NodeConsoleNotEnabled, exc.exc_info[0])

    @mock.patch.object(fake.FakeConsole, 'validate', autospec=True)
    def test_get_console_validate_fail(self, mock_val):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          console_enabled=True)
        mock_val.side_effect = exception.InvalidParameterValue('error')
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.get_console_information,
                                self.context, node.uuid)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.InvalidParameterValue, exc.exc_info[0])


@mgr_utils.mock_record_keepalive
class DestroyNodeTestCase(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase):

    def test_destroy_node(self):
        self._start_service()
        for state in states.DELETE_ALLOWED_STATES:
            node = obj_utils.create_test_node(self.context,
                                              provision_state=state)
            self.service.destroy_node(self.context, node.uuid)
            self.assertRaises(exception.NodeNotFound,
                              self.dbapi.get_node_by_uuid,
                              node.uuid)

    @mock.patch.object(objects.Node, 'release', autospec=True)
    def test_destroy_node_no_release_reservation(self,
                                                 mock_release):
        """Verify destroy_node clears task.node before __exit__.

        After a successful node deletion the task's node reference
        must be None so that release_resources() does not attempt to
        release the DB reservation on the already-deleted row. A
        stale read in release_node() can otherwise raise NodeLocked
        instead of NodeNotFound, surfacing a spurious HTTP 409 to
        the caller.
        """
        self._start_service()
        node = obj_utils.create_test_node(
            self.context,
            provision_state=states.MANAGEABLE)
        self.service.destroy_node(self.context, node.uuid)
        self.assertRaises(exception.NodeNotFound,
                          self.dbapi.get_node_by_uuid,
                          node.uuid)
        mock_release.assert_not_called()

    def test_destroy_node_reserved(self):
        self._start_service()
        fake_reservation = 'fake-reserv'
        node = obj_utils.create_test_node(self.context,
                                          reservation=fake_reservation)

        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.destroy_node,
                                self.context, node.uuid)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NodeLocked, exc.exc_info[0])
        # Verify existing reservation wasn't broken.
        node.refresh()
        self.assertEqual(fake_reservation, node.reservation)

    def test_destroy_node_associated(self):
        self._start_service()
        node = obj_utils.create_test_node(
            self.context, instance_uuid=uuidutils.generate_uuid())

        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.destroy_node,
                                self.context, node.uuid)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NodeAssociated, exc.exc_info[0])

        # Verify reservation was released.
        node.refresh()
        self.assertIsNone(node.reservation)

    def test_destroy_node_with_allocation(self):
        # Nodes with allocations can be deleted in maintenance
        node = obj_utils.create_test_node(self.context,
                                          provision_state=states.ACTIVE,
                                          maintenance=True)
        alloc = obj_utils.create_test_allocation(self.context)
        # Establish cross-linking between the node and the allocation
        alloc.node_id = node.id
        alloc.save()
        node.refresh()

        self.service.destroy_node(self.context, node.uuid)
        self.assertRaises(exception.NodeNotFound,
                          self.dbapi.get_node_by_uuid,
                          node.uuid)
        self.assertRaises(exception.AllocationNotFound,
                          self.dbapi.get_allocation_by_id,
                          alloc.id)

    def test_destroy_node_invalid_provision_state(self):
        self._start_service()
        node = obj_utils.create_test_node(self.context,
                                          provision_state=states.ACTIVE)

        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.destroy_node,
                                self.context, node.uuid)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.InvalidState, exc.exc_info[0])
        # Verify reservation was released.
        node.refresh()
        self.assertIsNone(node.reservation)

    def test_destroy_node_protected_provision_state_available(self):
        CONF.set_override('allow_deleting_available_nodes',
                          False, group='conductor')
        self._start_service()
        node = obj_utils.create_test_node(self.context,
                                          provision_state=states.AVAILABLE)

        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.destroy_node,
                                self.context, node.uuid)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.InvalidState, exc.exc_info[0])
        # Verify reservation was released.
        node.refresh()
        self.assertIsNone(node.reservation)

    def test_destroy_node_protected(self):
        self._start_service()
        node = obj_utils.create_test_node(self.context,
                                          provision_state=states.ACTIVE,
                                          protected=True,
                                          # Even in maintenance the protected
                                          # nodes are not deleted
                                          maintenance=True)

        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.destroy_node,
                                self.context, node.uuid)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NodeProtected, exc.exc_info[0])
        # Verify reservation was released.
        node.refresh()
        self.assertIsNone(node.reservation)

    def test_destroy_node_allowed_in_maintenance(self):
        self._start_service()
        node = obj_utils.create_test_node(
            self.context, instance_uuid=uuidutils.generate_uuid(),
            provision_state=states.ACTIVE, maintenance=True)
        self.service.destroy_node(self.context, node.uuid)
        self.assertRaises(exception.NodeNotFound,
                          self.dbapi.get_node_by_uuid,
                          node.uuid)

    def test_destroy_node_power_off(self):
        self._start_service()
        node = obj_utils.create_test_node(self.context,
                                          power_state=states.POWER_OFF)
        self.service.destroy_node(self.context, node.uuid)

    @mock.patch.object(fake.FakeConsole, 'stop_console', autospec=True)
    @mock.patch.object(notification_utils, 'emit_console_notification',
                       autospec=True)
    def test_destroy_node_console_enabled(self, mock_notify, mock_sc):
        self._start_service()
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          console_enabled=True)
        self.service.destroy_node(self.context, node.uuid)
        mock_sc.assert_called_once_with(mock.ANY, mock.ANY)
        self.assertRaises(exception.NodeNotFound,
                          self.dbapi.get_node_by_uuid,
                          node.uuid)
        mock_notify.assert_has_calls(
            [mock.call(mock.ANY, 'console_set',
                       obj_fields.NotificationStatus.START),
             mock.call(mock.ANY, 'console_set',
                       obj_fields.NotificationStatus.END)])

    @mock.patch.object(fake.FakeConsole, 'stop_console', autospec=True)
    @mock.patch.object(notification_utils, 'emit_console_notification',
                       autospec=True)
    def test_destroy_node_console_disable_fail(self, mock_notify, mock_sc):
        self._start_service()
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          console_enabled=True)
        mock_sc.side_effect = Exception()
        self.service.destroy_node(self.context, node.uuid)
        mock_sc.assert_called_once_with(mock.ANY, mock.ANY)
        self.assertRaises(exception.NodeNotFound,
                          self.dbapi.get_node_by_uuid,
                          node.uuid)
        mock_notify.assert_has_calls(
            [mock.call(mock.ANY, 'console_set',
                       obj_fields.NotificationStatus.START),
             mock.call(mock.ANY, 'console_set',
                       obj_fields.NotificationStatus.ERROR)])

    @mock.patch.object(fake.FakePower, 'set_power_state', autospec=True)
    def test_destroy_node_adopt_failed_no_power_change(self, mock_power):
        self._start_service()
        node = obj_utils.create_test_node(self.context,
                                          driver='fake-hardware',
                                          provision_state=states.ADOPTFAIL)
        self.service.destroy_node(self.context, node.uuid)
        self.assertFalse(mock_power.called)

    def test_destroy_node_broken_driver(self):
        node = obj_utils.create_test_node(self.context,
                                          power_interface='broken')
        self._start_service()
        self.service.destroy_node(self.context, node.uuid)
        self.assertRaises(exception.NodeNotFound,
                          self.dbapi.get_node_by_uuid,
                          node.uuid)

    @mock.patch.object(inspect_utils, 'clean_up_swift_entries', autospec=True)
    def test_inventory_in_swift_get_destroyed_after_destroying_a_node_by_uuid(
            self, mock_clean_up):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        CONF.set_override('data_backend', 'swift', group='inventory')
        self._start_service()
        self.service.destroy_node(self.context, node.uuid)
        mock_clean_up.assert_called_once_with(mock.ANY)

    @mock.patch.object(inspect_utils, 'clean_up_swift_entries', autospec=True)
    def test_inventory_in_swift_not_destroyed_SwiftOSE_maintenance(
            self, mock_clean_up):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          maintenance=True)
        CONF.set_override('data_backend', 'swift', group='inventory')
        mock_clean_up.side_effect = exception.SwiftObjectStillExists(
            obj="inventory-123", node=node.uuid)
        self._start_service()
        self.service.destroy_node(self.context, node.uuid)

    @mock.patch.object(inspect_utils, 'clean_up_swift_entries', autospec=True)
    def test_inventory_in_swift_not_destroyed_SwiftOSE_not_maintenance(
            self, mock_clean_up):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          maintenance=False)
        CONF.set_override('data_backend', 'swift', group='inventory')
        mock_clean_up.side_effect = exception.SwiftObjectStillExists(
            obj="inventory-123", node=node.uuid)
        self._start_service()
        self.assertRaises(exception.SwiftObjectStillExists,
                          self.service.destroy_node, self.context,
                          node.uuid)


@mgr_utils.mock_record_keepalive
class CreatePortTestCase(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase):

    @mock.patch.object(conductor_utils, 'validate_port_physnet', autospec=True)
    def test_create_port(self, mock_validate):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        port = obj_utils.get_test_port(self.context, node_id=node.id,
                                       extra={'foo': 'bar'})
        res = self.service.create_port(self.context, port)
        self.assertEqual({'foo': 'bar'}, res.extra)
        res = objects.Port.get_by_uuid(self.context, port['uuid'])
        self.assertEqual({'foo': 'bar'}, res.extra)
        mock_validate.assert_called_once_with(mock.ANY, port)

    @mock.patch.object(conductor_utils, 'validate_port_physnet', autospec=True)
    def test_create_port_mac_exists(self, mock_validate):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        port = obj_utils.create_test_port(self.context, node_id=node.id)
        port = obj_utils.get_test_port(self.context, node_id=node.id,
                                       uuid=uuidutils.generate_uuid())
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.create_port,
                                self.context, port)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.MACAlreadyExists, exc.exc_info[0])
        self.assertRaises(exception.PortNotFound, port.get_by_uuid,
                          self.context, port.uuid)

    @mock.patch.object(conductor_utils, 'validate_port_physnet', autospec=True)
    def test_create_port_physnet_validation_failure_conflict(self,
                                                             mock_validate):
        mock_validate.side_effect = exception.Conflict
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        port = obj_utils.get_test_port(self.context, node_id=node.id)
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.create_port,
                                self.context, port)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.Conflict, exc.exc_info[0])
        self.assertRaises(exception.PortNotFound, port.get_by_uuid,
                          self.context, port.uuid)

    @mock.patch.object(conductor_utils, 'validate_port_physnet', autospec=True)
    def test_create_port_physnet_validation_failure_inconsistent(
            self, mock_validate):
        mock_validate.side_effect = exception.PortgroupPhysnetInconsistent(
            portgroup='pg1', physical_networks='physnet1, physnet2')
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        port = obj_utils.get_test_port(self.context, node_id=node.id)
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.create_port,
                                self.context, port)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.PortgroupPhysnetInconsistent,
                         exc.exc_info[0])
        self.assertRaises(exception.PortNotFound, port.get_by_uuid,
                          self.context, port.uuid)


@mgr_utils.mock_record_keepalive
class UpdatePortTestCase(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase):

    @mock.patch.object(conductor_utils, 'validate_port_physnet', autospec=True)
    @mock.patch.object(n_flat.FlatNetwork, 'port_changed', autospec=True)
    @mock.patch.object(n_flat.FlatNetwork, 'validate', autospec=True)
    def test_update_port(self, mock_val, mock_pc, mock_vpp):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')

        port = obj_utils.create_test_port(self.context,
                                          node_id=node.id,
                                          extra={'foo': 'bar'})
        new_extra = {'foo': 'baz'}
        port.extra = new_extra
        res = self.service.update_port(self.context, port)
        self.assertEqual(new_extra, res.extra)
        mock_val.assert_called_once_with(mock.ANY, mock.ANY)
        mock_pc.assert_called_once_with(mock.ANY, mock.ANY, port)
        mock_vpp.assert_called_once_with(mock.ANY, port)

    def test_update_port_node_locked(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          reservation='fake-reserv')

        port = obj_utils.create_test_port(self.context, node_id=node.id)
        port.extra = {'foo': 'baz'}
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.update_port,
                                self.context, port)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NodeLocked, exc.exc_info[0])

    @mock.patch.object(n_flat.FlatNetwork, 'port_changed', autospec=True)
    @mock.patch.object(n_flat.FlatNetwork, 'validate', autospec=True)
    def test_update_port_port_changed_failure(self, mock_val, mock_pc):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')

        port = obj_utils.create_test_port(self.context,
                                          node_id=node.id)
        old_address = port.address
        port.address = '11:22:33:44:55:bb'
        mock_pc.side_effect = (exception.FailedToUpdateMacOnPort('boom'))
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.update_port,
                                self.context, port)
        mock_pc.assert_called_once_with(mock.ANY, mock.ANY, port)
        mock_val.assert_called_once_with(mock.ANY, mock.ANY)
        self.assertEqual(exception.FailedToUpdateMacOnPort, exc.exc_info[0])
        port.refresh()
        self.assertEqual(old_address, port.address)

    @mock.patch.object(n_flat.FlatNetwork, 'port_changed', autospec=True)
    @mock.patch.object(n_flat.FlatNetwork, 'validate', autospec=True)
    def test_update_port_address_active_node(self, mock_val, mock_pc):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          instance_uuid=None,
                                          provision_state='active')
        port = obj_utils.create_test_port(self.context,
                                          node_id=node.id,
                                          extra={'vif_port_id': 'fake-id'})
        old_address = port.address
        new_address = '11:22:33:44:55:bb'
        port.address = new_address
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.update_port,
                                self.context, port)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.InvalidState, exc.exc_info[0])
        port.refresh()
        self.assertEqual(old_address, port.address)
        self.assertFalse(mock_pc.called)
        self.assertFalse(mock_val.called)

    @mock.patch.object(n_flat.FlatNetwork, 'port_changed', autospec=True)
    @mock.patch.object(n_flat.FlatNetwork, 'validate', autospec=True)
    def test_update_port_address_maintenance(self, mock_val, mock_pc):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware', maintenance=True,
            instance_uuid=uuidutils.generate_uuid(), provision_state='active')
        port = obj_utils.create_test_port(self.context,
                                          node_id=node.id,
                                          extra={'vif_port_id': 'fake-id'})
        new_address = '11:22:33:44:55:bb'
        port.address = new_address
        res = self.service.update_port(self.context, port)
        self.assertEqual(new_address, res.address)
        mock_val.assert_called_once_with(mock.ANY, mock.ANY)
        mock_pc.assert_called_once_with(mock.ANY, mock.ANY, port)

    @mock.patch.object(n_flat.FlatNetwork, 'port_changed', autospec=True)
    @mock.patch.object(n_flat.FlatNetwork, 'validate', autospec=True)
    def test_update_port_portgroup_active_node(self, mock_val, mock_pc):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          instance_uuid=None,
                                          provision_state='active')
        pg1 = obj_utils.create_test_portgroup(self.context, node_id=node.id)
        pg2 = obj_utils.create_test_portgroup(
            self.context, node_id=node.id, name='bar',
            address='aa:bb:cc:dd:ee:ff', uuid=uuidutils.generate_uuid())
        port = obj_utils.create_test_port(self.context,
                                          node_id=node.id,
                                          portgroup_id=pg1.id)
        port.portgroup_id = pg2.id
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.update_port,
                                self.context, port)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.InvalidState, exc.exc_info[0])
        port.refresh()
        self.assertEqual(pg1.id, port.portgroup_id)
        self.assertFalse(mock_pc.called)
        self.assertFalse(mock_val.called)

    @mock.patch.object(n_flat.FlatNetwork, 'port_changed', autospec=True)
    @mock.patch.object(n_flat.FlatNetwork, 'validate', autospec=True)
    def test_update_port_portgroup_enroll_node(self, mock_val, mock_pc):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          instance_uuid=None,
                                          provision_state='enroll')
        pg1 = obj_utils.create_test_portgroup(self.context, node_id=node.id)
        pg2 = obj_utils.create_test_portgroup(
            self.context, node_id=node.id, name='bar',
            address='aa:bb:cc:dd:ee:ff', uuid=uuidutils.generate_uuid())
        port = obj_utils.create_test_port(self.context,
                                          node_id=node.id,
                                          portgroup_id=pg1.id)
        port.portgroup_id = pg2.id
        self.service.update_port(self.context, port)
        port.refresh()
        self.assertEqual(pg2.id, port.portgroup_id)
        mock_pc.assert_called_once_with(mock.ANY, mock.ANY, port)
        mock_val.assert_called_once_with(mock.ANY, mock.ANY)

    def test_update_port_node_deleting_state(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          provision_state=states.DELETING)
        port = obj_utils.create_test_port(self.context,
                                          node_id=node.id,
                                          extra={'foo': 'bar'})
        old_pxe = port.pxe_enabled
        port.pxe_enabled = True
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.update_port,
                                self.context, port)
        self.assertEqual(exception.InvalidState, exc.exc_info[0])
        port.refresh()
        self.assertEqual(old_pxe, port.pxe_enabled)

    @mock.patch.object(n_flat.FlatNetwork, 'port_changed', autospec=True)
    @mock.patch.object(n_flat.FlatNetwork, 'validate', autospec=True)
    def test_update_port_node_manageable_state(self, mock_val,
                                               mock_pc):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          provision_state=states.MANAGEABLE)
        port = obj_utils.create_test_port(self.context,
                                          node_id=node.id,
                                          extra={'foo': 'bar'})
        port.pxe_enabled = True
        self.service.update_port(self.context, port)
        port.refresh()
        self.assertEqual(True, port.pxe_enabled)
        mock_val.assert_called_once_with(mock.ANY, mock.ANY)
        mock_pc.assert_called_once_with(mock.ANY, mock.ANY, port)

    @mock.patch.object(n_flat.FlatNetwork, 'port_changed', autospec=True)
    @mock.patch.object(n_flat.FlatNetwork, 'validate', autospec=True)
    def test_update_port_to_node_in_inspect_wait_state(self, mock_val,
                                                       mock_pc):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          provision_state=states.INSPECTWAIT)
        port = obj_utils.create_test_port(self.context,
                                          node_id=node.id,
                                          extra={'foo': 'bar'})
        port.pxe_enabled = True
        self.service.update_port(self.context, port)
        port.refresh()
        self.assertEqual(True, port.pxe_enabled)
        mock_val.assert_called_once_with(mock.ANY, mock.ANY)
        mock_pc.assert_called_once_with(mock.ANY, mock.ANY, port)

    @mock.patch.object(n_flat.FlatNetwork, 'port_changed', autospec=True)
    @mock.patch.object(n_flat.FlatNetwork, 'validate', autospec=True)
    def test_update_port_node_active_state_and_maintenance(self, mock_val,
                                                           mock_pc):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          provision_state=states.ACTIVE,
                                          maintenance=True)
        port = obj_utils.create_test_port(self.context,
                                          node_id=node.id,
                                          extra={'foo': 'bar'})
        port.pxe_enabled = True
        self.service.update_port(self.context, port)
        port.refresh()
        self.assertEqual(True, port.pxe_enabled)
        mock_val.assert_called_once_with(mock.ANY, mock.ANY)
        mock_pc.assert_called_once_with(mock.ANY, mock.ANY, port)

    @mock.patch.object(n_flat.FlatNetwork, 'port_changed', autospec=True)
    @mock.patch.object(n_flat.FlatNetwork, 'validate', autospec=True)
    def test_update_port_physnet_maintenance(self, mock_val, mock_pc):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware', maintenance=True,
            instance_uuid=uuidutils.generate_uuid(), provision_state='active')
        port = obj_utils.create_test_port(self.context,
                                          node_id=node.id,
                                          extra={'vif_port_id': 'fake-id'})
        new_physnet = 'physnet1'
        port.physical_network = new_physnet
        res = self.service.update_port(self.context, port)
        self.assertEqual(new_physnet, res.physical_network)
        mock_val.assert_called_once_with(mock.ANY, mock.ANY)
        mock_pc.assert_called_once_with(mock.ANY, mock.ANY, port)

    def test_update_port_physnet_node_deleting_state(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          provision_state=states.DELETING)
        port = obj_utils.create_test_port(self.context,
                                          node_id=node.id,
                                          extra={'foo': 'bar'})
        old_physnet = port.physical_network
        port.physical_network = 'physnet1'
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.update_port,
                                self.context, port)
        self.assertEqual(exception.InvalidState, exc.exc_info[0])
        port.refresh()
        self.assertEqual(old_physnet, port.physical_network)

    @mock.patch.object(conductor_utils, 'validate_port_physnet', autospec=True)
    def test_update_port_physnet_validation_failure_conflict(self,
                                                             mock_validate):
        mock_validate.side_effect = exception.Conflict
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        port = obj_utils.create_test_port(self.context, node_id=node.id,
                                          uuid=uuidutils.generate_uuid())
        port.extra = {'foo': 'bar'}
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.update_port,
                                self.context, port)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.Conflict, exc.exc_info[0])
        mock_validate.assert_called_once_with(mock.ANY, port)

    @mock.patch.object(conductor_utils, 'validate_port_physnet', autospec=True)
    def test_update_port_physnet_validation_failure_inconsistent(
            self, mock_validate):
        mock_validate.side_effect = exception.PortgroupPhysnetInconsistent(
            portgroup='pg1', physical_networks='physnet1, physnet2')
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        port = obj_utils.create_test_port(self.context, node_id=node.id,
                                          uuid=uuidutils.generate_uuid())
        port.extra = {'foo': 'bar'}
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.update_port,
                                self.context, port)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.PortgroupPhysnetInconsistent,
                         exc.exc_info[0])
        mock_validate.assert_called_once_with(mock.ANY, port)


@mgr_utils.mock_record_keepalive
class SensorsTestCase(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase):

    def test__filter_out_unsupported_types_all(self):
        self._start_service()
        CONF.set_override('data_types', ['All'],
                          group='sensor_data')
        fake_sensors_data = {"t1": {'f1': 'v1'}, "t2": {'f1': 'v1'}}
        actual_result = (
            self.service._filter_out_unsupported_types(fake_sensors_data))
        expected_result = {"t1": {'f1': 'v1'}, "t2": {'f1': 'v1'}}
        self.assertEqual(expected_result, actual_result)

    def test__filter_out_unsupported_types_part(self):
        self._start_service()
        CONF.set_override('data_types', ['t1'],
                          group='sensor_data')
        fake_sensors_data = {"t1": {'f1': 'v1'}, "t2": {'f1': 'v1'}}
        actual_result = (
            self.service._filter_out_unsupported_types(fake_sensors_data))
        expected_result = {"t1": {'f1': 'v1'}}
        self.assertEqual(expected_result, actual_result)

    def test__filter_out_unsupported_types_non(self):
        self._start_service()
        CONF.set_override('data_types', ['t3'],
                          group='sensor_data')
        fake_sensors_data = {"t1": {'f1': 'v1'}, "t2": {'f1': 'v1'}}
        actual_result = (
            self.service._filter_out_unsupported_types(fake_sensors_data))
        expected_result = {}
        self.assertEqual(expected_result, actual_result)

    @mock.patch.object(messaging.Notifier, 'info', autospec=True)
    @mock.patch.object(task_manager, 'acquire', autospec=True)
    def test_send_sensor_task(self, acquire_mock, notifier_mock):
        nodes = queue.Queue()
        for i in range(5):
            nodes.put_nowait(('fake_uuid-%d' % i, 'fake-hardware', '', None))
        self._start_service()
        CONF.set_override('send_sensor_data', True,
                          group='sensor_data')

        task = acquire_mock.return_value.__enter__.return_value
        task.node.maintenance = False
        task.node.driver = 'fake'
        task.node.name = 'fake_node'
        get_sensors_data_mock = task.driver.management.get_sensors_data
        validate_mock = task.driver.management.validate
        get_sensors_data_mock.return_value = 'fake-sensor-data'
        self.service._sensors_nodes_task(self.context, nodes)
        self.assertEqual(5, acquire_mock.call_count)
        self.assertEqual(5, validate_mock.call_count)
        self.assertEqual(5, get_sensors_data_mock.call_count)
        self.assertEqual(5, notifier_mock.call_count)
        n_call = mock.call(mock.ANY, mock.ANY, 'hardware.fake.metrics',
                           {'event_type': 'hardware.fake.metrics.update',
                            'node_name': 'fake_node', 'timestamp': mock.ANY,
                            'message_id': mock.ANY,
                            'payload': 'fake-sensor-data',
                            'node_uuid': mock.ANY, 'instance_uuid': None})
        notifier_mock.assert_has_calls([n_call, n_call, n_call,
                                        n_call, n_call])

    @mock.patch.object(task_manager, 'acquire', autospec=True)
    def test_send_sensor_task_shutdown(self, acquire_mock):
        nodes = queue.Queue()
        nodes.put_nowait(('fake_uuid', 'fake-hardware', '', None))
        self._start_service()
        self.service._shutdown.set()
        CONF.set_override('send_sensor_data', True,
                          group='sensor_data')
        self.service._sensors_nodes_task(self.context, nodes)
        acquire_mock.return_value.__enter__.assert_not_called()
        self.service._shutdown.clear()

    @mock.patch.object(task_manager, 'acquire', autospec=True)
    def test_send_sensor_task_no_management(self, acquire_mock):
        nodes = queue.Queue()
        nodes.put_nowait(('fake_uuid', 'fake-hardware', '', None))

        CONF.set_override('send_sensor_data', True,
                          group='sensor_data')

        self._start_service()

        task = acquire_mock.return_value.__enter__.return_value
        task.node.maintenance = False
        task.driver.management = None

        self.service._sensors_nodes_task(self.context, nodes)

        self.assertTrue(acquire_mock.called)

    @mock.patch.object(manager.LOG, 'debug', autospec=True)
    @mock.patch.object(task_manager, 'acquire', autospec=True)
    def test_send_sensor_task_maintenance(self, acquire_mock, debug_log):
        nodes = queue.Queue()
        nodes.put_nowait(('fake_uuid', 'fake-hardware', '', None))
        self._start_service()
        CONF.set_override('send_sensor_data', True, group='sensor_data')

        task = acquire_mock.return_value.__enter__.return_value
        task.node.maintenance = True
        get_sensors_data_mock = task.driver.management.get_sensors_data
        validate_mock = task.driver.management.validate

        self.service._sensors_nodes_task(self.context, nodes)
        self.assertTrue(acquire_mock.called)
        self.assertFalse(validate_mock.called)
        self.assertFalse(get_sensors_data_mock.called)
        self.assertTrue(debug_log.called)

    @mock.patch.object(manager.ConductorManager, '_spawn_worker',
                       autospec=True)
    @mock.patch.object(manager.ConductorManager, '_mapped_to_this_conductor',
                       autospec=True)
    @mock.patch.object(dbapi.IMPL, 'get_nodeinfo_list', autospec=True)
    def test___send_sensor_data(self, get_nodeinfo_list_mock,
                                _mapped_to_this_conductor_mock,
                                mock_spawn):
        self._start_service()

        CONF.set_override('send_sensor_data', True, group='sensor_data')
        # NOTE(galyna): do not wait for threads to be finished in unittests
        CONF.set_override('wait_timeout', 0,
                          group='sensor_data')
        _mapped_to_this_conductor_mock.return_value = True
        get_nodeinfo_list_mock.return_value = [('fake_uuid', 'fake', None)]
        self.service._send_sensor_data(self.context)
        mock_spawn.assert_called_with(self.service,
                                      self.service._sensors_nodes_task,
                                      self.context, mock.ANY)

    @mock.patch.object(manager.ConductorManager, '_sensors_conductor',
                       autospec=True)
    @mock.patch.object(manager.ConductorManager, '_spawn_worker',
                       autospec=True)
    @mock.patch.object(manager.ConductorManager, '_mapped_to_this_conductor',
                       autospec=True)
    @mock.patch.object(dbapi.IMPL, 'get_nodeinfo_list', autospec=True)
    def test___send_sensor_data_disabled(
            self, get_nodeinfo_list_mock,
            _mapped_to_this_conductor_mock,
            mock_spawn, mock_sensors_conductor):
        self._start_service()

        CONF.set_override('send_sensor_data', True, group='sensor_data')
        CONF.set_override('enable_for_nodes', False,
                          group='sensor_data')
        CONF.set_override('enable_for_conductor', False,
                          group='sensor_data')
        # NOTE(galyna): do not wait for threads to be finished in unittests
        CONF.set_override('wait_timeout', 0,
                          group='sensor_data')
        _mapped_to_this_conductor_mock.return_value = True
        get_nodeinfo_list_mock.return_value = [('fake_uuid', 'fake', None)]

        # NOTE(cid): Patch here instead of as a decorator to avoid replacing
        # queue.Queue globally. ThreadPoolExecutor uses it in
        # _init_executors(), called by init_host(), and relies on real queue
        # semantics for backlog checks.
        with mock.patch('ironic.conductor.manager.queue.Queue',
                        autospec=True) as mock_queue:
            self.service._send_sensor_data(self.context)

        mock_sensors_conductor.assert_not_called()
        # NOTE(TheJulia): Can't use the spawn worker since it records other,
        # unrelated calls. So, queue works well here.
        mock_queue.assert_not_called()

    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    @mock.patch.object(manager.ConductorManager, '_mapped_to_this_conductor',
                       autospec=True)
    @mock.patch.object(dbapi.IMPL, 'get_nodeinfo_list', autospec=True)
    def test___send_sensor_data_multiple_workers(
            self, get_nodeinfo_list_mock, _mapped_to_this_conductor_mock,
            mock_spawn):
        self._start_service()
        mock_spawn.reset_mock()

        number_of_workers = 8
        CONF.set_override('send_sensor_data', True, group='sensor_data')
        CONF.set_override('workers', number_of_workers,
                          group='sensor_data')
        # NOTE(galyna): do not wait for threads to be finished in unittests
        CONF.set_override('wait_timeout', 0,
                          group='sensor_data')

        _mapped_to_this_conductor_mock.return_value = True
        get_nodeinfo_list_mock.return_value = [('fake_uuid', 'fake',
                                                None)] * 20
        self.service._send_sensor_data(self.context)
        self.assertEqual(number_of_workers + 1,
                         mock_spawn.call_count)

    # TODO(TheJulia): At some point, we should add a test to validate that
    # a modified filter to return all nodes actually works, although
    # the way the sensor tests are written, the list is all mocked.

    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    @mock.patch.object(manager.ConductorManager, '_mapped_to_this_conductor',
                       autospec=True)
    @mock.patch.object(dbapi.IMPL, 'get_nodeinfo_list', autospec=True)
    def test___send_sensor_data_one_worker(
            self, get_nodeinfo_list_mock, _mapped_to_this_conductor_mock,
            mock_spawn):
        self._start_service()
        mock_spawn.reset_mock()

        number_of_workers = 1
        CONF.set_override('send_sensor_data', True, group='sensor_data')
        CONF.set_override('workers', number_of_workers,
                          group='sensor_data')
        # NOTE(galyna): do not wait for threads to be finished in unittests
        CONF.set_override('wait_timeout', 0,
                          group='sensor_data')

        _mapped_to_this_conductor_mock.return_value = True
        get_nodeinfo_list_mock.return_value = [('fake_uuid', 'fake',
                                                None)] * 20
        self.service._send_sensor_data(self.context)
        self.assertEqual(number_of_workers,
                         mock_spawn.call_count)

    @mock.patch.object(messaging.Notifier, 'info', autospec=True)
    @mock.patch.object(ironic_metrics.MetricLogger,
                       'get_metrics_data', autospec=True)
    def test__sensors_conductor(self, mock_get_metrics, mock_notifier):
        metric = {'metric': 'data'}
        mock_get_metrics.return_value = metric
        self._start_service()
        self.service._sensors_conductor(self.context)
        self.assertEqual(mock_notifier.call_count, 1)
        self.assertEqual('ironic.metrics', mock_notifier.call_args.args[2])
        metrics_dict = mock_notifier.call_args.args[3]
        self.assertEqual(metrics_dict.get('event_type'),
                         'ironic.metrics.update')
        self.assertDictEqual(metrics_dict.get('payload'),
                             metric)


@mgr_utils.mock_record_keepalive
class BootDeviceTestCase(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase):

    @mock.patch.object(fake.FakeManagement, 'set_boot_device', autospec=True)
    @mock.patch.object(fake.FakeManagement, 'validate', autospec=True)
    def test_set_boot_device(self, mock_val, mock_sbd):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        self.service.set_boot_device(self.context, node.uuid,
                                     boot_devices.PXE)
        mock_val.assert_called_once_with(mock.ANY, mock.ANY)
        mock_sbd.assert_called_once_with(mock.ANY, mock.ANY, boot_devices.PXE,
                                         persistent=False)

    def test_set_boot_device_node_locked(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          reservation='fake-reserv')
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.set_boot_device,
                                self.context, node.uuid, boot_devices.DISK)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NodeLocked, exc.exc_info[0])

    @mock.patch.object(fake.FakeManagement, 'validate', autospec=True)
    def test_set_boot_device_validate_fail(self, mock_val):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        mock_val.side_effect = exception.InvalidParameterValue('error')
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.set_boot_device,
                                self.context, node.uuid, boot_devices.DISK)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.InvalidParameterValue, exc.exc_info[0])

    def test_get_boot_device(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        bootdev = self.service.get_boot_device(self.context, node.uuid)
        expected = {'boot_device': boot_devices.PXE, 'persistent': False}
        self.assertEqual(expected, bootdev)

    def test_get_boot_device_node_locked(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          reservation='fake-reserv')
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.get_boot_device,
                                self.context, node.uuid)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NodeLocked, exc.exc_info[0])

    @mock.patch.object(fake.FakeManagement, 'validate', autospec=True)
    def test_get_boot_device_validate_fail(self, mock_val):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        mock_val.side_effect = exception.InvalidParameterValue('error')
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.get_boot_device,
                                self.context, node.uuid)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.InvalidParameterValue, exc.exc_info[0])

    def test_get_supported_boot_devices(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        bootdevs = self.service.get_supported_boot_devices(self.context,
                                                           node.uuid)
        self.assertEqual([boot_devices.PXE], bootdevs)


@mgr_utils.mock_record_keepalive
class IndicatorsTestCase(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase):

    @mock.patch.object(fake.FakeManagement, 'set_indicator_state',
                       autospec=True)
    @mock.patch.object(fake.FakeManagement, 'validate', autospec=True)
    def test_set_indicator_state(self, mock_val, mock_sbd):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        self.service.set_indicator_state(
            self.context, node.uuid, components.CHASSIS,
            'led', indicator_states.ON)
        mock_val.assert_called_once_with(mock.ANY, mock.ANY)
        mock_sbd.assert_called_once_with(
            mock.ANY, mock.ANY, components.CHASSIS, 'led', indicator_states.ON)

    def test_get_indicator_state(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        state = self.service.get_indicator_state(
            self.context, node.uuid, components.CHASSIS, 'led-0')
        expected = indicator_states.ON
        self.assertEqual(expected, state)

    def test_get_supported_indicators(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        indicators = self.service.get_supported_indicators(
            self.context, node.uuid)
        expected = {
            'chassis': {
                'led-0': {
                    'readonly': True,
                    'states': [
                        indicator_states.OFF,
                        indicator_states.ON
                    ]
                }
            },
            'system': {
                'led': {
                    'readonly': False,
                    'states': [
                        indicator_states.BLINKING,
                        indicator_states.OFF,
                        indicator_states.ON
                    ]
                }
            }
        }
        self.assertEqual(expected, indicators)


@mgr_utils.mock_record_keepalive
class NmiTestCase(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase):

    @mock.patch.object(fake.FakeManagement, 'inject_nmi', autospec=True)
    @mock.patch.object(fake.FakeManagement, 'validate', autospec=True)
    def test_inject_nmi(self, mock_val, mock_nmi):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        self.service.inject_nmi(self.context, node.uuid)
        mock_val.assert_called_once_with(mock.ANY, mock.ANY)
        mock_nmi.assert_called_once_with(mock.ANY, mock.ANY)

    def test_inject_nmi_node_locked(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          reservation='fake-reserv')
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.inject_nmi,
                                self.context, node.uuid)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NodeLocked, exc.exc_info[0])

    @mock.patch.object(fake.FakeManagement, 'validate', autospec=True)
    def test_inject_nmi_validate_invalid_param(self, mock_val):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        mock_val.side_effect = exception.InvalidParameterValue('error')
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.inject_nmi,
                                self.context, node.uuid)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.InvalidParameterValue, exc.exc_info[0])

    @mock.patch.object(fake.FakeManagement, 'validate', autospec=True)
    def test_inject_nmi_validate_missing_param(self, mock_val):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        mock_val.side_effect = exception.MissingParameterValue('error')
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.inject_nmi,
                                self.context, node.uuid)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.MissingParameterValue, exc.exc_info[0])

    def test_inject_nmi_not_implemented(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.inject_nmi,
                                self.context, node.uuid)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.UnsupportedDriverExtension,
                         exc.exc_info[0])


@mgr_utils.mock_record_keepalive
@mock.patch.object(n_flat.FlatNetwork, 'validate', autospec=True)
class VifTestCase(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase):

    def setUp(self):
        super(VifTestCase, self).setUp()
        self.vif = {'id': 'fake'}

    @mock.patch.object(n_flat.FlatNetwork, 'vif_list', autospec=True)
    def test_vif_list(self, mock_list, mock_valid):
        mock_list.return_value = ['VIF_ID']
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        data = self.service.vif_list(self.context, node.uuid)
        mock_list.assert_called_once_with(mock.ANY, mock.ANY)
        mock_valid.assert_called_once_with(mock.ANY, mock.ANY)
        self.assertEqual(mock_list.return_value, data)

    @mock.patch.object(n_flat.FlatNetwork, 'vif_attach', autospec=True)
    def test_vif_attach(self, mock_attach, mock_valid):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        self.service.vif_attach(self.context, node.uuid, self.vif)
        mock_attach.assert_called_once_with(mock.ANY, mock.ANY, self.vif)
        mock_valid.assert_called_once_with(mock.ANY, mock.ANY)

    @mock.patch.object(n_flat.FlatNetwork, 'vif_attach', autospec=True)
    def test_vif_attach_node_locked(self, mock_attach, mock_valid):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          reservation='fake-reserv')
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.vif_attach,
                                self.context, node.uuid, self.vif)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NodeLocked, exc.exc_info[0])
        self.assertFalse(mock_attach.called)
        self.assertFalse(mock_valid.called)

    @mock.patch.object(n_flat.FlatNetwork, 'vif_attach', autospec=True)
    def test_vif_attach_raises_network_error(self, mock_attach,
                                             mock_valid):
        mock_attach.side_effect = exception.NetworkError("BOOM")
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.vif_attach,
                                self.context, node.uuid, self.vif)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NetworkError, exc.exc_info[0])
        mock_valid.assert_called_once_with(mock.ANY, mock.ANY)
        mock_attach.assert_called_once_with(mock.ANY, mock.ANY, self.vif)

    @mock.patch.object(n_flat.FlatNetwork, 'vif_attach', autospec=True)
    def test_vif_attach_raises_portgroup_physnet_inconsistent(
            self, mock_attach, mock_valid):
        mock_valid.side_effect = exception.PortgroupPhysnetInconsistent(
            portgroup='fake-pg', physical_networks='fake-physnet')
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.vif_attach,
                                self.context, node.uuid, self.vif)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.PortgroupPhysnetInconsistent,
                         exc.exc_info[0])
        mock_valid.assert_called_once_with(mock.ANY, mock.ANY)
        self.assertFalse(mock_attach.called)

    @mock.patch.object(n_flat.FlatNetwork, 'vif_attach', autospec=True)
    def test_vif_attach_raises_vif_invalid_for_attach(
            self, mock_attach, mock_valid):
        mock_valid.side_effect = exception.VifInvalidForAttach(
            node='fake-node', vif='fake-vif', reason='fake-reason')
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.vif_attach,
                                self.context, node.uuid, self.vif)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.VifInvalidForAttach,
                         exc.exc_info[0])
        mock_valid.assert_called_once_with(mock.ANY, mock.ANY)
        self.assertFalse(mock_attach.called)

    @mock.patch.object(n_flat.FlatNetwork, 'vif_attach', autospec=True)
    def test_vif_attach_validate_error(self, mock_attach,
                                       mock_valid):
        mock_valid.side_effect = exception.MissingParameterValue("BOOM")
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.vif_attach,
                                self.context, node.uuid, self.vif)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.MissingParameterValue, exc.exc_info[0])
        mock_valid.assert_called_once_with(mock.ANY, mock.ANY)
        self.assertFalse(mock_attach.called)

    @mock.patch.object(n_flat.FlatNetwork, 'vif_detach', autospec=True)
    def test_vif_detach(self, mock_detach, mock_valid):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        self.service.vif_detach(self.context, node.uuid, "interface")
        mock_detach.assert_called_once_with(mock.ANY, mock.ANY, "interface")
        mock_valid.assert_called_once_with(mock.ANY, mock.ANY)

    @mock.patch.object(n_flat.FlatNetwork, 'vif_detach', autospec=True)
    def test_vif_detach_node_locked(self, mock_detach, mock_valid):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          reservation='fake-reserv')
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.vif_detach,
                                self.context, node.uuid, "interface")
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NodeLocked, exc.exc_info[0])
        self.assertFalse(mock_detach.called)
        self.assertFalse(mock_valid.called)

    @mock.patch.object(n_flat.FlatNetwork, 'vif_detach', autospec=True)
    def test_vif_detach_raises_network_error(self, mock_detach,
                                             mock_valid):
        mock_detach.side_effect = exception.NetworkError("BOOM")
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.vif_detach,
                                self.context, node.uuid, "interface")
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NetworkError, exc.exc_info[0])
        mock_valid.assert_called_once_with(mock.ANY, mock.ANY)
        mock_detach.assert_called_once_with(mock.ANY, mock.ANY, "interface")

    @mock.patch.object(n_flat.FlatNetwork, 'vif_detach', autospec=True)
    def test_vif_detach_validate_error(self, mock_detach,
                                       mock_valid):
        mock_valid.side_effect = exception.MissingParameterValue("BOOM")
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.vif_detach,
                                self.context, node.uuid, "interface")
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.MissingParameterValue, exc.exc_info[0])
        mock_valid.assert_called_once_with(mock.ANY, mock.ANY)
        self.assertFalse(mock_detach.called)


@mgr_utils.mock_record_keepalive
class UpdatePortgroupTestCase(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase):
    @mock.patch.object(n_flat.FlatNetwork, 'portgroup_changed', autospec=True)
    @mock.patch.object(n_flat.FlatNetwork, 'validate', autospec=True)
    def test_update_portgroup(self, mock_val, mock_pc):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        portgroup = obj_utils.create_test_portgroup(self.context,
                                                    node_id=node.id,
                                                    extra={'foo': 'bar'})
        new_extra = {'foo': 'baz'}
        portgroup.extra = new_extra
        self.service.update_portgroup(self.context, portgroup)
        portgroup.refresh()
        self.assertEqual(new_extra, portgroup.extra)
        mock_val.assert_called_once_with(mock.ANY, mock.ANY)
        mock_pc.assert_called_once_with(mock.ANY, mock.ANY, portgroup)

    @mock.patch.object(n_flat.FlatNetwork, 'portgroup_changed', autospec=True)
    @mock.patch.object(n_flat.FlatNetwork, 'validate', autospec=True)
    def test_update_portgroup_failure(self, mock_val, mock_pc):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        portgroup = obj_utils.create_test_portgroup(self.context,
                                                    node_id=node.id,
                                                    extra={'foo': 'bar'})
        old_extra = portgroup.extra
        new_extra = {'foo': 'baz'}
        portgroup.extra = new_extra
        mock_pc.side_effect = (exception.FailedToUpdateMacOnPort('boom'))
        self.assertRaises(messaging.rpc.ExpectedException,
                          self.service.update_portgroup,
                          self.context, portgroup)
        portgroup.refresh()
        self.assertEqual(old_extra, portgroup.extra)
        mock_val.assert_called_once_with(mock.ANY, mock.ANY)
        mock_pc.assert_called_once_with(mock.ANY, mock.ANY, portgroup)

    def test_update_portgroup_node_locked(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          reservation='fake-reserv')
        portgroup = obj_utils.create_test_portgroup(self.context,
                                                    node_id=node.id)
        old_extra = portgroup.extra
        portgroup.extra = {'foo': 'baz'}
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.update_portgroup,
                                self.context, portgroup)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NodeLocked, exc.exc_info[0])
        portgroup.refresh()
        self.assertEqual(old_extra, portgroup.extra)

    def test_update_portgroup_to_node_in_deleting_state(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        portgroup = obj_utils.create_test_portgroup(self.context,
                                                    node_id=node.id,
                                                    extra={'foo': 'bar'})
        update_node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.DELETING,
            uuid=uuidutils.generate_uuid())

        old_node_id = portgroup.node_id
        portgroup.node_id = update_node.id
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.update_portgroup,
                                self.context, portgroup)
        self.assertEqual(exception.InvalidState, exc.exc_info[0])
        portgroup.refresh()
        self.assertEqual(old_node_id, portgroup.node_id)

    @mock.patch.object(dbapi.IMPL, 'get_ports_by_portgroup_id', autospec=True)
    @mock.patch.object(n_flat.FlatNetwork, 'portgroup_changed', autospec=True)
    @mock.patch.object(n_flat.FlatNetwork, 'validate', autospec=True)
    def test_update_portgroup_to_node_in_manageable_state(self, mock_val,
                                                          mock_pgc,
                                                          mock_get_ports):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        portgroup = obj_utils.create_test_portgroup(self.context,
                                                    node_id=node.id,
                                                    extra={'foo': 'bar'})
        update_node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.MANAGEABLE,
            uuid=uuidutils.generate_uuid())
        mock_get_ports.return_value = []

        self._start_service()

        portgroup.node_id = update_node.id
        self.service.update_portgroup(self.context, portgroup)
        portgroup.refresh()
        self.assertEqual(update_node.id, portgroup.node_id)
        mock_get_ports.assert_called_once_with(portgroup.id)
        mock_val.assert_called_once_with(mock.ANY, mock.ANY)
        mock_pgc.assert_called_once_with(mock.ANY, mock.ANY, portgroup)

    @mock.patch.object(dbapi.IMPL, 'get_ports_by_portgroup_id', autospec=True)
    @mock.patch.object(n_flat.FlatNetwork, 'portgroup_changed', autospec=True)
    @mock.patch.object(n_flat.FlatNetwork, 'validate', autospec=True)
    def test_update_portgroup_to_node_in_inspect_wait_state(self, mock_val,
                                                            mock_pgc,
                                                            mock_get_ports):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        portgroup = obj_utils.create_test_portgroup(self.context,
                                                    node_id=node.id,
                                                    extra={'foo': 'bar'})
        update_node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.INSPECTWAIT,
            uuid=uuidutils.generate_uuid())
        mock_get_ports.return_value = []

        self._start_service()

        portgroup.node_id = update_node.id
        self.service.update_portgroup(self.context, portgroup)
        portgroup.refresh()
        self.assertEqual(update_node.id, portgroup.node_id)
        mock_get_ports.assert_called_once_with(portgroup.id)
        mock_val.assert_called_once_with(mock.ANY, mock.ANY)
        mock_pgc.assert_called_once_with(mock.ANY, mock.ANY, portgroup)

    @mock.patch.object(dbapi.IMPL, 'get_ports_by_portgroup_id', autospec=True)
    @mock.patch.object(n_flat.FlatNetwork, 'portgroup_changed', autospec=True)
    @mock.patch.object(n_flat.FlatNetwork, 'validate', autospec=True)
    def test_update_portgroup_to_node_in_active_state_and_maintenance(
            self, mock_val, mock_pgc, mock_get_ports):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        portgroup = obj_utils.create_test_portgroup(self.context,
                                                    node_id=node.id,
                                                    extra={'foo': 'bar'})
        update_node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.ACTIVE,
            maintenance=True,
            uuid=uuidutils.generate_uuid())
        mock_get_ports.return_value = []

        self._start_service()

        portgroup.node_id = update_node.id
        self.service.update_portgroup(self.context, portgroup)
        portgroup.refresh()
        self.assertEqual(update_node.id, portgroup.node_id)
        mock_get_ports.assert_called_once_with(portgroup.id)
        mock_val.assert_called_once_with(mock.ANY, mock.ANY)
        mock_pgc.assert_called_once_with(mock.ANY, mock.ANY, portgroup)

    @mock.patch.object(dbapi.IMPL, 'get_ports_by_portgroup_id', autospec=True)
    @mock.patch.object(n_flat.FlatNetwork, 'portgroup_changed', autospec=True)
    @mock.patch.object(n_flat.FlatNetwork, 'validate', autospec=True)
    def test_update_portgroup_association_with_ports(self, mock_val,
                                                     mock_pgc, mock_get_ports):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        portgroup = obj_utils.create_test_portgroup(self.context,
                                                    node_id=node.id,
                                                    extra={'foo': 'bar'})
        update_node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            maintenance=True,
            uuid=uuidutils.generate_uuid())
        mock_get_ports.return_value = ['test_port']

        self._start_service()

        old_node_id = portgroup.node_id
        portgroup.node_id = update_node.id
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.update_portgroup,
                                self.context, portgroup)
        self.assertEqual(exception.PortgroupNotEmpty, exc.exc_info[0])
        portgroup.refresh()
        self.assertEqual(old_node_id, portgroup.node_id)
        mock_get_ports.assert_called_once_with(portgroup.id)
        self.assertFalse(mock_val.called)
        self.assertFalse(mock_pgc.called)


@mgr_utils.mock_record_keepalive
class RaidTestCases(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase):

    driver_name = 'fake-hardware'
    raid_interface = None

    def setUp(self):
        super(RaidTestCases, self).setUp()
        self.node = obj_utils.create_test_node(
            self.context, driver=self.driver_name,
            raid_interface=self.raid_interface,
            provision_state=states.MANAGEABLE)

    def test_get_raid_logical_disk_properties(self):
        self._start_service()
        properties = self.service.get_raid_logical_disk_properties(
            self.context, self.driver_name)
        self.assertIn('raid_level', properties)
        self.assertIn('size_gb', properties)

    def test_set_target_raid_config(self):
        raid_config = {'logical_disks': [{'size_gb': 100, 'raid_level': '1'}]}
        self.service.set_target_raid_config(
            self.context, self.node.uuid, raid_config)
        self.node.refresh()
        self.assertEqual(raid_config, self.node.target_raid_config)

    def test_set_target_raid_config_empty(self):
        self.node.target_raid_config = {'foo': 'bar'}
        self.node.save()
        raid_config = {}
        self.service.set_target_raid_config(
            self.context, self.node.uuid, raid_config)
        self.node.refresh()
        self.assertEqual({}, self.node.target_raid_config)

    def test_set_target_raid_config_invalid_parameter_value(self):
        # Missing raid_level in the below raid config.
        raid_config = {'logical_disks': [{'size_gb': 100}]}
        self.node.target_raid_config = {'foo': 'bar'}
        self.node.save()

        exc = self.assertRaises(
            messaging.rpc.ExpectedException,
            self.service.set_target_raid_config,
            self.context, self.node.uuid, raid_config)

        self.node.refresh()
        self.assertEqual({'foo': 'bar'}, self.node.target_raid_config)
        self.assertEqual(exception.InvalidParameterValue, exc.exc_info[0])


@mgr_utils.mock_record_keepalive
class RaidHardwareTypeTestCases(RaidTestCases):

    driver_name = 'fake-hardware'
    raid_interface = 'fake'

    def test_get_raid_logical_disk_properties_iface_not_supported(self):
        # NOTE(jroll) we don't run this test as get_logical_disk_properties
        # is supported on all RAID implementations, and we cannot have a
        # null interface for a hardware type
        pass

    def test_set_target_raid_config_iface_not_supported(self):
        # NOTE(jroll): it's impossible for a dynamic driver to have a null
        # interface (e.g. node.driver.raid), so this instead tests that
        # if validation fails, we blow up properly.
        # need a different raid interface and a hardware type that supports it
        self.node = obj_utils.create_test_node(
            self.context, driver='manual-management',
            raid_interface='no-raid',
            uuid=uuidutils.generate_uuid(),
            provision_state=states.MANAGEABLE)
        raid_config = {'logical_disks': [{'size_gb': 100, 'raid_level': '1'}]}

        exc = self.assertRaises(
            messaging.rpc.ExpectedException,
            self.service.set_target_raid_config,
            self.context, self.node.uuid, raid_config)
        self.node.refresh()
        self.assertEqual({}, self.node.target_raid_config)
        self.assertEqual(exception.UnsupportedDriverExtension, exc.exc_info[0])
        self.assertIn('manual-management', str(exc.exc_info[1]))


@mock.patch.object(conductor_utils, 'node_power_action', autospec=True)
class ManagerDoSyncPowerStateTestCase(db_base.DbTestCase):
    def setUp(self):
        super(ManagerDoSyncPowerStateTestCase, self).setUp()
        self.service = manager.ConductorManager('hostname', 'test-topic')
        self.driver = mock.Mock(spec_set=drivers_base.BareDriver)
        self.driver.management.detect_vendor.side_effect = \
            exception.UnsupportedDriverExtension
        self.driver.management.get_boot_mode.side_effect = \
            exception.UnsupportedDriverExtension
        self.driver.management.get_secure_boot_state.side_effect = \
            exception.UnsupportedDriverExtension
        self.driver.management.get_node_health.side_effect = \
            exception.UnsupportedDriverExtension
        self.power = self.driver.power
        self.node = obj_utils.create_test_node(
            self.context, driver='fake-hardware', maintenance=False,
            provision_state=states.AVAILABLE,
            instance_uuid=uuidutils.generate_uuid())
        self.task = mock.Mock(spec_set=['context', 'driver', 'node',
                                        'upgrade_lock', 'shared'])
        self.task.context = self.context
        self.task.driver = self.driver
        self.task.node = self.node
        self.task.shared = False
        self.config(force_power_state_during_sync=False, group='conductor')

    def _do_sync_power_state(self, old_power_state, new_power_states,
                             fail_validate=False):
        self.node.power_state = old_power_state
        if not isinstance(new_power_states, (list, tuple)):
            new_power_states = [new_power_states]
        if fail_validate:
            exc = exception.InvalidParameterValue('error')
            self.power.validate.side_effect = exc
        for new_power_state in new_power_states:
            self.node.power_state = old_power_state
            if isinstance(new_power_state, Exception):
                self.power.get_power_state.side_effect = new_power_state
            else:
                self.power.get_power_state.return_value = new_power_state
            count = manager.do_sync_power_state(
                self.task, self.service.power_state_sync_count[self.node.uuid])
            self.service.power_state_sync_count[self.node.uuid] = count

    def test_state_unchanged(self, node_power_action):
        self._do_sync_power_state('fake-power', 'fake-power')

        self.assertFalse(self.power.validate.called)
        self.power.get_power_state.assert_called_once_with(self.task)
        self.assertEqual('fake-power', self.node.power_state)
        self.assertFalse(node_power_action.called)
        self.assertFalse(self.task.upgrade_lock.called)

    def test_state_unchanged_for_fake_node(self, node_power_action):
        self._do_sync_power_state(None, None)

        self.power.validate.assert_called_once_with(self.task)
        self.power.get_power_state.assert_called_once_with(self.task)
        self.assertIsNone(self.node.power_state)
        self.assertFalse(node_power_action.called)
        self.assertFalse(self.task.upgrade_lock.called)

    @mock.patch.object(nova, 'power_update', autospec=True)
    def test_state_not_set(self, mock_power_update, node_power_action):
        self._do_sync_power_state(None, states.POWER_ON)

        self.power.validate.assert_called_once_with(self.task)
        self.power.get_power_state.assert_called_once_with(self.task)
        self.assertFalse(node_power_action.called)
        self.assertEqual(states.POWER_ON, self.node.power_state)
        self.task.upgrade_lock.assert_called_once_with()
        mock_power_update.assert_called_once_with(
            self.task.context, self.node.instance_uuid, states.POWER_ON)

    def test_validate_fail(self, node_power_action):
        self._do_sync_power_state(None, states.POWER_ON,
                                  fail_validate=True)

        self.power.validate.assert_called_once_with(self.task)
        self.assertFalse(self.power.get_power_state.called)
        self.assertFalse(node_power_action.called)
        self.assertIsNone(self.node.power_state)

    def test_get_power_state_fail(self, node_power_action):
        self._do_sync_power_state('fake',
                                  exception.IronicException('foo'))

        self.assertFalse(self.power.validate.called)
        self.power.get_power_state.assert_called_once_with(self.task)
        self.assertFalse(node_power_action.called)
        self.assertEqual('fake', self.node.power_state)
        self.assertEqual(1,
                         self.service.power_state_sync_count[self.node.uuid])

    def test_get_power_state_error(self, node_power_action):
        self._do_sync_power_state('fake', states.ERROR)
        self.assertFalse(self.power.validate.called)
        self.power.get_power_state.assert_called_once_with(self.task)
        self.assertFalse(node_power_action.called)
        self.assertEqual('fake', self.node.power_state)
        self.assertEqual(1,
                         self.service.power_state_sync_count[self.node.uuid])

    @mock.patch.object(nova, 'power_update', autospec=True)
    def test_state_changed_no_sync(self, mock_power_update, node_power_action):
        self._do_sync_power_state(states.POWER_ON, states.POWER_OFF)

        self.assertFalse(self.power.validate.called)
        self.power.get_power_state.assert_called_once_with(self.task)
        self.assertFalse(node_power_action.called)
        self.assertEqual(states.POWER_OFF, self.node.power_state)
        self.task.upgrade_lock.assert_called_once_with()
        mock_power_update.assert_called_once_with(
            self.task.context, self.node.instance_uuid, states.POWER_OFF)

    @mock.patch.object(nova, 'power_update', autospec=True)
    def test_state_changed_no_sync_with_disable_power_off(self,
                                                          mock_power_update,
                                                          node_power_action):
        self.config(force_power_state_during_sync=True, group='conductor')
        self.node.disable_power_off = True
        self._do_sync_power_state(states.POWER_ON, states.POWER_OFF)

        self.power.validate.assert_not_called()
        self.power.get_power_state.assert_called_once_with(self.task)
        node_power_action.assert_not_called()
        self.assertEqual(states.POWER_OFF, self.node.power_state)
        self.task.upgrade_lock.assert_called_once_with()
        mock_power_update.assert_called_once_with(
            self.task.context, self.node.instance_uuid, states.POWER_OFF)

    @mock.patch('ironic.objects.node.NodeCorrectedPowerStateNotification',
                autospec=True)
    @mock.patch.object(nova, 'power_update', autospec=True)
    def test_state_changed_no_sync_notify(self, mock_power_update, mock_notif,
                                          node_power_action):
        # Required for exception handling
        mock_notif.__name__ = 'NodeCorrectedPowerStateNotification'

        self._do_sync_power_state(states.POWER_ON, states.POWER_OFF)

        self.assertFalse(self.power.validate.called)
        self.power.get_power_state.assert_called_once_with(self.task)
        self.assertFalse(node_power_action.called)
        self.assertEqual(states.POWER_OFF, self.node.power_state)
        self.task.upgrade_lock.assert_called_once_with()

        # 1 notification should be sent:
        # baremetal.node.power_state_updated.success, indicating the DB was
        # updated to reflect the actual node power state
        mock_notif.assert_called_once_with(publisher=mock.ANY,
                                           event_type=mock.ANY,
                                           level=mock.ANY,
                                           payload=mock.ANY)
        mock_notif.return_value.emit.assert_called_once_with(mock.ANY)

        notif_args = mock_notif.call_args[1]
        self.assertNotificationEqual(
            notif_args, 'ironic-conductor', CONF.host,
            'baremetal.node.power_state_corrected.success',
            obj_fields.NotificationLevel.INFO)
        mock_power_update.assert_called_once_with(
            self.task.context, self.node.instance_uuid, states.POWER_OFF)

    def test_state_changed_sync(self, node_power_action):
        self.config(force_power_state_during_sync=True, group='conductor')
        self.config(power_state_sync_max_retries=1, group='conductor')

        self._do_sync_power_state(states.POWER_ON, states.POWER_OFF)

        self.assertFalse(self.power.validate.called)
        self.power.get_power_state.assert_called_once_with(self.task)
        node_power_action.assert_called_once_with(self.task, states.POWER_ON)
        self.assertEqual(states.POWER_ON, self.node.power_state)
        self.task.upgrade_lock.assert_called_once_with()

    def test_state_changed_sync_failed(self, node_power_action):
        self.config(force_power_state_during_sync=True, group='conductor')

        node_power_action.side_effect = exception.IronicException('test')
        self._do_sync_power_state(states.POWER_ON, states.POWER_OFF)

        # Just testing that this test doesn't raise.
        self.assertFalse(self.power.validate.called)
        self.power.get_power_state.assert_called_once_with(self.task)
        node_power_action.assert_called_once_with(self.task, states.POWER_ON)
        self.assertEqual(states.POWER_ON, self.node.power_state)
        self.assertEqual(1,
                         self.service.power_state_sync_count[self.node.uuid])

    @mock.patch.object(nova, 'power_update', autospec=True)
    def test_no_power_sync_support(self, mock_power_update, node_power_action):
        self.config(force_power_state_during_sync=True, group='conductor')
        self.power.supports_power_sync.return_value = False

        self._do_sync_power_state(states.POWER_ON, states.POWER_OFF)

        self.assertFalse(self.power.validate.called)
        self.power.get_power_state.assert_called_once_with(self.task)
        self.assertFalse(node_power_action.called)
        self.assertEqual(states.POWER_OFF, self.node.power_state)
        self.task.upgrade_lock.assert_called_once_with()
        mock_power_update.assert_called_once_with(
            self.task.context, self.node.instance_uuid, states.POWER_OFF)

    @mock.patch.object(nova, 'power_update', autospec=True)
    def test_max_retries_exceeded(self, mock_power_update, node_power_action):
        self.config(force_power_state_during_sync=True, group='conductor')
        self.config(power_state_sync_max_retries=1, group='conductor')

        self._do_sync_power_state(states.POWER_ON, [states.POWER_OFF,
                                                    states.POWER_OFF])

        self.assertFalse(self.power.validate.called)
        power_exp_calls = [mock.call(self.task)] * 2
        self.assertEqual(power_exp_calls,
                         self.power.get_power_state.call_args_list)
        node_power_action.assert_called_once_with(self.task, states.POWER_ON)
        self.assertEqual(states.POWER_OFF, self.node.power_state)
        self.assertEqual(2,
                         self.service.power_state_sync_count[self.node.uuid])
        self.assertTrue(self.node.maintenance)
        self.assertIsNotNone(self.node.maintenance_reason)
        self.assertEqual('power failure', self.node.fault)
        mock_power_update.assert_called_once_with(
            self.task.context, self.node.instance_uuid, states.POWER_OFF)

    @mock.patch.object(nova, 'power_update', autospec=True)
    def test_max_retries_exceeded2(self, mock_power_update, node_power_action):
        self.config(force_power_state_during_sync=True, group='conductor')
        self.config(power_state_sync_max_retries=2, group='conductor')

        self._do_sync_power_state(states.POWER_ON, [states.POWER_OFF,
                                                    states.POWER_OFF,
                                                    states.POWER_OFF])

        self.assertFalse(self.power.validate.called)
        power_exp_calls = [mock.call(self.task)] * 3
        self.assertEqual(power_exp_calls,
                         self.power.get_power_state.call_args_list)
        npa_exp_calls = [mock.call(self.task, states.POWER_ON)] * 2
        self.assertEqual(npa_exp_calls, node_power_action.call_args_list)
        self.assertEqual(states.POWER_OFF, self.node.power_state)
        self.assertEqual(3,
                         self.service.power_state_sync_count[self.node.uuid])
        self.assertTrue(self.node.maintenance)
        self.assertEqual('power failure', self.node.fault)
        mock_power_update.assert_called_once_with(
            self.task.context, self.node.instance_uuid, states.POWER_OFF)

    @mock.patch.object(nova, 'power_update', autospec=True)
    def test_max_retries_exceeded_preserve_admin_intent(self,
                                                        mock_power_update,
                                                        node_power_action):
        self.config(force_power_state_during_sync=True, group='conductor')
        self.config(power_state_sync_max_retries=1, group='conductor')

        self.node.maintenance = True
        self.node.maintenance_reason = 'admin set maintenance'
        self.node.fault = None

        self._do_sync_power_state(
            states.POWER_ON,
            [exception.IronicException('foo'),
             exception.IronicException('foo')]
        )

        self.assertFalse(self.power.validate.called)
        power_exp_calls = [mock.call(self.task)] * 2
        self.assertEqual(power_exp_calls,
                         self.power.get_power_state.call_args_list)
        node_power_action.assert_not_called()
        self.assertIsNone(self.node.power_state)
        self.assertEqual(2,
                         self.service.power_state_sync_count[self.node.uuid])

        self.assertTrue(self.node.maintenance)
        self.assertEqual('admin set maintenance', self.node.maintenance_reason)
        self.assertIsNone(self.node.fault)
        mock_power_update.assert_called_once_with(
            self.task.context, self.node.instance_uuid, None)

    @mock.patch('ironic.objects.node.NodeCorrectedPowerStateNotification',
                autospec=True)
    @mock.patch.object(nova, 'power_update', autospec=True)
    def test_max_retries_exceeded_notify(self, mock_power_update,
                                         mock_notif, node_power_action):
        self.config(force_power_state_during_sync=True, group='conductor')
        self.config(power_state_sync_max_retries=1, group='conductor')
        # Required for exception handling
        mock_notif.__name__ = 'NodeCorrectedPowerStateNotification'

        self._do_sync_power_state(states.POWER_ON, [states.POWER_OFF,
                                                    states.POWER_OFF])
        # 1 notification should be sent:
        # baremetal.node.power_state_corrected.success, indicating
        # the DB was updated to reflect the actual node power state
        mock_notif.assert_called_once_with(publisher=mock.ANY,
                                           event_type=mock.ANY,
                                           level=mock.ANY,
                                           payload=mock.ANY)
        mock_notif.return_value.emit.assert_called_once_with(mock.ANY)

        notif_args = mock_notif.call_args[1]
        self.assertNotificationEqual(
            notif_args, 'ironic-conductor', CONF.host,
            'baremetal.node.power_state_corrected.success',
            obj_fields.NotificationLevel.INFO)
        mock_power_update.assert_called_once_with(
            self.task.context, self.node.instance_uuid, states.POWER_OFF)

    def test_retry_then_success(self, node_power_action):
        self.config(force_power_state_during_sync=True, group='conductor')
        self.config(power_state_sync_max_retries=2, group='conductor')

        self._do_sync_power_state(states.POWER_ON, [states.POWER_OFF,
                                                    states.POWER_OFF,
                                                    states.POWER_ON])

        self.assertFalse(self.power.validate.called)
        power_exp_calls = [mock.call(self.task)] * 3
        self.assertEqual(power_exp_calls,
                         self.power.get_power_state.call_args_list)
        npa_exp_calls = [mock.call(self.task, states.POWER_ON)] * 2
        self.assertEqual(npa_exp_calls, node_power_action.call_args_list)
        self.assertEqual(states.POWER_ON, self.node.power_state)
        self.assertEqual(0,
                         self.service.power_state_sync_count[self.node.uuid])

    def test_power_state_sync_max_retries_gps_exception(self,
                                                        node_power_action):
        self.config(power_state_sync_max_retries=2, group='conductor')
        self.service.power_state_sync_count[self.node.uuid] = 2

        node_power_action.side_effect = exception.IronicException('test')
        self._do_sync_power_state('fake',
                                  exception.IronicException('SpongeBob'))

        self.assertFalse(self.power.validate.called)
        self.power.get_power_state.assert_called_once_with(self.task)

        self.assertIsNone(self.node.power_state)
        self.assertTrue(self.node.maintenance)

        self.assertFalse(node_power_action.called)
        # make sure the actual error is in the last_error attribute
        self.assertIn('SpongeBob', self.node.last_error)

    def test_maintenance_on_upgrade_lock(self, node_power_action):
        self.node.maintenance = True

        self._do_sync_power_state(states.POWER_ON, states.POWER_OFF)

        self.assertFalse(self.power.validate.called)
        self.power.get_power_state.assert_called_once_with(self.task)
        self.assertEqual(states.POWER_ON, self.node.power_state)
        self.assertFalse(node_power_action.called)
        self.task.upgrade_lock.assert_called_once_with()

    def test_wrong_provision_state_on_upgrade_lock(self, node_power_action):
        self.node.provision_state = states.DEPLOYWAIT

        self._do_sync_power_state(states.POWER_ON, states.POWER_OFF)

        self.assertFalse(self.power.validate.called)
        self.power.get_power_state.assert_called_once_with(self.task)
        self.assertEqual(states.POWER_ON, self.node.power_state)
        self.assertFalse(node_power_action.called)
        self.task.upgrade_lock.assert_called_once_with()

    def test_correct_power_state_on_upgrade_lock(self, node_power_action):
        def _fake_upgrade():
            self.node.power_state = states.POWER_OFF

        self.task.upgrade_lock.side_effect = _fake_upgrade

        self._do_sync_power_state(states.POWER_ON, states.POWER_OFF)

        self.assertFalse(self.power.validate.called)
        self.power.get_power_state.assert_called_once_with(self.task)
        self.assertFalse(node_power_action.called)
        self.task.upgrade_lock.assert_called_once_with()

    @mock.patch.object(nova, 'power_update', autospec=True)
    def test_vendor_detection(self, mock_power_update, node_power_action):
        self.driver.management.detect_vendor.side_effect = [
            "Fake Inc."
        ]
        self._do_sync_power_state(states.POWER_ON, states.POWER_OFF)

        self.assertFalse(self.power.validate.called)
        self.power.get_power_state.assert_called_once_with(self.task)
        self.assertFalse(node_power_action.called)
        self.assertEqual(states.POWER_OFF, self.node.power_state)
        # node_cache_vendor calls upgrade_lock, then power update does it once
        # more (which is safe because TaskManager checks its state)
        self.assertEqual(2, self.task.upgrade_lock.call_count)
        mock_power_update.assert_called_once_with(
            self.task.context, self.node.instance_uuid, states.POWER_OFF)
        self.assertEqual("Fake Inc.", self.node.properties['vendor'])

    def test_last_error_cleared_when_node_history_enabled(
            self, node_power_action):
        self.config(group='conductor', node_history=True)
        self.node.last_error = 'BMC unreachable'
        self._do_sync_power_state(states.POWER_ON, states.POWER_ON)
        self.assertIsNone(self.node.last_error)

    def test_last_error_not_cleared_when_node_history_disabled(
            self, node_power_action):
        self.config(group='conductor', node_history=False)
        self.node.last_error = 'BMC unreachable'
        self._do_sync_power_state(states.POWER_ON, states.POWER_ON)
        self.assertEqual('BMC unreachable', self.node.last_error)

    @mock.patch.object(nova, 'power_update', autospec=True)
    def test_clears_last_error_on_unknown_to_known_with_history(
            self, mock_power_update, node_power_action):
        self.config(group='conductor', node_history=True)
        self.node.power_state = None
        self.node.last_error = "Failed to fetch"
        self._do_sync_power_state(None, states.POWER_ON)
        self.assertIsNone(self.node.last_error)

    @mock.patch.object(nova, 'power_update', autospec=True)
    def test_does_not_clear_last_error_on_unknown_with_history_disabled(
            self, mock_power_update, node_power_action):
        self.config(group='conductor', node_history=False)
        self.node.power_state = None
        self.node.last_error = "Failed to fetch"
        self._do_sync_power_state(None, states.POWER_ON)
        self.assertEqual("Failed to fetch", self.node.last_error)

    @mock.patch.object(nova, 'power_update', autospec=True)
    def test_last_error_not_cleared_when_power_states_do_not_match(
            self, mock_power_update, node_power_action):
        self.config(group='conductor', node_history=True)
        self.node.power_state = states.POWER_ON
        self.node.last_error = "BMC timeout"

        self._do_sync_power_state(states.POWER_ON, states.POWER_OFF)

        self.assertEqual("BMC timeout", self.node.last_error)

    def test_health_monitoring_enabled_by_default(self, node_power_action):
        # Health monitoring should run by default
        # Mock get_node_health to verify it gets called
        self.driver.management.get_node_health.return_value = \
            health_states.HealthState.OK
        self._do_sync_power_state(states.POWER_ON, states.POWER_ON)
        self.driver.management.get_node_health.assert_called_once_with(
            self.task)

    def test_health_monitoring_disabled_by_config(self, node_power_action):
        # Health monitoring should not run when disabled
        self.config(group='conductor', enable_health_monitoring=False)
        # Mock get_node_health - it should not be called
        self.driver.management.get_node_health.return_value = \
            health_states.HealthState.OK
        self._do_sync_power_state(states.POWER_ON, states.POWER_ON)
        self.driver.management.get_node_health.assert_not_called()


@mock.patch.object(waiters, 'wait_for_all',
                   new=mock.MagicMock(return_value=(0, 0)))
@mock.patch.object(manager.ConductorManager, '_spawn_worker',
                   new=lambda self, fun, *args: fun(*args))
@mock.patch.object(manager, 'do_sync_power_state', autospec=True)
@mock.patch.object(task_manager, 'acquire', autospec=True)
@mock.patch.object(manager.ConductorManager, '_mapped_to_this_conductor',
                   autospec=True)
@mock.patch.object(dbapi.IMPL, 'get_nodeinfo_list', autospec=True)
class ManagerSyncPowerStatesTestCase(mgr_utils.CommonMixIn,
                                     db_base.DbTestCase):
    def setUp(self):
        super(ManagerSyncPowerStatesTestCase, self).setUp()
        self.service = manager.ConductorManager('hostname', 'test-topic')
        self.service.dbapi = self.dbapi
        self.node = self._create_node(driver_internal_info={})
        self.filters = {'maintenance': False}
        self.columns = ['uuid', 'driver', 'conductor_group', 'id']

    def test_node_not_mapped(self, get_nodeinfo_mock,
                             mapped_mock, acquire_mock, sync_mock):
        get_nodeinfo_mock.return_value = self._get_nodeinfo_list_response()
        mapped_mock.return_value = False

        self.service._sync_power_states(self.context)

        get_nodeinfo_mock.assert_called_once_with(
            columns=self.columns, filters=self.filters)
        mapped_mock.assert_called_once_with(self.service,
                                            self.node.uuid,
                                            self.node.driver,
                                            self.node.conductor_group)
        self.assertFalse(acquire_mock.called)
        self.assertFalse(sync_mock.called)

    def test_node_locked_on_acquire(self, get_nodeinfo_mock,
                                    mapped_mock, acquire_mock, sync_mock):
        get_nodeinfo_mock.return_value = self._get_nodeinfo_list_response()
        mapped_mock.return_value = True
        task = self._create_task(
            node_attrs=dict(reservation='host1', uuid=self.node.uuid))
        acquire_mock.side_effect = self._get_acquire_side_effect(task)

        self.service._sync_power_states(self.context)

        get_nodeinfo_mock.assert_called_once_with(
            columns=self.columns, filters=self.filters)
        mapped_mock.assert_called_once_with(self.service,
                                            self.node.uuid,
                                            self.node.driver,
                                            self.node.conductor_group)
        acquire_mock.assert_called_once_with(self.context, self.node.uuid,
                                             purpose=mock.ANY,
                                             shared=True)
        self.assertFalse(sync_mock.called)

    def test_node_in_deploywait_on_acquire(self, get_nodeinfo_mock,
                                           mapped_mock, acquire_mock,
                                           sync_mock):
        get_nodeinfo_mock.return_value = self._get_nodeinfo_list_response()
        mapped_mock.return_value = True
        task = self._create_task(
            node_attrs=dict(provision_state=states.DEPLOYWAIT,
                            target_provision_state=states.ACTIVE,
                            uuid=self.node.uuid))
        acquire_mock.side_effect = self._get_acquire_side_effect(task)

        self.service._sync_power_states(self.context)

        get_nodeinfo_mock.assert_called_once_with(
            columns=self.columns, filters=self.filters)
        mapped_mock.assert_called_once_with(self.service,
                                            self.node.uuid,
                                            self.node.driver,
                                            self.node.conductor_group)
        acquire_mock.assert_called_once_with(self.context, self.node.uuid,
                                             purpose=mock.ANY,
                                             shared=True)
        self.assertFalse(sync_mock.called)

    def test_node_in_enroll_on_acquire(self, get_nodeinfo_mock, mapped_mock,
                                       acquire_mock, sync_mock):
        get_nodeinfo_mock.return_value = self._get_nodeinfo_list_response()
        mapped_mock.return_value = True
        task = self._create_task(
            node_attrs=dict(provision_state=states.ENROLL,
                            target_provision_state=states.NOSTATE,
                            uuid=self.node.uuid))
        acquire_mock.side_effect = self._get_acquire_side_effect(task)

        self.service._sync_power_states(self.context)

        get_nodeinfo_mock.assert_called_once_with(
            columns=self.columns, filters=self.filters)
        mapped_mock.assert_called_once_with(self.service,
                                            self.node.uuid,
                                            self.node.driver,
                                            self.node.conductor_group)
        acquire_mock.assert_called_once_with(self.context, self.node.uuid,
                                             purpose=mock.ANY,
                                             shared=True)
        self.assertFalse(sync_mock.called)

    def test_node_in_power_transition_on_acquire(self, get_nodeinfo_mock,
                                                 mapped_mock, acquire_mock,
                                                 sync_mock):
        get_nodeinfo_mock.return_value = self._get_nodeinfo_list_response()
        mapped_mock.return_value = True
        task = self._create_task(
            node_attrs=dict(target_power_state=states.POWER_ON,
                            uuid=self.node.uuid))
        acquire_mock.side_effect = self._get_acquire_side_effect(task)

        self.service._sync_power_states(self.context)

        get_nodeinfo_mock.assert_called_once_with(
            columns=self.columns, filters=self.filters)
        mapped_mock.assert_called_once_with(self.service,
                                            self.node.uuid,
                                            self.node.driver,
                                            self.node.conductor_group)
        acquire_mock.assert_called_once_with(self.context, self.node.uuid,
                                             purpose=mock.ANY,
                                             shared=True)
        self.assertFalse(sync_mock.called)

    def test_node_in_maintenance_on_acquire(self, get_nodeinfo_mock,
                                            mapped_mock, acquire_mock,
                                            sync_mock):
        get_nodeinfo_mock.return_value = self._get_nodeinfo_list_response()
        mapped_mock.return_value = True
        task = self._create_task(
            node_attrs=dict(maintenance=True, uuid=self.node.uuid))
        acquire_mock.side_effect = self._get_acquire_side_effect(task)

        self.service._sync_power_states(self.context)

        get_nodeinfo_mock.assert_called_once_with(
            columns=self.columns, filters=self.filters)
        mapped_mock.assert_called_once_with(self.service,
                                            self.node.uuid,
                                            self.node.driver,
                                            self.node.conductor_group)
        acquire_mock.assert_called_once_with(self.context, self.node.uuid,
                                             purpose=mock.ANY,
                                             shared=True)
        self.assertFalse(sync_mock.called)

    def test_node_disappears_on_acquire(self, get_nodeinfo_mock,
                                        mapped_mock, acquire_mock, sync_mock):
        get_nodeinfo_mock.return_value = self._get_nodeinfo_list_response()
        mapped_mock.return_value = True
        acquire_mock.side_effect = exception.NodeNotFound(node=self.node.uuid,
                                                          host='fake')

        self.service._sync_power_states(self.context)

        get_nodeinfo_mock.assert_called_once_with(
            columns=self.columns, filters=self.filters)
        mapped_mock.assert_called_once_with(self.service,
                                            self.node.uuid,
                                            self.node.driver,
                                            self.node.conductor_group)
        acquire_mock.assert_called_once_with(self.context, self.node.uuid,
                                             purpose=mock.ANY,
                                             shared=True)
        self.assertFalse(sync_mock.called)

    def test_single_node(self, get_nodeinfo_mock,
                         mapped_mock, acquire_mock, sync_mock):
        get_nodeinfo_mock.return_value = self._get_nodeinfo_list_response()
        mapped_mock.return_value = True
        task = self._create_task(node_attrs=dict(uuid=self.node.uuid,
                                                 driver_internal_info={}))
        acquire_mock.side_effect = self._get_acquire_side_effect(task)

        self.service._sync_power_states(self.context)

        get_nodeinfo_mock.assert_called_once_with(
            columns=self.columns, filters=self.filters)
        mapped_mock.assert_called_once_with(self.service,
                                            self.node.uuid,
                                            self.node.driver,
                                            self.node.conductor_group)
        acquire_mock.assert_called_once_with(self.context, self.node.uuid,
                                             purpose=mock.ANY,
                                             shared=True)
        sync_mock.assert_called_once_with(task, mock.ANY)

    def test_single_node_with_firmware_update(self, get_nodeinfo_mock,
                                              mapped_mock, acquire_mock,
                                              sync_mock):
        get_nodeinfo_mock.return_value = self._get_nodeinfo_list_response()
        mapped_mock.return_value = True
        task = self._create_task(
            node_attrs=dict(uuid=self.node.uuid,
                            driver_internal_info={'redfish_fw_updates': {
                                'component': 'bmc', 'url': 'http://example.com'}}))
        acquire_mock.side_effect = self._get_acquire_side_effect(task)

        self.service._sync_power_states(self.context)

        get_nodeinfo_mock.assert_called_once_with(
            columns=self.columns, filters=self.filters)
        mapped_mock.assert_called_once_with(self.service,
                                            self.node.uuid,
                                            self.node.driver,
                                            self.node.conductor_group)
        acquire_mock.assert_called_once_with(self.context, self.node.uuid,
                                             purpose=mock.ANY,
                                             shared=True)
        self.assertFalse(sync_mock.called)

    def test_single_node_adopt_failed(self, get_nodeinfo_mock,
                                      mapped_mock, acquire_mock, sync_mock):
        get_nodeinfo_mock.return_value = self._get_nodeinfo_list_response()
        mapped_mock.return_value = True
        task = self._create_task(
            node_attrs=dict(uuid=self.node.uuid,
                            provision_state=states.ADOPTFAIL))
        acquire_mock.side_effect = self._get_acquire_side_effect(task)

        self.service._sync_power_states(self.context)

        get_nodeinfo_mock.assert_called_once_with(
            columns=self.columns, filters=self.filters)
        mapped_mock.assert_called_once_with(self.service,
                                            self.node.uuid,
                                            self.node.driver,
                                            self.node.conductor_group)
        acquire_mock.assert_called_once_with(self.context, self.node.uuid,
                                             purpose=mock.ANY,
                                             shared=True)
        sync_mock.assert_not_called()

    def test__sync_power_state_multiple_nodes(self, get_nodeinfo_mock,
                                              mapped_mock, acquire_mock,
                                              sync_mock):
        # Create 8 nodes:
        # 1st node: Should acquire and try to sync
        # 2nd node: Not mapped to this conductor
        # 3rd node: In DEPLOYWAIT provision_state
        # 4th node: In maintenance mode
        # 5th node: Is in power transition
        # 6th node: Disappears after getting nodeinfo list
        # 7th node: Should acquire and try to sync
        # 8th node: do_sync_power_state raises NodeLocked
        nodes = []
        node_attrs = {}
        mapped_map = {}
        for i in range(1, 8):
            attrs = {'id': i,
                     'uuid': uuidutils.generate_uuid(),
                     'driver_internal_info': {}}
            if i == 3:
                attrs['provision_state'] = states.DEPLOYWAIT
                attrs['target_provision_state'] = states.ACTIVE
            elif i == 4:
                attrs['maintenance'] = True
            elif i == 5:
                attrs['target_power_state'] = states.POWER_ON

            n = self._create_node(**attrs)
            nodes.append(n)
            node_attrs[n.uuid] = attrs
            mapped_map[n.uuid] = False if i == 2 else True

        tasks = [self._create_task(node_attrs=node_attrs[x.uuid])
                 for x in nodes if x.id != 2]
        # not found during acquire (4 = index of Node6 after removing Node2)
        tasks[4] = exception.NodeNotFound(node=6)
        sync_results = [0] * 7 + [exception.NodeLocked(node=8, host='')]

        get_nodeinfo_mock.return_value = (
            self._get_nodeinfo_list_response(nodes))
        mapped_mock.side_effect = lambda q, x, y, z: mapped_map[x]
        acquire_mock.side_effect = self._get_acquire_side_effect(tasks)
        sync_mock.side_effect = sync_results

        with mock.patch.object(time, 'sleep', autospec=True) as sleep_mock:
            self.service._sync_power_states(self.context)
            # Ensure we've yielded on every iteration, except for node
            # not mapped to this conductor
            self.assertEqual(len(nodes) - 1, sleep_mock.call_count)

        get_nodeinfo_mock.assert_called_once_with(
            columns=self.columns, filters=self.filters)
        mapped_calls = [mock.call(self.service, x.uuid, x.driver,
                                  x.conductor_group) for x in nodes]
        self.assertEqual(mapped_calls, mapped_mock.call_args_list)
        acquire_calls = [mock.call(self.context, x.uuid,
                                   purpose=mock.ANY,
                                   shared=True)
                         for x in nodes if x.id != 2]
        self.assertEqual(acquire_calls, acquire_mock.call_args_list)
        # Nodes 1 and 7 (5 = index of Node7 after removing Node2)
        sync_calls = [mock.call(tasks[0], mock.ANY),
                      mock.call(tasks[5], mock.ANY)]
        self.assertEqual(sync_calls, sync_mock.call_args_list)


@mock.patch.object(task_manager, 'acquire', autospec=True)
@mock.patch.object(manager.ConductorManager, '_mapped_to_this_conductor',
                   autospec=True)
@mock.patch.object(dbapi.IMPL, 'get_nodeinfo_list', autospec=True)
class ManagerPowerRecoveryTestCase(mgr_utils.CommonMixIn,
                                   db_base.DbTestCase):
    def setUp(self):
        super(ManagerPowerRecoveryTestCase, self).setUp()
        self.service = manager.ConductorManager('hostname', 'test-topic')
        self.service.dbapi = self.dbapi
        self.driver = mock.Mock(spec_set=drivers_base.BareDriver)
        self.power = self.driver.power
        self.task = mock.Mock(spec_set=['context', 'driver', 'node',
                                        'upgrade_lock', 'shared'])
        self.node = self._create_node(maintenance=True,
                                      fault='power failure',
                                      maintenance_reason='Unreachable BMC')
        self.task.node = self.node
        self.task.driver = self.driver
        self.filters = {'maintenance': True,
                        'fault': 'power failure'}
        self.columns = ['uuid', 'driver', 'conductor_group']

    def test_node_not_mapped(self, get_nodeinfo_mock,
                             mapped_mock, acquire_mock):
        get_nodeinfo_mock.return_value = self._get_nodeinfo_list_response()
        mapped_mock.return_value = False

        self.service._power_failure_recovery(self.context)

        get_nodeinfo_mock.assert_called_once_with(
            columns=self.columns, filters=self.filters)
        mapped_mock.assert_called_once_with(self.service,
                                            self.node.uuid,
                                            self.node.driver,
                                            self.node.conductor_group)
        self.assertFalse(acquire_mock.called)
        self.assertFalse(self.power.validate.called)

    def _power_failure_recovery(self, node_dict, get_nodeinfo_mock,
                                mapped_mock, acquire_mock):
        get_nodeinfo_mock.return_value = self._get_nodeinfo_list_response()
        mapped_mock.return_value = True

        task = self._create_task(node_attrs=node_dict)
        acquire_mock.side_effect = self._get_acquire_side_effect(task)

        self.service._power_failure_recovery(self.context)

        get_nodeinfo_mock.assert_called_once_with(
            columns=self.columns, filters=self.filters)
        mapped_mock.assert_called_once_with(self.service,
                                            self.node.uuid,
                                            self.node.driver,
                                            self.node.conductor_group)
        acquire_mock.assert_called_once_with(self.context, self.node.uuid,
                                             purpose=mock.ANY,
                                             shared=True)
        self.assertFalse(self.power.validate.called)

    def test_node_locked_on_acquire(self, get_nodeinfo_mock, mapped_mock,
                                    acquire_mock):
        node_dict = dict(reservation='host1', uuid=self.node.uuid)
        self._power_failure_recovery(node_dict, get_nodeinfo_mock,
                                     mapped_mock, acquire_mock)

    def test_node_in_enroll_on_acquire(self, get_nodeinfo_mock, mapped_mock,
                                       acquire_mock):
        node_dict = dict(provision_state=states.ENROLL,
                         target_provision_state=states.NOSTATE,
                         maintenance=True, uuid=self.node.uuid)
        self._power_failure_recovery(node_dict, get_nodeinfo_mock,
                                     mapped_mock, acquire_mock)

    def test_node_in_power_transition_on_acquire(self, get_nodeinfo_mock,
                                                 mapped_mock, acquire_mock):
        node_dict = dict(target_power_state=states.POWER_ON,
                         maintenance=True, uuid=self.node.uuid)
        self._power_failure_recovery(node_dict, get_nodeinfo_mock,
                                     mapped_mock, acquire_mock)

    def test_node_not_in_maintenance_on_acquire(self, get_nodeinfo_mock,
                                                mapped_mock, acquire_mock):
        node_dict = dict(maintenance=False, uuid=self.node.uuid)
        self._power_failure_recovery(node_dict, get_nodeinfo_mock,
                                     mapped_mock, acquire_mock)

    def test_node_disappears_on_acquire(self, get_nodeinfo_mock,
                                        mapped_mock, acquire_mock):
        get_nodeinfo_mock.return_value = self._get_nodeinfo_list_response()
        mapped_mock.return_value = True
        acquire_mock.side_effect = exception.NodeNotFound(node=self.node.uuid,
                                                          host='fake')

        self.service._power_failure_recovery(self.context)

        get_nodeinfo_mock.assert_called_once_with(
            columns=self.columns, filters=self.filters)
        mapped_mock.assert_called_once_with(self.service,
                                            self.node.uuid,
                                            self.node.driver,
                                            self.node.conductor_group)
        acquire_mock.assert_called_once_with(self.context, self.node.uuid,
                                             purpose=mock.ANY,
                                             shared=True)
        self.assertFalse(self.power.validate.called)

    @mock.patch.object(notification_utils,
                       'emit_power_state_corrected_notification',
                       autospec=True)
    @mock.patch.object(nova, 'power_update', autospec=True)
    def test_node_recovery_success(self, mock_power_update, notify_mock,
                                   get_nodeinfo_mock, mapped_mock,
                                   acquire_mock):
        self.node.power_state = states.POWER_ON
        get_nodeinfo_mock.return_value = self._get_nodeinfo_list_response()
        mapped_mock.return_value = True
        acquire_mock.side_effect = self._get_acquire_side_effect(self.task)
        self.power.get_power_state.return_value = states.POWER_OFF

        self.service._power_failure_recovery(self.context)

        get_nodeinfo_mock.assert_called_once_with(
            columns=self.columns, filters=self.filters)
        mapped_mock.assert_called_once_with(self.service,
                                            self.node.uuid,
                                            self.node.driver,
                                            self.node.conductor_group)
        acquire_mock.assert_called_once_with(self.context, self.node.uuid,
                                             purpose=mock.ANY,
                                             shared=True)
        self.power.validate.assert_called_once_with(self.task)
        self.power.get_power_state.assert_called_once_with(self.task)
        self.task.upgrade_lock.assert_called_once_with()
        self.assertFalse(self.node.maintenance)
        self.assertIsNone(self.node.fault)
        self.assertIsNone(self.node.maintenance_reason)
        self.assertEqual(states.POWER_OFF, self.node.power_state)
        notify_mock.assert_called_once_with(self.task, states.POWER_ON)
        mock_power_update.assert_called_once_with(
            self.task.context, self.node.instance_uuid, states.POWER_OFF)

    def test_node_recovery_failed(self, get_nodeinfo_mock,
                                  mapped_mock, acquire_mock):
        get_nodeinfo_mock.return_value = self._get_nodeinfo_list_response()
        mapped_mock.return_value = True
        acquire_mock.side_effect = self._get_acquire_side_effect(self.task)
        self.power.get_power_state.return_value = states.ERROR

        self.service._power_failure_recovery(self.context)

        get_nodeinfo_mock.assert_called_once_with(
            columns=self.columns, filters=self.filters)
        mapped_mock.assert_called_once_with(self.service,
                                            self.node.uuid,
                                            self.node.driver,
                                            self.node.conductor_group)
        acquire_mock.assert_called_once_with(self.context, self.node.uuid,
                                             purpose=mock.ANY,
                                             shared=True)
        self.power.validate.assert_called_once_with(self.task)
        self.power.get_power_state.assert_called_once_with(self.task)
        self.assertFalse(self.task.upgrade_lock.called)
        self.assertTrue(self.node.maintenance)
        self.assertEqual('power failure', self.node.fault)
        self.assertEqual('Unreachable BMC', self.node.maintenance_reason)


@mock.patch.object(task_manager, 'acquire', autospec=True)
@mock.patch.object(manager.ConductorManager, '_mapped_to_this_conductor',
                   autospec=True)
@mock.patch.object(dbapi.IMPL, 'get_nodeinfo_list', autospec=True)
class ManagerCheckDeployTimeoutsTestCase(mgr_utils.CommonMixIn,
                                         db_base.DbTestCase):
    def setUp(self):
        super(ManagerCheckDeployTimeoutsTestCase, self).setUp()
        self.config(deploy_callback_timeout=300, group='conductor')
        self.service = manager.ConductorManager('hostname', 'test-topic')
        self.service.dbapi = self.dbapi

        self.node = self._create_node(provision_state=states.DEPLOYWAIT,
                                      target_provision_state=states.ACTIVE)
        self.task = self._create_task(node=self.node)

        self.node2 = self._create_node(provision_state=states.DEPLOYWAIT,
                                       target_provision_state=states.ACTIVE)
        self.task2 = self._create_task(node=self.node2)

        self.filters = {'reserved': False, 'maintenance': False,
                        'provisioned_before': 300,
                        'provision_state': states.DEPLOYWAIT}
        self.columns = ['uuid', 'driver', 'conductor_group']

    def _assert_get_nodeinfo_args(self, get_nodeinfo_mock):
        get_nodeinfo_mock.assert_called_once_with(
            columns=self.columns, filters=self.filters,
            sort_key='provision_updated_at', sort_dir='asc')

    def test_not_mapped(self, get_nodeinfo_mock, mapped_mock, acquire_mock):
        get_nodeinfo_mock.return_value = self._get_nodeinfo_list_response()
        mapped_mock.return_value = False

        self.service._check_deploy_timeouts(self.context)

        self._assert_get_nodeinfo_args(get_nodeinfo_mock)
        mapped_mock.assert_called_once_with(self.service,
                                            self.node.uuid, self.node.driver,
                                            self.node.conductor_group)
        self.assertFalse(acquire_mock.called)

    def test_timeout(self, get_nodeinfo_mock, mapped_mock, acquire_mock):
        get_nodeinfo_mock.return_value = self._get_nodeinfo_list_response()
        mapped_mock.return_value = True
        acquire_mock.side_effect = self._get_acquire_side_effect(self.task)

        self.service._check_deploy_timeouts(self.context)

        self._assert_get_nodeinfo_args(get_nodeinfo_mock)
        mapped_mock.assert_called_once_with(self.service,
                                            self.node.uuid, self.node.driver,
                                            self.node.conductor_group)
        acquire_mock.assert_called_once_with(self.context, self.node.uuid,
                                             purpose=mock.ANY)
        self.task.process_event.assert_called_with(
            'fail',
            callback=self.service._spawn_worker,
            call_args=(conductor_utils.cleanup_after_timeout, self.task),
            err_handler=conductor_utils.provisioning_error_handler,
            target_state=None)

    def test_acquire_node_disappears(self, get_nodeinfo_mock, mapped_mock,
                                     acquire_mock):
        get_nodeinfo_mock.return_value = self._get_nodeinfo_list_response()
        mapped_mock.return_value = True
        acquire_mock.side_effect = exception.NodeNotFound(node='fake')

        # Exception eaten
        self.service._check_deploy_timeouts(self.context)

        self._assert_get_nodeinfo_args(get_nodeinfo_mock)
        mapped_mock.assert_called_once_with(self.service,
                                            self.node.uuid,
                                            self.node.driver,
                                            self.node.conductor_group)
        acquire_mock.assert_called_once_with(self.context,
                                             self.node.uuid,
                                             purpose=mock.ANY)
        self.assertFalse(self.task.spawn_after.called)

    def test_acquire_node_locked(self, get_nodeinfo_mock, mapped_mock,
                                 acquire_mock):
        get_nodeinfo_mock.return_value = self._get_nodeinfo_list_response()
        mapped_mock.return_value = True
        acquire_mock.side_effect = exception.NodeLocked(node='fake',
                                                        host='fake')

        # Exception eaten
        self.service._check_deploy_timeouts(self.context)

        self._assert_get_nodeinfo_args(get_nodeinfo_mock)
        mapped_mock.assert_called_once_with(self.service,
                                            self.node.uuid,
                                            self.node.driver,
                                            self.node.conductor_group)
        acquire_mock.assert_called_once_with(self.context,
                                             self.node.uuid,
                                             purpose=mock.ANY)
        self.assertFalse(self.task.spawn_after.called)

    def test_no_deploywait_after_lock(self, get_nodeinfo_mock, mapped_mock,
                                      acquire_mock):
        task = self._create_task(
            node_attrs=dict(provision_state=states.AVAILABLE,
                            uuid=self.node.uuid))
        get_nodeinfo_mock.return_value = self._get_nodeinfo_list_response()
        mapped_mock.return_value = True
        acquire_mock.side_effect = self._get_acquire_side_effect(task)

        self.service._check_deploy_timeouts(self.context)

        self._assert_get_nodeinfo_args(get_nodeinfo_mock)
        mapped_mock.assert_called_once_with(self.service,
                                            self.node.uuid,
                                            self.node.driver,
                                            self.node.conductor_group)
        acquire_mock.assert_called_once_with(self.context,
                                             self.node.uuid,
                                             purpose=mock.ANY)
        self.assertFalse(task.spawn_after.called)

    def test_maintenance_after_lock(self, get_nodeinfo_mock, mapped_mock,
                                    acquire_mock):
        task = self._create_task(
            node_attrs=dict(provision_state=states.DEPLOYWAIT,
                            target_provision_state=states.ACTIVE,
                            maintenance=True,
                            uuid=self.node.uuid))
        get_nodeinfo_mock.return_value = (
            self._get_nodeinfo_list_response([task.node, self.node2]))
        mapped_mock.return_value = True
        acquire_mock.side_effect = (
            self._get_acquire_side_effect([task, self.task2]))

        self.service._check_deploy_timeouts(self.context)

        self._assert_get_nodeinfo_args(get_nodeinfo_mock)
        self.assertEqual([mock.call(self.service,
                                    self.node.uuid, task.node.driver,
                                    task.node.conductor_group),
                          mock.call(self.service,
                                    self.node2.uuid, self.node2.driver,
                                    self.node2.conductor_group)],
                         mapped_mock.call_args_list)
        self.assertEqual([mock.call(self.context, self.node.uuid,
                                    purpose=mock.ANY),
                          mock.call(self.context, self.node2.uuid,
                                    purpose=mock.ANY)],
                         acquire_mock.call_args_list)
        # First node skipped
        self.assertFalse(task.spawn_after.called)
        # Second node spawned
        self.task2.process_event.assert_called_with(
            'fail',
            callback=self.service._spawn_worker,
            call_args=(conductor_utils.cleanup_after_timeout, self.task2),
            err_handler=conductor_utils.provisioning_error_handler,
            target_state=None)

    def test_exiting_no_worker_avail(self, get_nodeinfo_mock, mapped_mock,
                                     acquire_mock):
        get_nodeinfo_mock.return_value = (
            self._get_nodeinfo_list_response([self.node, self.node2]))
        mapped_mock.return_value = True
        acquire_mock.side_effect = self._get_acquire_side_effect(
            [(self.task, exception.NoFreeConductorWorker()), self.task2])

        # Exception should be nuked
        self.service._check_deploy_timeouts(self.context)

        self._assert_get_nodeinfo_args(get_nodeinfo_mock)
        # mapped should be only called for the first node as we should
        # have exited the loop early due to NoFreeConductorWorker
        mapped_mock.assert_called_once_with(self.service,
                                            self.node.uuid,
                                            self.node.driver,
                                            self.node.conductor_group)
        acquire_mock.assert_called_once_with(self.context,
                                             self.node.uuid,
                                             purpose=mock.ANY)
        self.task.process_event.assert_called_with(
            'fail',
            callback=self.service._spawn_worker,
            call_args=(conductor_utils.cleanup_after_timeout, self.task),
            err_handler=conductor_utils.provisioning_error_handler,
            target_state=None)

    def test_exiting_with_other_exception(self, get_nodeinfo_mock,
                                          mapped_mock, acquire_mock):
        get_nodeinfo_mock.return_value = (
            self._get_nodeinfo_list_response([self.node, self.node2]))
        mapped_mock.return_value = True
        acquire_mock.side_effect = self._get_acquire_side_effect(
            [(self.task, exception.IronicException('foo')), self.task2])

        # Should re-raise
        self.assertRaises(exception.IronicException,
                          self.service._check_deploy_timeouts,
                          self.context)

        self._assert_get_nodeinfo_args(get_nodeinfo_mock)
        # mapped should be only called for the first node as we should
        # have exited the loop early due to unknown exception
        mapped_mock.assert_called_once_with(self.service,
                                            self.node.uuid, self.node.driver,
                                            self.node.conductor_group)
        acquire_mock.assert_called_once_with(self.context,
                                             self.node.uuid,
                                             purpose=mock.ANY)
        self.task.process_event.assert_called_with(
            'fail',
            callback=self.service._spawn_worker,
            call_args=(conductor_utils.cleanup_after_timeout, self.task),
            err_handler=conductor_utils.provisioning_error_handler,
            target_state=None)

    def test_worker_limit(self, get_nodeinfo_mock, mapped_mock, acquire_mock):
        self.config(periodic_max_workers=2, group='conductor')

        # Use the same nodes/tasks to make life easier in the tests
        # here

        get_nodeinfo_mock.return_value = (
            self._get_nodeinfo_list_response([self.node] * 3))
        mapped_mock.return_value = True
        acquire_mock.side_effect = (
            self._get_acquire_side_effect([self.task] * 3))

        self.service._check_deploy_timeouts(self.context)

        # Should only have ran 2.
        self.assertEqual([mock.call(self.service,
                                    self.node.uuid, self.node.driver,
                                    self.node.conductor_group)] * 2,
                         mapped_mock.call_args_list)
        self.assertEqual([mock.call(self.context, self.node.uuid,
                                    purpose=mock.ANY)] * 2,
                         acquire_mock.call_args_list)
        process_event_call = mock.call(
            'fail',
            callback=self.service._spawn_worker,
            call_args=(conductor_utils.cleanup_after_timeout, self.task),
            err_handler=conductor_utils.provisioning_error_handler,
            target_state=None)
        self.assertEqual([process_event_call] * 2,
                         self.task.process_event.call_args_list)


@mgr_utils.mock_record_keepalive
class ManagerTestProperties(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase):

    def setUp(self):
        super(ManagerTestProperties, self).setUp()
        self.service = manager.ConductorManager('test-host', 'test-topic')

    def _check_driver_properties(self, hw_type, expected, agent_common=True,
                                 pxe_common=True):
        properties = self.service.get_driver_properties(self.context, hw_type)
        if agent_common:
            expected.extend(['agent_verify_ca',
                             'deploy_kernel', 'deploy_ramdisk',
                             'deploy_forces_oob_reboot',
                             'image_download_source',
                             'image_http_proxy', 'image_https_proxy',
                             'image_no_proxy'])
        if pxe_common:
            expected.extend(['kernel_append_params',
                             'rescue_kernel', 'rescue_ramdisk'])
        expected.append('force_persistent_boot_device')
        self.assertCountEqual(expected, properties)

    def test_driver_properties_ipmi(self):
        self.config(enabled_hardware_types='ipmi',
                    enabled_power_interfaces=['ipmitool'],
                    enabled_management_interfaces=['ipmitool'],
                    enabled_console_interfaces=['ipmitool-socat'])
        expected = ['ipmi_address', 'ipmi_terminal_port',
                    'ipmi_password', 'ipmi_port', 'ipmi_priv_level',
                    'ipmi_username', 'ipmi_bridging', 'ipmi_transit_channel',
                    'ipmi_transit_address', 'ipmi_target_channel',
                    'ipmi_target_address', 'ipmi_local_address',
                    'ipmi_protocol_version', 'ipmi_force_boot_device',
                    'ipmi_disable_boot_timeout', 'ipmi_hex_kg_key',
                    'ipmi_cipher_suite']
        self._check_driver_properties("ipmi", expected)

    def test_driver_properties_manual_management(self):
        self.config(enabled_hardware_types=['manual-management'])
        self._check_driver_properties('manual-management', [])

    def test_driver_properties_fail(self):
        self._start_service()
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.get_driver_properties,
                                self.context, "bad-driver")
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.DriverNotFound, exc.exc_info[0])


@mock.patch.object(waiters, 'wait_for_all', autospec=True)
@mock.patch.object(manager.ConductorManager, '_spawn_worker', autospec=True)
@mock.patch.object(manager.ConductorManager, '_sync_power_state_nodes_task',
                   autospec=True)
class ParallelPowerSyncTestCase(mgr_utils.CommonMixIn, db_base.DbTestCase):

    def setUp(self):
        super(ParallelPowerSyncTestCase, self).setUp()
        self.service = manager.ConductorManager('hostname', 'test-topic')

    def test__sync_power_states_9_nodes_8_workers(
            self, sync_mock, spawn_mock, waiter_mock):

        CONF.set_override('sync_power_state_workers', 8, group='conductor')

        with mock.patch.object(self.service, 'iter_nodes',
                               new=mock.MagicMock(return_value=[[0]] * 9)):

            self.service._sync_power_states(self.context)

            self.assertEqual(7, spawn_mock.call_count)
            self.assertEqual(1, sync_mock.call_count)
            self.assertEqual(1, waiter_mock.call_count)

    def test__sync_power_states_6_nodes_8_workers(
            self, sync_mock, spawn_mock, waiter_mock):

        CONF.set_override('sync_power_state_workers', 8, group='conductor')

        with mock.patch.object(self.service, 'iter_nodes',
                               new=mock.MagicMock(return_value=[[0]] * 6)):

            self.service._sync_power_states(self.context)

            self.assertEqual(5, spawn_mock.call_count)
            self.assertEqual(1, sync_mock.call_count)
            self.assertEqual(1, waiter_mock.call_count)

    def test__sync_power_states_1_nodes_8_workers(
            self, sync_mock, spawn_mock, waiter_mock):

        CONF.set_override('sync_power_state_workers', 8, group='conductor')

        with mock.patch.object(self.service, 'iter_nodes',
                               new=mock.MagicMock(return_value=[[0]])):

            self.service._sync_power_states(self.context)

            self.assertEqual(0, spawn_mock.call_count)
            self.assertEqual(1, sync_mock.call_count)
            self.assertEqual(1, waiter_mock.call_count)

    def test__sync_power_states_9_nodes_1_worker(
            self, sync_mock, spawn_mock, waiter_mock):

        CONF.set_override('sync_power_state_workers', 1, group='conductor')

        with mock.patch.object(self.service, 'iter_nodes',
                               new=mock.MagicMock(return_value=[[0]] * 9)):

            self.service._sync_power_states(self.context)

            self.assertEqual(0, spawn_mock.call_count)
            self.assertEqual(1, sync_mock.call_count)
            self.assertEqual(1, waiter_mock.call_count)

    @mock.patch.object(queue, 'Queue', autospec=True)
    def test__sync_power_states_node_prioritization(
            self, queue_mock, sync_mock, spawn_mock, waiter_mock):

        CONF.set_override('sync_power_state_workers', 1, group='conductor')

        with mock.patch.object(
            self.service, 'iter_nodes',
            new=mock.MagicMock(return_value=[[0], [1], [2]])
        ), mock.patch.dict(
                self.service.power_state_sync_count,
                {0: 1, 1: 0, 2: 2}, clear=True):

            queue_mock.return_value.qsize.return_value = 0

            self.service._sync_power_states(self.context)

            expected_calls = [mock.call([2]), mock.call([0]), mock.call([1])]
            queue_mock.return_value.put.assert_has_calls(expected_calls)


@mgr_utils.mock_record_keepalive
@mock.patch.object(task_manager, 'acquire', autospec=True)
class GetStepsForAutomatedCleaningTestCase(mgr_utils.ServiceSetUpMixin,
                                           mgr_utils.CommonMixIn,
                                           db_base.DbTestCase):
    def _set_node_trait(self, name):
        self.node.traits = objects.TraitList.create(context=self.context,
                                                    node_id=self.node.id,
                                                    traits=[name])

    def setUp(self):
        super(GetStepsForAutomatedCleaningTestCase, self).setUp()
        self.node = obj_utils.create_test_node(
            self.context,
            driver='fake-hardware',
            uuid=uuidutils.generate_uuid())
        self.node.create()
        self.task = self._create_task(node=self.node)
        self.rb_name = 'CUSTOM_BM_RB'
        self._set_node_trait(self.rb_name)

    @mock.patch.object(manager.ConductorManager, 'get_runbook', autospec=True)
    def test_gsfac_runbook_by_resource_class(self, mock_get_runbook, mt):
        CONF.set_override('automated_cleaning_step_source', 'runbook',
                          group='conductor')
        CONF.set_override('automated_cleaning_runbook_by_resource_class',
                          {'baremetal': self.rb_name},
                          group='conductor')
        self.node.resource_class = 'baremetal'
        # Legacy runbook: mock with spec but without traits attribute
        # (pre-migration)
        rb = mock.Mock(spec_set=objects.Runbook)
        rb.name = self.rb_name
        # Remove traits attribute to simulate legacy runbook
        delattr(rb, 'traits')
        exp_steps = [{'interface': 'deploy', 'step': 'test_step', 'args': {}}]
        rb.steps = exp_steps
        rb.disable_ramdisk = True
        mock_get_runbook.return_value = rb

        steps, dr = self.service._get_steps_for_automated_cleaning(self.task)

        mock_get_runbook.assert_called_once_with(mock.ANY, mock.ANY,
                                                 self.rb_name)
        self.assertEqual(exp_steps, steps)
        self.assertTrue(dr)

    @mock.patch.object(manager.ConductorManager, 'get_runbook', autospec=True)
    def test_gsfac_fallback_to_default_runbook(self, mock_get_runbook, mt):
        rb_to_use = 'CUSTOM_DEFAULT'
        CONF.set_override('automated_cleaning_step_source', 'runbook',
                          group='conductor')
        CONF.set_override('automated_cleaning_runbook', rb_to_use,
                          group='conductor')
        CONF.set_override('automated_cleaning_runbook_by_resource_class',
                          {'other': 'CUSTOM_OTHER'}, group='conductor')
        self.node.resource_class = 'baremetal'
        self._set_node_trait(rb_to_use)
        # Legacy runbook: mock with spec but without traits attribute
        # (pre-migration)
        rb = mock.Mock(spec_set=objects.Runbook)
        rb.name = rb_to_use
        # Remove traits attribute to simulate legacy runbook
        delattr(rb, 'traits')
        exp_steps = [{'interface': 'deploy', 'step': 'test_step', 'args': {}}]
        rb.steps = exp_steps
        rb.disable_ramdisk = True
        mock_get_runbook.return_value = rb
        steps, dr = self.service._get_steps_for_automated_cleaning(self.task)

        mock_get_runbook.assert_called_once_with(mock.ANY, mock.ANY, rb_to_use)
        self.assertEqual(exp_steps, steps)
        self.assertTrue(dr)

    @mock.patch.object(manager.ConductorManager, 'get_runbook', autospec=True)
    def test_gsfac_autogenerated(self, mock_get_runbook, mt):
        CONF.set_override('automated_cleaning_step_source', 'autogenerated',
                          group='conductor')
        CONF.set_override('automated_cleaning_runbook_by_resource_class',
                          {'baremetal': self.rb_name},
                          group='conductor')
        self.node.resource_class = 'baremetal'

        steps, dr = self.service._get_steps_for_automated_cleaning(self.task)

        self.assertIsNone(steps)
        self.assertFalse(dr)
        mock_get_runbook.assert_not_called()

    @mock.patch.object(manager.ConductorManager, 'get_runbook', autospec=True)
    def test_gsfac_runbook_from_node(self, mock_get_runbook, mt):
        rb_to_use = 'CUSTOM_NODE_RB'
        CONF.set_override('automated_cleaning_step_source', 'runbook',
                          group='conductor')
        CONF.set_override('automated_cleaning_runbook_from_node', True,
                          group='conductor')
        self.node.driver_info = {'cleaning_runbook': rb_to_use}
        self._set_node_trait(rb_to_use)
        # Legacy runbook: mock with spec but without traits attribute
        # (pre-migration)
        rb = mock.Mock(spec_set=objects.Runbook)
        rb.name = rb_to_use
        # Remove traits attribute to simulate legacy runbook
        delattr(rb, 'traits')
        exp_steps = [{'interface': 'deploy', 'step': 'test_step', 'args': {}}]
        rb.steps = exp_steps
        rb.disable_ramdisk = True
        mock_get_runbook.return_value = rb

        steps, dr = self.service._get_steps_for_automated_cleaning(self.task)

        mock_get_runbook.assert_called_once_with(mock.ANY, mock.ANY,
                                                 rb_to_use)
        self.assertEqual(steps, exp_steps)
        self.assertTrue(dr)

    @mock.patch.object(manager.ConductorManager, 'get_runbook', autospec=True)
    def test_gsfac_hybrid_no_runbook(self, mock_get_runbook, mt):
        CONF.set_override('automated_cleaning_step_source', 'hybrid',
                          group='conductor')
        CONF.set_override('automated_cleaning_runbook', None,
                          group='conductor')
        CONF.set_override('automated_cleaning_runbook_by_resource_class',
                          {}, group='conductor')

        steps, dr = self.service._get_steps_for_automated_cleaning(self.task)

        self.assertFalse(mock_get_runbook.called)
        self.assertIsNone(steps)
        self.assertFalse(dr)

    @mock.patch.object(manager.ConductorManager, 'get_runbook', autospec=True)
    def test_gsfac_no_runbook_required(self, mock_get_runbook, mt):
        CONF.set_override('automated_cleaning_step_source', 'runbook',
                          group='conductor')
        CONF.set_override('automated_cleaning_runbook', None,
                          group='conductor')
        CONF.set_override('automated_cleaning_runbook_by_resource_class',
                          {}, group='conductor')

        self.assertRaises(exception.NodeCleaningFailure,
                          self.service._get_steps_for_automated_cleaning,
                          self.task)
        self.assertFalse(mock_get_runbook.called)

    @mock.patch.object(manager.ConductorManager, 'get_runbook', autospec=True)
    def test_gsfac_runbook_incompatible_with_node(self, mock_get_runbook, mt):
        bad_rb_name = "CUSTOM_INCOMPATIBLE_RB"
        CONF.set_override('automated_cleaning_step_source', 'runbook',
                          group='conductor')
        CONF.set_override('automated_cleaning_runbook_by_resource_class',
                          {'baremetal': bad_rb_name},
                          group='conductor')
        # NOTE(JayF): We specifically are not adding bad_rb_name to traits,
        #             which means this should be considered incompatible
        self.node.resource_class = 'baremetal'
        rb = mock.Mock(spec_set=objects.Runbook)
        rb.name = bad_rb_name
        rb.traits = []  # legacy path: name must match a node trait
        mock_get_runbook.return_value = rb

        self.assertRaises(exception.NodeCleaningFailure,
                          self.service._get_steps_for_automated_cleaning,
                          self.task)
        # NOTE(JayF): get_runbook is called in this case because we need to
        #             fetch the runbook to know it's incompatible
        self.assertTrue(mock_get_runbook.called)

    @mock.patch.object(manager.ConductorManager, 'get_runbook', autospec=True)
    def test_gsfac_runbook_incompatible_with_node_ignored(
            self, mock_get_runbook, mt):
        bad_rb_name = "CUSTOM_INCOMPATIBLE_RB"
        CONF.set_override('automated_cleaning_runbook_validate_traits', False,
                          group='conductor')
        CONF.set_override('automated_cleaning_step_source', 'runbook',
                          group='conductor')
        CONF.set_override('automated_cleaning_runbook_by_resource_class',
                          {'baremetal': bad_rb_name},
                          group='conductor')
        # NOTE(JayF): We specifically are not adding bad_rb_name to traits,
        #             which means this should be considered incompatible
        self.node.resource_class = 'baremetal'
        rb = mock.Mock(spec_set=objects.Runbook)
        rb.name = bad_rb_name
        exp_steps = [{'interface': 'deploy', 'step': 'test_step', 'args': {}}]
        rb.steps = exp_steps
        rb.disable_ramdisk = True
        mock_get_runbook.return_value = rb

        steps, dr = self.service._get_steps_for_automated_cleaning(self.task)

        mock_get_runbook.assert_called_once_with(mock.ANY, mock.ANY,
                                                 bad_rb_name)
        self.assertEqual(steps, exp_steps)
        self.assertTrue(dr)

    @mock.patch.object(manager.ConductorManager, 'get_runbook', autospec=True)
    def test_gsfac_runbook_traits_intersection_valid(
            self, mock_get_runbook, mt):
        """Runbook has traits; node has at least one matching, steps returned.

        When runbook.traits is non-empty and intersects node traits, the
        validation should pass and the runbook steps should be returned.
        """
        CONF.set_override('automated_cleaning_step_source', 'runbook',
                          group='conductor')
        CONF.set_override('automated_cleaning_runbook', self.rb_name,
                          group='conductor')
        # node has CUSTOM_BM_RB (from setUp) and one extra trait
        self._set_node_trait('CUSTOM_EXTRA')
        rb = mock.Mock(spec_set=objects.Runbook)
        rb.name = self.rb_name
        rb.traits = [self.rb_name, 'CUSTOM_EXTRA']
        exp_steps = [{'interface': 'deploy', 'step': 'do_thing', 'args': {}}]
        rb.steps = exp_steps
        rb.disable_ramdisk = False
        mock_get_runbook.return_value = rb

        steps, dr = self.service._get_steps_for_automated_cleaning(self.task)

        mock_get_runbook.assert_called_once_with(mock.ANY, mock.ANY,
                                                 self.rb_name)
        self.assertEqual(exp_steps, steps)
        self.assertFalse(dr)

    @mock.patch.object(manager.ConductorManager, 'get_runbook', autospec=True)
    def test_gsfac_runbook_traits_no_intersection_raises(
            self, mock_get_runbook, mt):
        """Runbook has traits; node has none of them, NodeCleaningFailure."""
        CONF.set_override('automated_cleaning_step_source', 'runbook',
                          group='conductor')
        CONF.set_override('automated_cleaning_runbook', 'CUSTOM_OTHER',
                          group='conductor')
        rb = mock.Mock(spec_set=objects.Runbook)
        rb.name = 'CUSTOM_OTHER'
        # node traits contain only CUSTOM_BM_RB, not CUSTOM_X or CUSTOM_Y
        rb.traits = ['CUSTOM_X', 'CUSTOM_Y']
        mock_get_runbook.return_value = rb

        self.assertRaises(
            exception.NodeCleaningFailure,
            self.service._get_steps_for_automated_cleaning,
            self.task)
        self.assertTrue(mock_get_runbook.called)

    @mock.patch.object(manager.ConductorManager, 'get_runbook', autospec=True)
    def test_gsfac_runbook_legacy_name_match(self, mock_get_runbook, mt):
        """Legacy (no traits attribute) runbook: name in node traits.

        Succeeds when name matches.
        """
        CONF.set_override('automated_cleaning_step_source', 'runbook',
                          group='conductor')
        CONF.set_override('automated_cleaning_runbook', self.rb_name,
                          group='conductor')
        # Legacy runbook: mock with spec but without traits attribute
        # (pre-migration)
        rb = mock.Mock(spec_set=objects.Runbook)
        rb.name = self.rb_name
        # Remove traits attribute to simulate legacy runbook
        delattr(rb, 'traits')
        exp_steps = [{'interface': 'deploy', 'step': 'do_thing', 'args': {}}]
        rb.steps = exp_steps
        rb.disable_ramdisk = False
        mock_get_runbook.return_value = rb

        steps, dr = self.service._get_steps_for_automated_cleaning(self.task)

        self.assertEqual(exp_steps, steps)
        self.assertFalse(dr)

    @mock.patch.object(manager.ConductorManager, 'get_runbook', autospec=True)
    def test_gsfac_runbook_legacy_name_no_match_raises(
            self, mock_get_runbook, mt):
        """Legacy (no traits attribute) runbook: name absent from node traits.

        Fails when name doesn't match any node traits.
        """
        CONF.set_override('automated_cleaning_step_source', 'runbook',
                          group='conductor')
        CONF.set_override('automated_cleaning_runbook', 'CUSTOM_ABSENT',
                          group='conductor')
        # Legacy runbook: mock with spec but without traits attribute
        # (pre-migration)
        rb = mock.Mock(spec_set=objects.Runbook)
        rb.name = 'CUSTOM_ABSENT'
        # Remove traits attribute to simulate legacy runbook
        delattr(rb, 'traits')
        mock_get_runbook.return_value = rb

        self.assertRaises(
            exception.NodeCleaningFailure,
            self.service._get_steps_for_automated_cleaning,
            self.task)
        self.assertTrue(mock_get_runbook.called)

    @mock.patch.object(manager.ConductorManager, 'get_runbook', autospec=True)
    def test_gsfac_runbook_empty_traits_raises(self, mock_get_runbook, mt):
        """Runbook with empty traits list: should fail (no nodes can match)."""
        CONF.set_override('automated_cleaning_step_source', 'runbook',
                          group='conductor')
        CONF.set_override('automated_cleaning_runbook', 'CUSTOM_EMPTY',
                          group='conductor')
        rb = mock.Mock(spec_set=objects.Runbook)
        rb.name = 'CUSTOM_EMPTY'
        rb.traits = []  # Empty traits list - new runbook with no traits
        # configured
        mock_get_runbook.return_value = rb

        self.assertRaises(
            exception.NodeCleaningFailure,
            self.service._get_steps_for_automated_cleaning,
            self.task)
        self.assertTrue(mock_get_runbook.called)


@mock.patch.object(task_manager, 'acquire', autospec=True)
@mock.patch.object(manager.ConductorManager, '_mapped_to_this_conductor',
                   autospec=True)
@mock.patch.object(dbapi.IMPL, 'get_nodeinfo_list', autospec=True)
class ManagerSyncLocalStateTestCase(mgr_utils.CommonMixIn, db_base.DbTestCase):

    def setUp(self):
        super(ManagerSyncLocalStateTestCase, self).setUp()

        self.service = manager.ConductorManager('hostname', 'test-topic')

        self.service.conductor = mock.Mock()
        self.service.dbapi = self.dbapi
        self.service.ring_manager = mock.Mock()

        self.node = self._create_node(provision_state=states.ACTIVE,
                                      target_provision_state=states.NOSTATE)
        self.task = self._create_task(node=self.node)

        self.filters = {'reserved': False,
                        'maintenance': False,
                        'provision_state': states.ACTIVE}
        self.columns = ['uuid', 'driver', 'conductor_group',
                        'conductor_affinity']

    def _assert_get_nodeinfo_args(self, get_nodeinfo_mock):
        get_nodeinfo_mock.assert_called_once_with(
            columns=self.columns, filters=self.filters)

    def test_not_mapped(self, get_nodeinfo_mock, mapped_mock, acquire_mock):
        get_nodeinfo_mock.return_value = self._get_nodeinfo_list_response()
        mapped_mock.return_value = False

        self.service._sync_local_state(self.context)

        self._assert_get_nodeinfo_args(get_nodeinfo_mock)
        mapped_mock.assert_called_once_with(
            self.service, self.node.uuid, self.node.driver,
            self.node.conductor_group)
        self.assertFalse(acquire_mock.called)

    def test_already_mapped(self, get_nodeinfo_mock, mapped_mock,
                            acquire_mock):
        # Node is already mapped to the conductor running the periodic task
        self.node.conductor_affinity = 123
        self.service.conductor.id = 123

        get_nodeinfo_mock.return_value = self._get_nodeinfo_list_response()
        mapped_mock.return_value = True

        self.service._sync_local_state(self.context)

        self._assert_get_nodeinfo_args(get_nodeinfo_mock)
        mapped_mock.assert_called_once_with(
            self.service, self.node.uuid, self.node.driver,
            self.node.conductor_group)
        self.assertFalse(acquire_mock.called)

    def test_good(self, get_nodeinfo_mock, mapped_mock, acquire_mock):
        get_nodeinfo_mock.return_value = self._get_nodeinfo_list_response()
        mapped_mock.return_value = True
        acquire_mock.side_effect = self._get_acquire_side_effect(self.task)

        self.service._sync_local_state(self.context)

        self._assert_get_nodeinfo_args(get_nodeinfo_mock)
        mapped_mock.assert_called_once_with(
            self.service, self.node.uuid, self.node.driver,
            self.node.conductor_group)
        acquire_mock.assert_called_once_with(self.context, self.node.uuid,
                                             purpose=mock.ANY, shared=False)
        # assert spawn_after has been called
        self.task.spawn_after.assert_called_once_with(
            self.service._spawn_worker,
            self.service._do_takeover, self.task)

    def test_no_free_worker(self, get_nodeinfo_mock, mapped_mock,
                            acquire_mock):
        mapped_mock.return_value = True
        acquire_mock.side_effect = (
            self._get_acquire_side_effect([self.task] * 3))
        self.task.spawn_after.side_effect = [
            None,
            exception.NoFreeConductorWorker('error')
        ]

        # 3 nodes to be checked
        get_nodeinfo_mock.return_value = (
            self._get_nodeinfo_list_response([self.node] * 3))

        self.service._sync_local_state(self.context)

        self._assert_get_nodeinfo_args(get_nodeinfo_mock)

        # assert  _mapped_to_this_conductor() gets called 2 times only
        # instead of 3. When NoFreeConductorWorker is raised the loop
        # should be broken
        expected = [mock.call(self.service, self.node.uuid, self.node.driver,
                              self.node.conductor_group)] * 2
        self.assertEqual(expected, mapped_mock.call_args_list)

        # assert  acquire() gets called 2 times only instead of 3. When
        # NoFreeConductorWorker is raised the loop should be broken
        expected = [mock.call(self.context, self.node.uuid,
                              purpose=mock.ANY, shared=False)] * 2
        self.assertEqual(expected, acquire_mock.call_args_list)

        # assert spawn_after has been called twice
        expected = [mock.call(self.service._spawn_worker,
                    self.service._do_takeover, self.task)] * 2
        self.assertEqual(expected, self.task.spawn_after.call_args_list)

    def test_node_locked(self, get_nodeinfo_mock, mapped_mock, acquire_mock,):
        mapped_mock.return_value = True
        acquire_mock.side_effect = self._get_acquire_side_effect(
            [self.task, exception.NodeLocked('error'), self.task])
        self.task.spawn_after.side_effect = [None, None]

        # 3 nodes to be checked
        get_nodeinfo_mock.return_value = (
            self._get_nodeinfo_list_response([self.node] * 3))

        self.service._sync_local_state(self.context)

        self._assert_get_nodeinfo_args(get_nodeinfo_mock)

        # assert _mapped_to_this_conductor() gets called 3 times
        expected = [mock.call(
            self.service, self.node.uuid, self.node.driver,
            self.node.conductor_group)] * 3
        self.assertEqual(expected, mapped_mock.call_args_list)

        # assert acquire() gets called 3 times
        expected = [mock.call(self.context, self.node.uuid,
                              purpose=mock.ANY, shared=False)] * 3
        self.assertEqual(expected, acquire_mock.call_args_list)

        # assert spawn_after has been called only 2 times
        expected = [mock.call(self.service._spawn_worker,
                    self.service._do_takeover, self.task)] * 2
        self.assertEqual(expected, self.task.spawn_after.call_args_list)

    def test_worker_limit(self, get_nodeinfo_mock, mapped_mock, acquire_mock):
        # Limit to only 1 worker
        self.config(periodic_max_workers=1, group='conductor')
        mapped_mock.return_value = True
        acquire_mock.side_effect = (
            self._get_acquire_side_effect([self.task] * 3))
        self.task.spawn_after.side_effect = [None] * 3

        # 3 nodes to be checked
        get_nodeinfo_mock.return_value = (
            self._get_nodeinfo_list_response([self.node] * 3))

        self.service._sync_local_state(self.context)

        self._assert_get_nodeinfo_args(get_nodeinfo_mock)

        # assert _mapped_to_this_conductor() gets called only once
        # because of the worker limit
        mapped_mock.assert_called_once_with(
            self.service, self.node.uuid, self.node.driver,
            self.node.conductor_group)

        # assert acquire() gets called only once because of the worker limit
        acquire_mock.assert_called_once_with(self.context, self.node.uuid,
                                             purpose=mock.ANY, shared=False)

        # assert spawn_after has been called
        self.task.spawn_after.assert_called_once_with(
            self.service._spawn_worker,
            self.service._do_takeover, self.task)


@mgr_utils.mock_record_keepalive
class NodeInspectHardware(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase):

    def test__check_inspect_wait_timeouts(self):
        self._start_service()
        CONF.set_override('inspect_wait_timeout', 1, group='conductor')
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.INSPECTWAIT,
            target_provision_state=states.MANAGEABLE,
            provision_updated_at=datetime.datetime(2000, 1, 1, 0, 0),
            inspection_started_at=datetime.datetime(2000, 1, 1, 0, 0))

        self.service._check_inspect_wait_timeouts(self.context)
        node.refresh()
        self.assertEqual(states.INSPECTFAIL, node.provision_state)
        self.assertEqual(states.MANAGEABLE, node.target_provision_state)
        self.assertIsNotNone(node.last_error)

    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_inspect_hardware_worker_pool_full(self, mock_spawn):
        prv_state = states.MANAGEABLE
        tgt_prv_state = states.NOSTATE
        node = obj_utils.create_test_node(self.context,
                                          provision_state=prv_state,
                                          target_provision_state=tgt_prv_state,
                                          last_error=None,
                                          driver='fake-hardware')
        self._start_service()

        mock_spawn.side_effect = exception.NoFreeConductorWorker()

        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.inspect_hardware,
                                self.context, node.uuid)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NoFreeConductorWorker, exc.exc_info[0])
        node.refresh()
        # Make sure things were rolled back
        self.assertEqual(prv_state, node.provision_state)
        self.assertEqual(tgt_prv_state, node.target_provision_state)
        self.assertIsNotNone(node.last_error)
        # Verify reservation has been cleared.
        self.assertIsNone(node.reservation)

    def _test_inspect_hardware_validate_fail(self, mock_validate):
        mock_validate.side_effect = exception.InvalidParameterValue(
            'Fake error message')
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.inspect_hardware,
                                self.context, node.uuid)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.InvalidParameterValue, exc.exc_info[0])

        mock_validate.side_effect = exception.MissingParameterValue(
            'Fake error message')
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.inspect_hardware,
                                self.context, node.uuid)
        self.assertEqual(exception.MissingParameterValue, exc.exc_info[0])

        # This is a sync operation last_error should be None.
        self.assertIsNone(node.last_error)
        # Verify reservation has been cleared.
        self.assertIsNone(node.reservation)

    @mock.patch('ironic.drivers.modules.fake.FakeInspect.validate',
                autospec=True)
    def test_inspect_hardware_validate_fail(self, mock_validate):
        self._test_inspect_hardware_validate_fail(mock_validate)

    @mock.patch('ironic.drivers.modules.fake.FakePower.validate',
                autospec=True)
    def test_inspect_hardware_power_validate_fail(self, mock_validate):
        self._test_inspect_hardware_validate_fail(mock_validate)


@mock.patch.object(conductor_utils, 'node_history_record',
                   mock.Mock(spec=conductor_utils.node_history_record))
@mock.patch.object(task_manager, 'acquire', autospec=True)
@mock.patch.object(manager.ConductorManager, '_mapped_to_this_conductor',
                   autospec=True)
@mock.patch.object(dbapi.IMPL, 'get_nodeinfo_list', autospec=True)
class ManagerCheckInspectWaitTimeoutsTestCase(mgr_utils.CommonMixIn,
                                              db_base.DbTestCase):
    def setUp(self):
        super(ManagerCheckInspectWaitTimeoutsTestCase, self).setUp()
        self.config(inspect_wait_timeout=300, group='conductor')
        self.service = manager.ConductorManager('hostname', 'test-topic')
        self.service.dbapi = self.dbapi

        self.node = self._create_node(provision_state=states.INSPECTWAIT,
                                      target_provision_state=states.MANAGEABLE)
        self.task = self._create_task(node=self.node)

        self.node2 = self._create_node(
            provision_state=states.INSPECTWAIT,
            target_provision_state=states.MANAGEABLE)
        self.task2 = self._create_task(node=self.node2)

        self.filters = {'reserved': False,
                        'maintenance': False,
                        'inspection_started_before': 300,
                        'provision_state': states.INSPECTWAIT}
        self.columns = ['uuid', 'driver', 'conductor_group']

    def _assert_get_nodeinfo_args(self, get_nodeinfo_mock):
        get_nodeinfo_mock.assert_called_once_with(
            sort_dir='asc', columns=self.columns, filters=self.filters,
            sort_key='inspection_started_at')

    def test__check_inspect_timeouts_not_mapped(self, get_nodeinfo_mock,
                                                mapped_mock, acquire_mock):
        get_nodeinfo_mock.return_value = self._get_nodeinfo_list_response()
        mapped_mock.return_value = False

        self.service._check_inspect_wait_timeouts(self.context)

        self._assert_get_nodeinfo_args(get_nodeinfo_mock)
        mapped_mock.assert_called_once_with(self.service,
                                            self.node.uuid, self.node.driver,
                                            self.node.conductor_group)
        self.assertFalse(acquire_mock.called)

    def test__check_inspect_timeout(self, get_nodeinfo_mock,
                                    mapped_mock, acquire_mock):
        get_nodeinfo_mock.return_value = self._get_nodeinfo_list_response()
        mapped_mock.return_value = True
        acquire_mock.side_effect = self._get_acquire_side_effect(self.task)

        self.service._check_inspect_wait_timeouts(self.context)

        self._assert_get_nodeinfo_args(get_nodeinfo_mock)
        mapped_mock.assert_called_once_with(self.service,
                                            self.node.uuid, self.node.driver,
                                            self.node.conductor_group)
        acquire_mock.assert_called_once_with(self.context, self.node.uuid,
                                             purpose=mock.ANY)
        self.task.process_event.assert_called_with(
            'fail', callback=mock.ANY, call_args=mock.ANY,
            err_handler=mock.ANY, target_state=None)

    def test__check_inspect_timeouts_acquire_node_disappears(self,
                                                             get_nodeinfo_mock,
                                                             mapped_mock,
                                                             acquire_mock):
        get_nodeinfo_mock.return_value = self._get_nodeinfo_list_response()
        mapped_mock.return_value = True
        acquire_mock.side_effect = exception.NodeNotFound(node='fake')

        # Exception eaten
        self.service._check_inspect_wait_timeouts(self.context)

        self._assert_get_nodeinfo_args(get_nodeinfo_mock)
        mapped_mock.assert_called_once_with(self.service,
                                            self.node.uuid,
                                            self.node.driver,
                                            self.node.conductor_group)
        acquire_mock.assert_called_once_with(self.context,
                                             self.node.uuid,
                                             purpose=mock.ANY)
        self.assertFalse(self.task.process_event.called)

    def test__check_inspect_timeouts_acquire_node_locked(self,
                                                         get_nodeinfo_mock,
                                                         mapped_mock,
                                                         acquire_mock):
        get_nodeinfo_mock.return_value = self._get_nodeinfo_list_response()
        mapped_mock.return_value = True
        acquire_mock.side_effect = exception.NodeLocked(node='fake',
                                                        host='fake')

        # Exception eaten
        self.service._check_inspect_wait_timeouts(self.context)

        self._assert_get_nodeinfo_args(get_nodeinfo_mock)
        mapped_mock.assert_called_once_with(self.service,
                                            self.node.uuid,
                                            self.node.driver,
                                            self.node.conductor_group)
        acquire_mock.assert_called_once_with(self.context,
                                             self.node.uuid,
                                             purpose=mock.ANY)
        self.assertFalse(self.task.process_event.called)

    def test__check_inspect_timeouts_no_acquire_after_lock(self,
                                                           get_nodeinfo_mock,
                                                           mapped_mock,
                                                           acquire_mock):
        task = self._create_task(
            node_attrs=dict(provision_state=states.AVAILABLE,
                            uuid=self.node.uuid))
        get_nodeinfo_mock.return_value = self._get_nodeinfo_list_response()
        mapped_mock.return_value = True
        acquire_mock.side_effect = self._get_acquire_side_effect(task)

        self.service._check_inspect_wait_timeouts(self.context)

        self._assert_get_nodeinfo_args(get_nodeinfo_mock)
        mapped_mock.assert_called_once_with(self.service,
                                            self.node.uuid,
                                            self.node.driver,
                                            self.node.conductor_group)
        acquire_mock.assert_called_once_with(self.context,
                                             self.node.uuid,
                                             purpose=mock.ANY)
        self.assertFalse(task.process_event.called)

    def test__check_inspect_timeouts_to_maintenance_after_lock(
            self, get_nodeinfo_mock, mapped_mock, acquire_mock):
        task = self._create_task(
            node_attrs=dict(provision_state=states.INSPECTWAIT,
                            target_provision_state=states.MANAGEABLE,
                            maintenance=True,
                            uuid=self.node.uuid))
        get_nodeinfo_mock.return_value = (
            self._get_nodeinfo_list_response([task.node, self.node2]))
        mapped_mock.return_value = True
        acquire_mock.side_effect = (
            self._get_acquire_side_effect([task, self.task2]))

        self.service._check_inspect_wait_timeouts(self.context)

        self._assert_get_nodeinfo_args(get_nodeinfo_mock)
        self.assertEqual([mock.call(self.service,
                                    self.node.uuid, task.node.driver,
                                    task.node.conductor_group),
                          mock.call(self.service,
                                    self.node2.uuid, self.node2.driver,
                                    self.node2.conductor_group)],
                         mapped_mock.call_args_list)
        self.assertEqual([mock.call(self.context, self.node.uuid,
                                    purpose=mock.ANY),
                          mock.call(self.context, self.node2.uuid,
                                    purpose=mock.ANY)],
                         acquire_mock.call_args_list)
        # First node skipped
        self.assertFalse(task.process_event.called)
        # Second node spawned
        self.task2.process_event.assert_called_with(
            'fail', callback=mock.ANY, call_args=mock.ANY,
            err_handler=mock.ANY, target_state=None)

    def test__check_inspect_timeouts_exiting_no_worker_avail(
            self, get_nodeinfo_mock, mapped_mock, acquire_mock):
        get_nodeinfo_mock.return_value = (
            self._get_nodeinfo_list_response([self.node, self.node2]))
        mapped_mock.return_value = True
        acquire_mock.side_effect = self._get_acquire_side_effect(
            [(self.task, exception.NoFreeConductorWorker()), self.task2])

        # Exception should be nuked
        self.service._check_inspect_wait_timeouts(self.context)

        self._assert_get_nodeinfo_args(get_nodeinfo_mock)
        # mapped should be only called for the first node as we should
        # have exited the loop early due to NoFreeConductorWorker
        mapped_mock.assert_called_once_with(self.service,
                                            self.node.uuid,
                                            self.node.driver,
                                            self.node.conductor_group)
        acquire_mock.assert_called_once_with(self.context,
                                             self.node.uuid,
                                             purpose=mock.ANY)
        self.task.process_event.assert_called_with(
            'fail', callback=mock.ANY, call_args=mock.ANY,
            err_handler=mock.ANY, target_state=None)

    def test__check_inspect_timeouts_exit_with_other_exception(
            self, get_nodeinfo_mock, mapped_mock, acquire_mock):
        get_nodeinfo_mock.return_value = (
            self._get_nodeinfo_list_response([self.node, self.node2]))
        mapped_mock.return_value = True
        acquire_mock.side_effect = self._get_acquire_side_effect(
            [(self.task, exception.IronicException('foo')), self.task2])

        # Should re-raise
        self.assertRaises(exception.IronicException,
                          self.service._check_inspect_wait_timeouts,
                          self.context)

        self._assert_get_nodeinfo_args(get_nodeinfo_mock)
        # mapped should be only called for the first node as we should
        # have exited the loop early due to unknown exception
        mapped_mock.assert_called_once_with(self.service,
                                            self.node.uuid,
                                            self.node.driver,
                                            self.node.conductor_group)
        acquire_mock.assert_called_once_with(self.context,
                                             self.node.uuid,
                                             purpose=mock.ANY)
        self.task.process_event.assert_called_with(
            'fail', callback=mock.ANY, call_args=mock.ANY,
            err_handler=mock.ANY, target_state=None)

    def test__check_inspect_timeouts_worker_limit(self, get_nodeinfo_mock,
                                                  mapped_mock, acquire_mock):
        self.config(periodic_max_workers=2, group='conductor')

        # Use the same nodes/tasks to make life easier in the tests
        # here

        get_nodeinfo_mock.return_value = (
            self._get_nodeinfo_list_response([self.node] * 3))
        mapped_mock.return_value = True
        acquire_mock.side_effect = (
            self._get_acquire_side_effect([self.task] * 3))

        self.service._check_inspect_wait_timeouts(self.context)

        # Should only have ran 2.
        self.assertEqual([mock.call(self.service,
                                    self.node.uuid, self.node.driver,
                                    self.node.conductor_group)] * 2,
                         mapped_mock.call_args_list)
        self.assertEqual([mock.call(self.context, self.node.uuid,
                                    purpose=mock.ANY)] * 2,
                         acquire_mock.call_args_list)
        process_event_call = mock.call('fail', callback=mock.ANY,
                                       call_args=mock.ANY,
                                       err_handler=mock.ANY,
                                       target_state=None)
        self.assertEqual([process_event_call] * 2,
                         self.task.process_event.call_args_list)


@mgr_utils.mock_record_keepalive
class DestroyPortTestCase(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase):
    def test_destroy_port(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')

        port = obj_utils.create_test_port(self.context,
                                          node_id=node.id)
        self.service.destroy_port(self.context, port)
        self.assertRaises(exception.PortNotFound, port.refresh)

    def test_destroy_port_node_locked(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          reservation='fake-reserv')

        port = obj_utils.create_test_port(self.context, node_id=node.id)
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.destroy_port,
                                self.context, port)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NodeLocked, exc.exc_info[0])

    def test_destroy_port_node_active_state(self):
        instance_uuid = uuidutils.generate_uuid()
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          instance_uuid=instance_uuid,
                                          provision_state='active')
        port = obj_utils.create_test_port(
            self.context,
            node_id=node.id,
            internal_info={n_common.NetType.TENANT.vif_key: 'foo'})
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.destroy_port,
                                self.context, port)
        self.assertEqual(exception.InvalidState, exc.exc_info[0])

    def test_destroy_port_node_active_and_maintenance_vif_present(self):
        instance_uuid = uuidutils.generate_uuid()
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          instance_uuid=instance_uuid,
                                          provision_state='active',
                                          maintenance=True)
        port = obj_utils.create_test_port(
            self.context,
            node_id=node.id,
            internal_info={n_common.NetType.TENANT.vif_key: 'fake-id'})
        self.service.destroy_port(self.context, port)
        self.assertRaises(exception.PortNotFound, port.refresh)

    def test_destroy_port_node_active_and_maintenance_no_vif(self):
        instance_uuid = uuidutils.generate_uuid()
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          instance_uuid=instance_uuid,
                                          provision_state='active',
                                          maintenance=True)
        port = obj_utils.create_test_port(self.context,
                                          node_id=node.id)
        self.service.destroy_port(self.context, port)
        self.assertRaises(exception.PortNotFound,
                          self.dbapi.get_port_by_uuid,
                          port.uuid)

    def test_destroy_port_with_instance_not_in_active_port_unbound(self):
        instance_uuid = uuidutils.generate_uuid()
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          instance_uuid=instance_uuid,
                                          provision_state='deploy failed')
        port = obj_utils.create_test_port(self.context,
                                          node_id=node.id)
        self.service.destroy_port(self.context, port)
        self.assertRaises(exception.PortNotFound,
                          self.dbapi.get_port_by_uuid,
                          port.uuid)

    def test_destroy_port_with_instance_not_in_active_port_bound(self):
        instance_uuid = uuidutils.generate_uuid()
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          instance_uuid=instance_uuid,
                                          provision_state='deploy failed')
        port = obj_utils.create_test_port(
            self.context,
            node_id=node.id,
            internal_info={n_common.NetType.TENANT.vif_key: 'foo'})
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.destroy_port,
                                self.context, port)
        self.assertEqual(exception.InvalidState, exc.exc_info[0])

    def test_destroy_port_node_active_port_unbound(self):
        instance_uuid = uuidutils.generate_uuid()
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          instance_uuid=instance_uuid,
                                          provision_state='active')
        port = obj_utils.create_test_port(self.context,
                                          node_id=node.id)
        self.service.destroy_port(self.context, port)
        self.assertRaises(exception.PortNotFound,
                          self.dbapi.get_port_by_uuid,
                          port.uuid)


@mgr_utils.mock_record_keepalive
class DestroyPortgroupTestCase(mgr_utils.ServiceSetUpMixin,
                               db_base.DbTestCase):
    def test_destroy_portgroup(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        portgroup = obj_utils.create_test_portgroup(self.context,
                                                    node_id=node.id)
        self.service.destroy_portgroup(self.context, portgroup)
        self.assertRaises(exception.PortgroupNotFound, portgroup.refresh)

    def test_destroy_portgroup_node_locked(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          reservation='fake-reserv')
        portgroup = obj_utils.create_test_portgroup(self.context,
                                                    node_id=node.id)
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.destroy_portgroup,
                                self.context, portgroup)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NodeLocked, exc.exc_info[0])


@mgr_utils.mock_record_keepalive
class UpdatePortgroupPhysicalNetworkTestCase(mgr_utils.ServiceSetUpMixin,
                                             db_base.DbTestCase):
    def test_update_portgroup_physical_network(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        portgroup = obj_utils.create_test_portgroup(
            self.context, physical_network='old_network', node_id=node.id)

        ports = []
        for i in range(0, 5):
            ports.append(
                obj_utils.create_test_port(self.context,
                                           uuid=uuidutils.generate_uuid(),
                                           physical_network='old_network',
                                           address=f"52:54:00:cf:2d:0{i}",
                                           node_id=node.id,
                                           portgroup_id=portgroup.id))

        self.service.update_portgroup_physical_network(
            self.context, portgroup, 'new_physnet')

        self.assertEqual(portgroup.physical_network, 'new_physnet')
        for i in range(0, 5):
            ports[i].refresh(self.context)
            self.assertEqual(portgroup.physical_network,
                             ports[i].physical_network)

    def test_update_portgroup_physical_nework_node_locked(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          reservation='fake-reserv')
        portgroup = obj_utils.create_test_portgroup(self.context,
                                                    node_id=node.id)
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.update_portgroup_physical_network,
                                self.context, portgroup, 'new_physnet')
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NodeLocked, exc.exc_info[0])


@mgr_utils.mock_record_keepalive
@mock.patch.object(manager.ConductorManager, '_fail_if_in_state',
                   autospec=True)
@mock.patch.object(manager.ConductorManager, '_mapped_to_this_conductor',
                   autospec=True)
@mock.patch.object(dbapi.IMPL, 'get_offline_conductors', autospec=True)
class ManagerCheckOrphanNodesTestCase(mgr_utils.ServiceSetUpMixin,
                                      db_base.DbTestCase):
    def setUp(self):
        super(ManagerCheckOrphanNodesTestCase, self).setUp()
        self._start_service()

        self.node = obj_utils.create_test_node(
            self.context, id=1, uuid=uuidutils.generate_uuid(),
            driver='fake-hardware', provision_state=states.DEPLOYING,
            target_provision_state=states.ACTIVE,
            target_power_state=states.POWER_ON,
            reservation='fake-conductor')

        # create a second node in a different state to test the
        # filtering nodes in DEPLOYING state
        obj_utils.create_test_node(
            self.context, id=10, uuid=uuidutils.generate_uuid(),
            driver='fake-hardware', provision_state=states.AVAILABLE,
            target_provision_state=states.NOSTATE)

    def test__check_orphan_nodes(self, mock_off_cond, mock_mapped,
                                 mock_fail_if):
        mock_off_cond.return_value = ['fake-conductor']

        self.service._check_orphan_nodes(self.context)

        self.node.refresh()
        mock_off_cond.assert_called_once_with()
        mock_mapped.assert_called_once_with(
            self.service, self.node.uuid, 'fake-hardware', '')
        mock_fail_if.assert_called_once_with(
            self.service,
            mock.ANY, {'uuid': self.node.uuid},
            {states.DEPLOYING, states.CLEANING},
            'provision_updated_at',
            callback_method=conductor_utils.abort_on_conductor_take_over,
            err_handler=conductor_utils.provisioning_error_handler)
        # assert node was released
        self.assertIsNone(self.node.reservation)
        self.assertIsNone(self.node.target_power_state)
        self.assertIsNotNone(self.node.last_error)

    def test__check_orphan_nodes_cleaning(self, mock_off_cond, mock_mapped,
                                          mock_fail_if):
        self.node.provision_state = states.CLEANING
        self.node.save()
        mock_off_cond.return_value = ['fake-conductor']

        self.service._check_orphan_nodes(self.context)

        self.node.refresh()
        mock_off_cond.assert_called_once_with()
        mock_mapped.assert_called_once_with(
            self.service, self.node.uuid, 'fake-hardware', '')
        mock_fail_if.assert_called_once_with(
            self.service,
            mock.ANY, {'uuid': self.node.uuid},
            {states.DEPLOYING, states.CLEANING},
            'provision_updated_at',
            callback_method=conductor_utils.abort_on_conductor_take_over,
            err_handler=conductor_utils.provisioning_error_handler)
        # assert node was released
        self.assertIsNone(self.node.reservation)
        self.assertIsNone(self.node.target_power_state)
        self.assertIsNotNone(self.node.last_error)

    def test__check_orphan_nodes_alive(self, mock_off_cond,
                                       mock_mapped, mock_fail_if):
        mock_off_cond.return_value = []

        self.service._check_orphan_nodes(self.context)

        self.node.refresh()
        mock_off_cond.assert_called_once_with()
        self.assertFalse(mock_mapped.called)
        self.assertFalse(mock_fail_if.called)
        # assert node still locked
        self.assertIsNotNone(self.node.reservation)

    @mock.patch.object(objects.Node, 'release', autospec=True)
    def test__check_orphan_nodes_release_exceptions_skipping(
            self, mock_release, mock_off_cond, mock_mapped, mock_fail_if):
        mock_off_cond.return_value = ['fake-conductor']
        # Add another node so we can check both exceptions
        node2 = obj_utils.create_test_node(
            self.context, id=2, uuid=uuidutils.generate_uuid(),
            driver='fake-hardware', provision_state=states.DEPLOYING,
            target_provision_state=states.DEPLOYDONE,
            reservation='fake-conductor')

        mock_mapped.return_value = True
        mock_release.side_effect = [exception.NodeNotFound('not found'),
                                    exception.NodeLocked('locked')]
        self.service._check_orphan_nodes(self.context)

        self.node.refresh()
        mock_off_cond.assert_called_once_with()
        expected_calls = [
            mock.call(self.service, self.node.uuid, 'fake-hardware', ''),
            mock.call(self.service, node2.uuid, 'fake-hardware', '')
        ]
        mock_mapped.assert_has_calls(expected_calls)
        # Assert we skipped and didn't try to call _fail_if_in_state
        self.assertFalse(mock_fail_if.called)

    def test__check_orphan_nodes_release_node_not_locked(
            self, mock_off_cond, mock_mapped, mock_fail_if):
        # this simulates releasing the node elsewhere
        count = [0]

        def _fake_release(*args, **kwargs):
            self.node.reservation = None
            self.node.save()
            # raise an exception only the first time release is called
            count[0] += 1
            if count[0] == 1:
                raise exception.NodeNotLocked('not locked')

        mock_off_cond.return_value = ['fake-conductor']
        mock_mapped.return_value = True
        with mock.patch.object(objects.Node, 'release',
                               side_effect=_fake_release,
                               autospec=True) as mock_release:
            self.service._check_orphan_nodes(self.context)
            mock_release.assert_called_with(self.context, mock.ANY,
                                            self.node.id)

        mock_off_cond.assert_called_once_with()
        mock_mapped.assert_called_once_with(
            self.service, self.node.uuid, 'fake-hardware', '')
        mock_fail_if.assert_called_once_with(
            self.service,
            mock.ANY, {'uuid': self.node.uuid},
            {states.DEPLOYING, states.CLEANING},
            'provision_updated_at',
            callback_method=conductor_utils.abort_on_conductor_take_over,
            err_handler=conductor_utils.provisioning_error_handler)

    def test__check_orphan_nodes_maintenance(self, mock_off_cond, mock_mapped,
                                             mock_fail_if):
        self.node.maintenance = True
        self.node.save()
        mock_off_cond.return_value = ['fake-conductor']

        self.service._check_orphan_nodes(self.context)

        self.node.refresh()
        mock_off_cond.assert_called_once_with()
        mock_mapped.assert_called_once_with(
            self.service, self.node.uuid, 'fake-hardware', '')
        # assert node was released
        self.assertIsNone(self.node.reservation)
        # not changing states in maintenance
        self.assertFalse(mock_fail_if.called)
        self.assertIsNotNone(self.node.target_power_state)


class TestIndirectionApiConductor(db_base.DbTestCase):

    def setUp(self):
        super(TestIndirectionApiConductor, self).setUp()
        self.conductor = manager.ConductorManager('test-host', 'test-topic')

    def _test_object_action(self, is_classmethod, raise_exception,
                            return_object=False):
        @obj_base.IronicObjectRegistry.register
        class TestObject(obj_base.IronicObject):
            context = self.context

            def foo(self, context, raise_exception=False, return_object=False):
                if raise_exception:
                    raise Exception('test')
                elif return_object:
                    return obj
                else:
                    return 'test'

            @classmethod
            def bar(cls, context, raise_exception=False, return_object=False):
                if raise_exception:
                    raise Exception('test')
                elif return_object:
                    return obj
                else:
                    return 'test'

        obj = TestObject(self.context)
        if is_classmethod:
            versions = ovo_base.obj_tree_get_versions(TestObject.obj_name())
            result = self.conductor.object_class_action_versions(
                self.context, TestObject.obj_name(), 'bar', versions,
                tuple(), {'raise_exception': raise_exception,
                          'return_object': return_object})
        else:
            updates, result = self.conductor.object_action(
                self.context, obj, 'foo', tuple(),
                {'raise_exception': raise_exception,
                 'return_object': return_object})
        if return_object:
            self.assertEqual(obj, result)
        else:
            self.assertEqual('test', result)

    def test_object_action(self):
        self._test_object_action(False, False)

    def test_object_action_on_raise(self):
        self.assertRaises(messaging.ExpectedException,
                          self._test_object_action, False, True)

    def test_object_action_on_object(self):
        self._test_object_action(False, False, True)

    def test_object_class_action(self):
        self._test_object_action(True, False)

    def test_object_class_action_on_raise(self):
        self.assertRaises(messaging.ExpectedException,
                          self._test_object_action, True, True)

    def test_object_class_action_on_object(self):
        self._test_object_action(True, False, False)

    def test_object_action_copies_object(self):
        @obj_base.IronicObjectRegistry.register
        class TestObject(obj_base.IronicObject):
            fields = {'dict': fields.DictOfStringsField()}

            def touch_dict(self, context):
                self.dict['foo'] = 'bar'
                self.obj_reset_changes()

        obj = TestObject(self.context)
        obj.dict = {}
        obj.obj_reset_changes()
        updates, result = self.conductor.object_action(
            self.context, obj, 'touch_dict', tuple(), {})
        # NOTE(danms): If conductor did not properly copy the object, then
        # the new and reference copies of the nested dict object will be
        # the same, and thus 'dict' will not be reported as changed
        self.assertIn('dict', updates)
        self.assertEqual({'foo': 'bar'}, updates['dict'])

    def test_object_backport_versions(self):
        fake_backported_obj = 'fake-backported-obj'
        obj_name = 'fake-obj'
        test_obj = mock.Mock()
        test_obj.obj_name.return_value = obj_name
        test_obj.obj_to_primitive.return_value = fake_backported_obj
        fake_version_manifest = {obj_name: '1.0'}

        result = self.conductor.object_backport_versions(
            self.context, test_obj, fake_version_manifest)

        self.assertEqual(result, fake_backported_obj)
        test_obj.obj_to_primitive.assert_called_once_with(
            target_version='1.0', version_manifest=fake_version_manifest)


@mgr_utils.mock_record_keepalive
class DoNodeTakeOverTestCase(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase):

    @mock.patch('ironic.drivers.modules.fake.FakeConsole.start_console',
                autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.take_over',
                autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.prepare',
                autospec=True)
    def test__do_takeover(self, mock_prepare, mock_take_over,
                          mock_start_console):
        self._start_service()
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        task = task_manager.TaskManager(self.context, node.uuid)

        self.service._do_takeover(task)
        node.refresh()
        self.assertIsNone(node.last_error)
        self.assertFalse(node.console_enabled)
        mock_prepare.assert_called_once_with(task.driver.deploy, task)
        mock_take_over.assert_called_once_with(task.driver.deploy, task)
        self.assertFalse(mock_start_console.called)

    @mock.patch.object(notification_utils, 'emit_console_notification',
                       autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakeConsole.start_console',
                autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.take_over',
                autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.prepare',
                autospec=True)
    def test__do_takeover_with_console_enabled(self, mock_prepare,
                                               mock_take_over,
                                               mock_start_console,
                                               mock_notify):
        self._start_service()
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          console_enabled=True)
        task = task_manager.TaskManager(self.context, node.uuid)

        self.service._do_takeover(task)
        node.refresh()
        self.assertIsNone(node.last_error)
        self.assertTrue(node.console_enabled)
        mock_prepare.assert_called_once_with(task.driver.deploy, task)
        mock_take_over.assert_called_once_with(task.driver.deploy, task)
        mock_start_console.assert_called_once_with(task.driver.console, task)
        mock_notify.assert_has_calls(
            [mock.call(task, 'console_restore',
                       obj_fields.NotificationStatus.START),
             mock.call(task, 'console_restore',
                       obj_fields.NotificationStatus.END)])

    @mock.patch.object(notification_utils, 'emit_console_notification',
                       autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakeConsole.start_console',
                autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.take_over',
                autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.prepare',
                autospec=True)
    def test__do_takeover_with_console_exception(self, mock_prepare,
                                                 mock_take_over,
                                                 mock_start_console,
                                                 mock_notify):
        self._start_service()
        mock_start_console.side_effect = Exception()
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          console_enabled=True)
        task = task_manager.TaskManager(self.context, node.uuid)

        self.service._do_takeover(task)
        node.refresh()
        self.assertIsNotNone(node.last_error)
        self.assertFalse(node.console_enabled)
        mock_prepare.assert_called_once_with(task.driver.deploy, task)
        mock_take_over.assert_called_once_with(task.driver.deploy, task)
        mock_start_console.assert_called_once_with(task.driver.console, task)
        mock_notify.assert_has_calls(
            [mock.call(task, 'console_restore',
                       obj_fields.NotificationStatus.START),
             mock.call(task, 'console_restore',
                       obj_fields.NotificationStatus.ERROR)])


@mgr_utils.mock_record_keepalive
class DoNodeAdoptionTestCase(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase):

    def _fake_spawn(self, conductor_obj, func, *args, **kwargs):
        func(*args, **kwargs)
        return mock.MagicMock()

    @mock.patch('ironic.drivers.modules.fake.FakePower.validate',
                autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakeBoot.validate', autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakeConsole.start_console',
                autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.take_over',
                autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.prepare',
                autospec=True)
    def test__do_adoption_with_takeover(self,
                                        mock_prepare,
                                        mock_take_over,
                                        mock_start_console,
                                        mock_boot_validate,
                                        mock_power_validate):
        """Test a successful node adoption"""
        self._start_service()
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.ADOPTING)
        task = task_manager.TaskManager(self.context, node.uuid)

        self.service._do_adoption(task)
        node.refresh()

        self.assertEqual(states.ACTIVE, node.provision_state)
        self.assertIsNone(node.last_error)
        self.assertFalse(node.console_enabled)
        mock_prepare.assert_called_once_with(task.driver.deploy, task)
        mock_take_over.assert_called_once_with(task.driver.deploy, task)
        self.assertFalse(mock_start_console.called)
        mock_boot_validate.assert_not_called()

    @mock.patch('ironic.common.image_service.HttpImageService.validate_href',
                autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakePower.validate',
                autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakeBoot.validate', autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakeConsole.start_console',
                autospec=True)
    @mock.patch('ironic.drivers.modules.ramdisk.RamdiskDeploy.take_over',
                autospec=True)
    @mock.patch('ironic.drivers.modules.ramdisk.RamdiskDeploy.prepare',
                autospec=True)
    def test__do_adoption_ramdisk_deploy(self,
                                         mock_prepare,
                                         mock_take_over,
                                         mock_start_console,
                                         mock_boot_validate,
                                         mock_power_validate,
                                         mock_validate_href):
        """Test a successful node adoption"""
        self._start_service()
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            deploy_interface='ramdisk',
            provision_state=states.ADOPTING)
        task = task_manager.TaskManager(self.context, node.uuid)

        self.service._do_adoption(task)
        node.refresh()

        self.assertEqual(states.ACTIVE, node.provision_state)
        self.assertIsNone(node.last_error)
        self.assertFalse(node.console_enabled)
        mock_prepare.assert_called_once_with(task.driver.deploy, task)
        mock_take_over.assert_called_once_with(task.driver.deploy, task)
        self.assertFalse(mock_start_console.called)
        mock_boot_validate.assert_called_once_with(task.driver.boot, task)

    @mock.patch('ironic.drivers.modules.fake.FakeBoot.validate', autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakeConsole.start_console',
                autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.take_over',
                autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.prepare',
                autospec=True)
    def test__do_adoption_take_over_failure(self,
                                            mock_prepare,
                                            mock_take_over,
                                            mock_start_console,
                                            mock_boot_validate):
        """Test that adoption failed if an exception is raised"""
        # Note(TheJulia): Use of an actual possible exception that
        # can be raised due to a misconfiguration.
        mock_take_over.side_effect = exception.IPMIFailure(
            "something went wrong")

        self._start_service()
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.ADOPTING,
            power_state=states.POWER_ON)
        # NOTE(TheJulia): When nodes are created for adoption, they
        # would have no power state. Under normal circumstances
        # during validate the node object is updated with power state
        # however we need to make sure that we wipe preserved state
        # as part of failure handling.
        task = task_manager.TaskManager(self.context, node.uuid)

        self.service._do_adoption(task)
        node.refresh()

        self.assertEqual(states.ADOPTFAIL, node.provision_state)
        self.assertIsNotNone(node.last_error)
        self.assertFalse(node.console_enabled)
        mock_prepare.assert_called_once_with(task.driver.deploy, task)
        mock_take_over.assert_called_once_with(task.driver.deploy, task)
        self.assertFalse(mock_start_console.called)
        mock_boot_validate.assert_not_called()
        self.assertEqual(states.NOSTATE, node.power_state)

    @mock.patch('ironic.drivers.modules.fake.FakeBoot.validate', autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakeConsole.start_console',
                autospec=True)
    @mock.patch('ironic.drivers.modules.ramdisk.RamdiskDeploy.take_over',
                autospec=True)
    @mock.patch('ironic.drivers.modules.ramdisk.RamdiskDeploy.prepare',
                autospec=True)
    def test__do_adoption_boot_validate_failure(self,
                                                mock_prepare,
                                                mock_take_over,
                                                mock_start_console,
                                                mock_boot_validate):
        """Test that adoption fails if the boot validation fails"""
        # Note(TheJulia): Use of an actual possible exception that
        # can be raised due to a misconfiguration.
        mock_boot_validate.side_effect = exception.MissingParameterValue(
            "something is missing")

        self._start_service()
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            deploy_interface='ramdisk',
            provision_state=states.ADOPTING)
        task = task_manager.TaskManager(self.context, node.uuid)

        self.service._do_adoption(task)
        node.refresh()

        self.assertEqual(states.ADOPTFAIL, node.provision_state)
        self.assertIsNotNone(node.last_error)
        self.assertFalse(node.console_enabled)
        self.assertFalse(mock_prepare.called)
        self.assertFalse(mock_take_over.called)
        self.assertFalse(mock_start_console.called)
        mock_boot_validate.assert_called_once_with(task.driver.boot, task)

    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_do_provisioning_action_adopt_node(self, mock_spawn):
        """Test an adoption request results in the node in ADOPTING"""
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.MANAGEABLE,
            target_provision_state=states.NOSTATE)

        self._start_service()
        self.service.do_provisioning_action(self.context, node.uuid, 'adopt')
        node.refresh()
        self.assertEqual(states.ADOPTING, node.provision_state)
        self.assertEqual(states.ACTIVE, node.target_provision_state)
        self.assertIsNone(node.last_error)
        mock_spawn.assert_called_with(self.service,
                                      self.service._do_adoption, mock.ANY)

    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_do_provisioning_action_adopt_node_retry(self, mock_spawn):
        """Test a retried adoption from ADOPTFAIL results in ADOPTING state"""
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.ADOPTFAIL,
            target_provision_state=states.ACTIVE)

        self._start_service()
        self.service.do_provisioning_action(self.context, node.uuid, 'adopt')
        node.refresh()
        self.assertEqual(states.ADOPTING, node.provision_state)
        self.assertEqual(states.ACTIVE, node.target_provision_state)
        self.assertIsNone(node.last_error)
        mock_spawn.assert_called_with(self.service,
                                      self.service._do_adoption, mock.ANY)

    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.switch_interface',
                autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.take_over',
                autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.prepare',
                autospec=True)
    def test__do_adoption_calls_switch_interface(self,
                                                 mock_prepare,
                                                 mock_take_over,
                                                 mock_switch):
        """Test that adoption calls switch_interface before takeover."""
        self._start_service()
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.ADOPTING)
        task = task_manager.TaskManager(self.context, node.uuid)

        self.service._do_adoption(task)
        node.refresh()

        self.assertEqual(states.ACTIVE, node.provision_state)
        mock_switch.assert_called_once_with(task.driver.deploy, task)
        mock_prepare.assert_called_once_with(task.driver.deploy, task)
        mock_take_over.assert_called_once_with(task.driver.deploy, task)

    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.restore_interface',
                autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.switch_interface',
                autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.take_over',
                autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.prepare',
                autospec=True)
    def test__do_adoption_failure_calls_restore_interface(self,
                                                          mock_prepare,
                                                          mock_take_over,
                                                          mock_switch,
                                                          mock_restore):
        """Test that failed adoption calls restore_interface."""
        mock_take_over.side_effect = exception.IPMIFailure(
            "something went wrong")

        self._start_service()
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.ADOPTING,
            power_state=states.POWER_ON)
        task = task_manager.TaskManager(self.context, node.uuid)

        self.service._do_adoption(task)
        node.refresh()

        self.assertEqual(states.ADOPTFAIL, node.provision_state)
        mock_switch.assert_called_once_with(task.driver.deploy, task)
        mock_restore.assert_called_once_with(task.driver.deploy, task)

    def test_do_provisioning_action_manage_of_failed_adoption(self):
        """Test a node in ADOPTFAIL can be taken to MANAGEABLE"""
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.ADOPTFAIL,
            target_provision_state=states.ACTIVE)

        self._start_service()
        self.service.do_provisioning_action(self.context, node.uuid, 'manage')
        node.refresh()

        self.assertEqual(states.MANAGEABLE, node.provision_state)
        self.assertEqual(states.NOSTATE, node.target_provision_state)
        self.assertIsNone(node.last_error)


@mgr_utils.mock_record_keepalive
class HeartbeatTestCase(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase):

    def _fake_spawn(self, conductor_obj, func, *args, **kwargs):
        self.assertFalse(kwargs.pop('_allow_reserved_pool'))
        func(*args, **kwargs)
        return mock.MagicMock()

    # TODO(TheJulia): We should double check if these heartbeat tests need
    # to move. I have this strange feeling we were lacking rpc testing of
    # heartbeat until we did adoption testing....

    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.heartbeat',
                autospec=True)
    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_heartbeat_without_version(self, mock_spawn, mock_heartbeat):
        """Test heartbeating."""
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.DEPLOYING,
            target_provision_state=states.ACTIVE,
            driver_internal_info={'agent_secret_token': 'magic'})

        self._start_service()

        mock_spawn.reset_mock()

        mock_spawn.side_effect = self._fake_spawn

        exc = self.assertRaises(
            messaging.rpc.ExpectedException,
            self.service.heartbeat,
            self.context, node.uuid, 'http://callback',
            agent_token='magic')
        expected_string = ('Agent did not transmit a version, and a version '
                           'is required. Please update the agent being used.')
        self.assertEqual(exception.InvalidParameterValue, exc.exc_info[0])
        self.assertEqual(expected_string, str(exc.exc_info[1]))

    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.heartbeat',
                autospec=True)
    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_heartbeat_without_agent_version_anaconda(self, mock_spawn,
                                                      mock_heartbeat):
        """Test heartbeating anaconda deploy ramdisk without agent_version"""
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.DEPLOYING,
            target_provision_state=states.ACTIVE,
            driver_internal_info={'agent_secret_token': 'magic'})

        self._start_service()

        mock_spawn.reset_mock()

        mock_spawn.side_effect = self._fake_spawn

        self.service.heartbeat(self.context, node.uuid, 'https://callback',
                               agent_version=None, agent_token='magic',
                               agent_status='start')
        mock_heartbeat.assert_called_with(mock.ANY, mock.ANY,
                                          'https://callback', None,
                                          None, 'start', None)

    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.heartbeat',
                autospec=True)
    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_heartbeat_with_agent_version(self, mock_spawn, mock_heartbeat):
        """Test heartbeating."""
        self.config(require_tls=False, group='agent')
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.DEPLOYING,
            target_provision_state=states.ACTIVE,
            driver_internal_info={'agent_secret_token': 'magic'})

        self._start_service()

        mock_spawn.reset_mock()

        mock_spawn.side_effect = self._fake_spawn

        self.service.heartbeat(self.context, node.uuid, 'http://callback',
                               '1.4.1', agent_token='magic')
        mock_heartbeat.assert_called_with(mock.ANY, mock.ANY,
                                          'http://callback', '1.4.1', None,
                                          None, None)

    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.heartbeat',
                autospec=True)
    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_heartbeat_with_no_required_agent_token(self, mock_spawn,
                                                    mock_heartbeat):
        """Tests that we kill the heartbeat attempt very early on."""
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.DEPLOYING,
            target_provision_state=states.ACTIVE)

        self._start_service()

        mock_spawn.reset_mock()

        mock_spawn.side_effect = self._fake_spawn

        exc = self.assertRaises(
            messaging.rpc.ExpectedException, self.service.heartbeat,
            self.context, node.uuid, 'http://callback', agent_token=None)
        self.assertEqual(exception.InvalidParameterValue, exc.exc_info[0])
        self.assertFalse(mock_heartbeat.called)

    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.heartbeat',
                autospec=True)
    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_heartbeat_with_required_agent_token(self, mock_spawn,
                                                 mock_heartbeat):
        """Test heartbeat works when token matches."""
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.DEPLOYING,
            target_provision_state=states.ACTIVE,
            driver_internal_info={'agent_secret_token': 'a secret'})

        self._start_service()

        mock_spawn.reset_mock()

        mock_spawn.side_effect = self._fake_spawn

        self.service.heartbeat(self.context, node.uuid, 'https://callback',
                               '6.1.0', agent_token='a secret')
        mock_heartbeat.assert_called_with(mock.ANY, mock.ANY,
                                          'https://callback', '6.1.0', None,
                                          None, None)

    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.heartbeat',
                autospec=True)
    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_heartbeat_with_agent_token(self, mock_spawn,
                                        mock_heartbeat):
        """Test heartbeat works when token matches."""
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.DEPLOYING,
            target_provision_state=states.ACTIVE,
            driver_internal_info={'agent_secret_token': 'a secret'})

        self._start_service()

        mock_spawn.reset_mock()

        mock_spawn.side_effect = self._fake_spawn

        self.service.heartbeat(self.context, node.uuid, 'https://callback',
                               '6.1.0', agent_token='a secret')
        mock_heartbeat.assert_called_with(mock.ANY, mock.ANY,
                                          'https://callback', '6.1.0', None,
                                          None, None)

    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.heartbeat',
                autospec=True)
    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_heartbeat_invalid_agent_token(self, mock_spawn,
                                           mock_heartbeat):
        """Heartbeat fails when it does not match."""
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.DEPLOYING,
            target_provision_state=states.ACTIVE,
            driver_internal_info={'agent_secret_token': 'a secret'})

        self._start_service()

        mock_spawn.reset_mock()

        mock_spawn.side_effect = self._fake_spawn

        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.heartbeat, self.context,
                                node.uuid, 'https://callback',
                                agent_token='evil', agent_version='5.0.0b23')
        self.assertEqual(exception.InvalidParameterValue, exc.exc_info[0])
        self.assertFalse(mock_heartbeat.called)

    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.heartbeat',
                autospec=True)
    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_heartbeat_invalid_agent_token_older_version(
            self, mock_spawn, mock_heartbeat):
        """Heartbeat is rejected if token is received that is invalid."""
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.DEPLOYING,
            target_provision_state=states.ACTIVE,
            driver_internal_info={'agent_secret_token': 'a secret'})

        self._start_service()

        mock_spawn.reset_mock()

        mock_spawn.side_effect = self._fake_spawn
        # Intentionally sending an older client in case something fishy
        # occurs.
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.heartbeat, self.context,
                                node.uuid, 'https://callback',
                                agent_token='evil', agent_version='4.0.0')
        self.assertEqual(exception.InvalidParameterValue, exc.exc_info[0])
        self.assertFalse(mock_heartbeat.called)

    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.heartbeat',
                autospec=True)
    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_heartbeat_invalid_newer_version(
            self, mock_spawn, mock_heartbeat):
        """Heartbeat rejected if client should be sending a token."""
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.DEPLOYING,
            target_provision_state=states.ACTIVE)

        self._start_service()

        mock_spawn.reset_mock()

        mock_spawn.side_effect = self._fake_spawn

        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.heartbeat, self.context,
                                node.uuid, 'https://callback',
                                agent_token=None, agent_version='6.1.5')
        self.assertEqual(exception.InvalidParameterValue, exc.exc_info[0])
        self.assertFalse(mock_heartbeat.called)

    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.heartbeat',
                autospec=True)
    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_heartbeat_tls_required(self, mock_spawn, mock_heartbeat):
        """Heartbeat fails when it does not match."""
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.DEPLOYING,
            target_provision_state=states.ACTIVE,
            driver_internal_info={'agent_secret_token': 'a secret'})

        self._start_service()

        mock_spawn.reset_mock()

        mock_spawn.side_effect = self._fake_spawn
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.heartbeat, self.context,
                                node.uuid, 'http://callback',
                                agent_token='a secret',
                                agent_version='3.0.0')
        self.assertEqual(exception.InvalidParameterValue, exc.exc_info[0])
        self.assertIn('TLS is required', str(exc.exc_info[1]))
        self.assertFalse(mock_heartbeat.called)

    @mock.patch.object(conductor_utils, 'store_agent_certificate',
                       autospec=True)
    @mock.patch('ironic.drivers.modules.fake.FakeDeploy.heartbeat',
                autospec=True)
    @mock.patch('ironic.conductor.manager.ConductorManager._spawn_worker',
                autospec=True)
    def test_heartbeat_with_agent_verify_ca(self, mock_spawn,
                                            mock_heartbeat,
                                            mock_store_cert):
        node = obj_utils.create_test_node(
            self.context, driver='fake-hardware',
            provision_state=states.DEPLOYING,
            target_provision_state=states.ACTIVE,
            driver_internal_info={'agent_secret_token': 'a secret'})
        mock_store_cert.return_value = '/path/to/crt'

        self._start_service()

        mock_spawn.reset_mock()

        mock_spawn.side_effect = self._fake_spawn
        self.service.heartbeat(self.context, node.uuid, 'https://callback',
                               agent_version='6.1.0', agent_token='a secret',
                               agent_verify_ca='abcd')
        mock_heartbeat.assert_called_with(
            mock.ANY, mock.ANY, 'https://callback', '6.1.0', '/path/to/crt',
            None, None)


@mgr_utils.mock_record_keepalive
class DestroyVolumeConnectorTestCase(mgr_utils.ServiceSetUpMixin,
                                     db_base.DbTestCase):
    def test_destroy_volume_connector(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          power_state=states.POWER_OFF)

        volume_connector = obj_utils.create_test_volume_connector(
            self.context, node_id=node.id)
        self.service.destroy_volume_connector(self.context, volume_connector)
        self.assertRaises(exception.VolumeConnectorNotFound,
                          volume_connector.refresh)
        self.assertRaises(exception.VolumeConnectorNotFound,
                          self.dbapi.get_volume_connector_by_uuid,
                          volume_connector.uuid)

    def test_destroy_volume_connector_node_locked(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          reservation='fake-reserv')

        volume_connector = obj_utils.create_test_volume_connector(
            self.context, node_id=node.id)
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.destroy_volume_connector,
                                self.context, volume_connector)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NodeLocked, exc.exc_info[0])

    def test_destroy_volume_connector_node_power_on(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          power_state=states.POWER_ON)

        volume_connector = obj_utils.create_test_volume_connector(
            self.context, node_id=node.id)
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.destroy_volume_connector,
                                self.context, volume_connector)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.InvalidStateRequested, exc.exc_info[0])


@mgr_utils.mock_record_keepalive
class UpdateVolumeConnectorTestCase(mgr_utils.ServiceSetUpMixin,
                                    db_base.DbTestCase):
    def test_update_volume_connector(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          power_state=states.POWER_OFF)

        volume_connector = obj_utils.create_test_volume_connector(
            self.context, node_id=node.id, extra={'foo': 'bar'})
        new_extra = {'foo': 'baz'}
        volume_connector.extra = new_extra
        res = self.service.update_volume_connector(self.context,
                                                   volume_connector)
        self.assertEqual(new_extra, res.extra)

    def test_update_volume_connector_node_locked(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          reservation='fake-reserv')

        volume_connector = obj_utils.create_test_volume_connector(
            self.context, node_id=node.id)
        volume_connector.extra = {'foo': 'baz'}
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.update_volume_connector,
                                self.context, volume_connector)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NodeLocked, exc.exc_info[0])

    def test_update_volume_connector_type(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          power_state=states.POWER_OFF)
        volume_connector = obj_utils.create_test_volume_connector(
            self.context, node_id=node.id, extra={'vol_id': 'fake-id'})
        new_type = 'wwnn'
        volume_connector.type = new_type
        res = self.service.update_volume_connector(self.context,
                                                   volume_connector)
        self.assertEqual(new_type, res.type)

    def test_update_volume_connector_uuid(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          power_state=states.POWER_OFF)
        volume_connector = obj_utils.create_test_volume_connector(
            self.context, node_id=node.id)
        volume_connector.uuid = uuidutils.generate_uuid()
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.update_volume_connector,
                                self.context, volume_connector)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.InvalidParameterValue, exc.exc_info[0])

    def test_update_volume_connector_duplicate(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          power_state=states.POWER_OFF)
        volume_connector1 = obj_utils.create_test_volume_connector(
            self.context, node_id=node.id)
        volume_connector2 = obj_utils.create_test_volume_connector(
            self.context, node_id=node.id, uuid=uuidutils.generate_uuid(),
            type='diff_type')
        volume_connector2.type = volume_connector1.type
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.update_volume_connector,
                                self.context, volume_connector2)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.VolumeConnectorTypeAndIdAlreadyExists,
                         exc.exc_info[0])

    def test_update_volume_connector_node_power_on(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          power_state=states.POWER_ON)

        volume_connector = obj_utils.create_test_volume_connector(
            self.context, node_id=node.id)
        volume_connector.extra = {'foo': 'baz'}
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.update_volume_connector,
                                self.context, volume_connector)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.InvalidStateRequested, exc.exc_info[0])


@mgr_utils.mock_record_keepalive
class DestroyVolumeTargetTestCase(mgr_utils.ServiceSetUpMixin,
                                  db_base.DbTestCase):
    def test_destroy_volume_target(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          power_state=states.POWER_OFF)

        volume_target = obj_utils.create_test_volume_target(self.context,
                                                            node_id=node.id)
        self.service.destroy_volume_target(self.context, volume_target)
        self.assertRaises(exception.VolumeTargetNotFound,
                          volume_target.refresh)
        self.assertRaises(exception.VolumeTargetNotFound,
                          self.dbapi.get_volume_target_by_uuid,
                          volume_target.uuid)

    def test_destroy_volume_target_node_locked(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          reservation='fake-reserv')

        volume_target = obj_utils.create_test_volume_target(self.context,
                                                            node_id=node.id)
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.destroy_volume_target,
                                self.context, volume_target)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NodeLocked, exc.exc_info[0])

    def test_destroy_volume_target_node_gone(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware')
        volume_target = obj_utils.create_test_volume_target(self.context,
                                                            node_id=node.id)
        self.service.destroy_node(self.context, node.id)

        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.destroy_volume_target,
                                self.context, volume_target)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NodeNotFound, exc.exc_info[0])

    def test_destroy_volume_target_already_destroyed(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          power_state=states.POWER_OFF)
        volume_target = obj_utils.create_test_volume_target(self.context,
                                                            node_id=node.id)
        self.service.destroy_volume_target(self.context, volume_target)
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.destroy_volume_target,
                                self.context, volume_target)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.VolumeTargetNotFound, exc.exc_info[0])

    def test_destroy_volume_target_node_power_on(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          power_state=states.POWER_ON)

        volume_target = obj_utils.create_test_volume_target(self.context,
                                                            node_id=node.id)
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.destroy_volume_target,
                                self.context, volume_target)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.InvalidStateRequested, exc.exc_info[0])


@mgr_utils.mock_record_keepalive
class UpdateVolumeTargetTestCase(mgr_utils.ServiceSetUpMixin,
                                 db_base.DbTestCase):
    def test_update_volume_target(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          power_state=states.POWER_OFF)

        volume_target = obj_utils.create_test_volume_target(
            self.context, node_id=node.id, extra={'foo': 'bar'})
        new_extra = {'foo': 'baz'}
        volume_target.extra = new_extra
        res = self.service.update_volume_target(self.context, volume_target)
        self.assertEqual(new_extra, res.extra)

    def test_update_volume_target_node_locked(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          reservation='fake-reserv')
        volume_target = obj_utils.create_test_volume_target(self.context,
                                                            node_id=node.id)
        volume_target.extra = {'foo': 'baz'}
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.update_volume_target,
                                self.context, volume_target)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.NodeLocked, exc.exc_info[0])

    def test_update_volume_target_volume_type(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          power_state=states.POWER_OFF)
        volume_target = obj_utils.create_test_volume_target(
            self.context, node_id=node.id, extra={'vol_id': 'fake-id'})
        new_volume_type = 'fibre_channel'
        volume_target.volume_type = new_volume_type
        res = self.service.update_volume_target(self.context,
                                                volume_target)
        self.assertEqual(new_volume_type, res.volume_type)

    def test_update_volume_target_uuid(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          power_state=states.POWER_OFF)
        volume_target = obj_utils.create_test_volume_target(
            self.context, node_id=node.id)
        volume_target.uuid = uuidutils.generate_uuid()
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.update_volume_target,
                                self.context, volume_target)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.InvalidParameterValue, exc.exc_info[0])

    def test_update_volume_target_duplicate(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          power_state=states.POWER_OFF)
        volume_target1 = obj_utils.create_test_volume_target(
            self.context, node_id=node.id)
        volume_target2 = obj_utils.create_test_volume_target(
            self.context, node_id=node.id, uuid=uuidutils.generate_uuid(),
            boot_index=volume_target1.boot_index + 1)
        volume_target2.boot_index = volume_target1.boot_index
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.update_volume_target,
                                self.context, volume_target2)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.VolumeTargetBootIndexAlreadyExists,
                         exc.exc_info[0])

    def _test_update_volume_target_exception(self, expected_exc):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          power_state=states.POWER_OFF)
        volume_target = obj_utils.create_test_volume_target(
            self.context, node_id=node.id, extra={'vol_id': 'fake-id'})
        new_volume_type = 'fibre_channel'
        volume_target.volume_type = new_volume_type
        with mock.patch.object(objects.VolumeTarget, 'save',
                               autospec=True) as mock_save:
            mock_save.side_effect = expected_exc('Boo')
            exc = self.assertRaises(messaging.rpc.ExpectedException,
                                    self.service.update_volume_target,
                                    self.context, volume_target)
            # Compare true exception hidden by @messaging.expected_exceptions
            self.assertEqual(expected_exc, exc.exc_info[0])

    def test_update_volume_target_node_not_found(self):
        self._test_update_volume_target_exception(exception.NodeNotFound)

    def test_update_volume_target_not_found(self):
        self._test_update_volume_target_exception(
            exception.VolumeTargetNotFound)

    def test_update_volume_target_node_power_on(self):
        node = obj_utils.create_test_node(self.context, driver='fake-hardware',
                                          power_state=states.POWER_ON)
        volume_target = obj_utils.create_test_volume_target(self.context,
                                                            node_id=node.id)
        volume_target.extra = {'foo': 'baz'}
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.update_volume_target,
                                self.context, volume_target)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(exception.InvalidStateRequested, exc.exc_info[0])


@mgr_utils.mock_record_keepalive
class NodeTraitsTestCase(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase):

    def setUp(self):
        super(NodeTraitsTestCase, self).setUp()
        self.traits = ['trait1', 'trait2']
        self.node = obj_utils.create_test_node(self.context,
                                               driver='fake-hardware')

    def test_add_node_traits(self):
        self.service.add_node_traits(self.context, self.node.id,
                                     self.traits[:1])
        traits = objects.TraitList.get_by_node_id(self.context, self.node.id)
        self.assertEqual(self.traits[:1], [trait.trait for trait in traits])

        self.service.add_node_traits(self.context, self.node.id,
                                     self.traits[1:])
        traits = objects.TraitList.get_by_node_id(self.context, self.node.id)
        self.assertEqual(self.traits, [trait.trait for trait in traits])

    def test_add_node_traits_replace(self):
        self.service.add_node_traits(self.context, self.node.id,
                                     self.traits[:1], replace=True)
        traits = objects.TraitList.get_by_node_id(self.context, self.node.id)
        self.assertEqual(self.traits[:1], [trait.trait for trait in traits])

        self.service.add_node_traits(self.context, self.node.id,
                                     self.traits[1:], replace=True)
        traits = objects.TraitList.get_by_node_id(self.context, self.node.id)
        self.assertEqual(self.traits[1:], [trait.trait for trait in traits])

    def _test_add_node_traits_exception(self, expected_exc):
        with mock.patch.object(objects.Trait, 'create',
                               autospec=True) as mock_create:
            mock_create.side_effect = expected_exc('Boo')
            exc = self.assertRaises(messaging.rpc.ExpectedException,
                                    self.service.add_node_traits, self.context,
                                    self.node.id, self.traits)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(expected_exc, exc.exc_info[0])
        traits = objects.TraitList.get_by_node_id(self.context, self.node.id)
        self.assertEqual([], traits.objects)

    def test_add_node_traits_invalid_parameter_value(self):
        self._test_add_node_traits_exception(exception.InvalidParameterValue)

    def test_add_node_traits_node_locked(self):
        self._test_add_node_traits_exception(exception.NodeLocked)

    def test_add_node_traits_node_not_found(self):
        self._test_add_node_traits_exception(exception.NodeNotFound)

    def test_remove_node_traits(self):
        objects.TraitList.create(self.context, self.node.id, self.traits)
        self.service.remove_node_traits(self.context, self.node.id,
                                        self.traits[:1])
        traits = objects.TraitList.get_by_node_id(self.context, self.node.id)
        self.assertEqual(self.traits[1:], [trait.trait for trait in traits])

        self.service.remove_node_traits(self.context, self.node.id,
                                        self.traits[1:])
        traits = objects.TraitList.get_by_node_id(self.context, self.node.id)
        self.assertEqual([], traits.objects)

    def test_remove_node_traits_all(self):
        objects.TraitList.create(self.context, self.node.id, self.traits)
        self.service.remove_node_traits(self.context, self.node.id, None)
        traits = objects.TraitList.get_by_node_id(self.context, self.node.id)
        self.assertEqual([], traits.objects)

    def test_remove_node_traits_empty(self):
        objects.TraitList.create(self.context, self.node.id, self.traits)
        self.service.remove_node_traits(self.context, self.node.id, [])
        traits = objects.TraitList.get_by_node_id(self.context, self.node.id)
        self.assertEqual(self.traits, [trait.trait for trait in traits])

    def _test_remove_node_traits_exception(self, expected_exc):
        objects.TraitList.create(self.context, self.node.id, self.traits)
        with mock.patch.object(objects.Trait, 'destroy',
                               autospec=True) as mock_destroy:
            mock_destroy.side_effect = expected_exc('Boo')
            exc = self.assertRaises(messaging.rpc.ExpectedException,
                                    self.service.remove_node_traits,
                                    self.context, self.node.id, self.traits)
        # Compare true exception hidden by @messaging.expected_exceptions
        self.assertEqual(expected_exc, exc.exc_info[0])
        traits = objects.TraitList.get_by_node_id(self.context, self.node.id)
        self.assertEqual(self.traits, [trait.trait for trait in traits])

    def test_remove_node_traits_node_locked(self):
        self._test_remove_node_traits_exception(exception.NodeLocked)

    def test_remove_node_traits_node_not_found(self):
        self._test_remove_node_traits_exception(exception.NodeNotFound)

    def test_remove_node_traits_node_trait_not_found(self):
        self._test_remove_node_traits_exception(exception.NodeTraitNotFound)


@mgr_utils.mock_record_keepalive
@mock.patch('ironic.drivers.modules.fake.FakeInspect.abort', autospec=True)
@mock.patch('ironic.conductor.task_manager.acquire', autospec=True)
class DoNodeInspectAbortTestCase(mgr_utils.CommonMixIn,
                                 mgr_utils.ServiceSetUpMixin,
                                 db_base.DbTestCase):
    @mock.patch.object(inspection, 'LOG', autospec=True)
    def test_do_inspect_abort_interface_not_support(self, mock_log,
                                                    mock_acquire, mock_abort):
        node = obj_utils.create_test_node(self.context,
                                          driver='fake-hardware',
                                          provision_state=states.INSPECTWAIT)
        task = task_manager.TaskManager(self.context, node.uuid)
        mock_acquire.side_effect = self._get_acquire_side_effect(task)
        mock_abort.side_effect = exception.UnsupportedDriverExtension(
            driver='fake-hardware', extension='inspect')
        self._start_service()
        exc = self.assertRaises(messaging.rpc.ExpectedException,
                                self.service.do_provisioning_action,
                                self.context, task.node.uuid,
                                "abort")
        self.assertEqual(exception.UnsupportedDriverExtension,
                         exc.exc_info[0])
        self.assertTrue(mock_log.error.called)

    @mock.patch.object(inspection, 'LOG', autospec=True)
    def test_do_inspect_abort_interface_return_failed(self, mock_log,
                                                      mock_acquire,
                                                      mock_abort):
        mock_abort.side_effect = exception.IronicException('Oops')
        self._start_service()
        node = obj_utils.create_test_node(self.context,
                                          driver='fake-hardware',
                                          provision_state=states.INSPECTWAIT)
        task = task_manager.TaskManager(self.context, node.uuid)
        mock_acquire.side_effect = self._get_acquire_side_effect(task)
        self.assertRaises(exception.IronicException,
                          self.service.do_provisioning_action,
                          self.context, task.node.uuid,
                          "abort")
        node.refresh()
        self.assertTrue(mock_log.exception.called)
        self.assertIn('Failed to abort inspection', node.last_error)

    def test_do_inspect_abort_succeeded(self, mock_acquire, mock_abort):
        self._start_service()
        node = obj_utils.create_test_node(
            self.context,
            driver='fake-hardware',
            provision_state=states.INSPECTWAIT,
            driver_internal_info={'agent_url': 'url',
                                  'agent_secret_token': 'token'})
        task = task_manager.TaskManager(self.context, node.uuid)
        mock_acquire.side_effect = self._get_acquire_side_effect(task)
        self.service.do_provisioning_action(self.context, task.node.uuid,
                                            "abort")
        node.refresh()
        self.assertEqual(states.INSPECTFAIL, node.provision_state)
        self.assertIn('Inspection was aborted', node.last_error)
        self.assertNotIn('agent_url', node.driver_internal_info)
        self.assertNotIn('agent_secret_token', node.driver_internal_info)

    def test_do_inspect_abort_state_set_by_driver(self, mock_acquire,
                                                  mock_abort):
        def abort(driver, task):
            task.process_event('abort')

        mock_abort.side_effect = abort
        self._start_service()
        node = obj_utils.create_test_node(
            self.context,
            driver='fake-hardware',
            provision_state=states.INSPECTWAIT,
            driver_internal_info={'agent_url': 'url',
                                  'agent_secret_token': 'token'})
        task = task_manager.TaskManager(self.context, node.uuid)
        mock_acquire.side_effect = self._get_acquire_side_effect(task)
        self.service.do_provisioning_action(self.context, task.node.uuid,
                                            "abort")
        node.refresh()
        self.assertEqual(states.INSPECTFAIL, node.provision_state)
        self.assertIn('Inspection was aborted', node.last_error)
        self.assertNotIn('agent_url', node.driver_internal_info)
        self.assertNotIn('agent_secret_token', node.driver_internal_info)


class NodeHistoryRecordCleanupTestCase(mgr_utils.ServiceSetUpMixin,
                                       db_base.DbTestCase):

    def setUp(self):
        super(NodeHistoryRecordCleanupTestCase, self).setUp()
        CONF.set_override('node_history_max_entries', 2, group='conductor')
        CONF.set_override('node_history_cleanup_batch_count', 2,
                          group='conductor')
        self._start_service()
        self.node1 = obj_utils.get_test_node(
            self.context,
            driver='fake-hardware',
            id=10,
            uuid=uuidutils.generate_uuid(),
            conductor_affinity=self.service.conductor.id)
        self.node2 = obj_utils.get_test_node(
            self.context,
            driver='fake-hardware',
            id=11,
            uuid=uuidutils.generate_uuid(),
            conductor_affinity=self.service.conductor.id)
        self.node3 = obj_utils.get_test_node(
            self.context,
            driver='fake-hardware',
            id=12,
            uuid=uuidutils.generate_uuid(),
            conductor_affinity=self.service.conductor.id)
        self.nodes = [self.node1, self.node2, self.node3]
        # Create the nodes, as the tasks need to operate across tables.
        self.node1.create()
        self.node2.create()
        self.node3.create()

    def test_history_is_pruned_to_config(self):
        for node in self.nodes:
            for event in ['one', 'two', 'three']:
                conductor_utils.node_history_record(node, event=event)
        conductor_utils.node_history_record(self.node1, event="final")
        self.service._manage_node_history(self.context)
        events = objects.NodeHistory.list(self.context)
        self.assertEqual(8, len(events))
        events = objects.NodeHistory.list_by_node_id(self.context, 10)
        self.assertEqual('three', events[0].event)
        self.assertEqual('final', events[1].event)
        self.assertEqual(2, len(events))
        events = objects.NodeHistory.list_by_node_id(self.context, 11)
        self.assertEqual(3, len(events))
        self.assertEqual('one', events[0].event)
        self.assertEqual('two', events[1].event)
        self.assertEqual('three', events[2].event)
        events = objects.NodeHistory.list_by_node_id(self.context, 12)
        self.assertEqual(3, len(events))
        self.assertEqual('one', events[0].event)
        self.assertEqual('two', events[1].event)
        self.assertEqual('three', events[2].event)

    def test_history_is_pruned_to_config_two_pass(self):
        for node in self.nodes:
            for event in ['one', 'two', 'three']:
                conductor_utils.node_history_record(node, event=event)
        conductor_utils.node_history_record(self.node1, event="final")
        with mock.patch.object(manager.LOG, 'warning',
                               autospec=True) as mock_log:
            self.service._manage_node_history(self.context)
            # assert we did call a warning entry
            self.assertEqual(1, mock_log.call_count)
        events = objects.NodeHistory.list(self.context)
        self.assertEqual(8, len(events))
        self.service._manage_node_history(self.context)
        events = objects.NodeHistory.list(self.context)
        self.assertEqual(6, len(events))
        events = objects.NodeHistory.list_by_node_id(self.context, 10)
        self.assertEqual(2, len(events))
        self.assertEqual('three', events[0].event)
        self.assertEqual('final', events[1].event)
        events = objects.NodeHistory.list_by_node_id(self.context, 11)
        self.assertEqual(2, len(events))
        self.assertEqual('two', events[0].event)
        self.assertEqual('three', events[1].event)
        events = objects.NodeHistory.list_by_node_id(self.context, 12)
        self.assertEqual(2, len(events))
        self.assertEqual('two', events[0].event)
        self.assertEqual('three', events[1].event)

    def test_history_is_pruned_from_all_nodes_one_pass(self):
        CONF.set_override('node_history_cleanup_batch_count', 15,
                          group='conductor')
        for node in self.nodes:
            for event in ['one', 'two', 'three']:
                conductor_utils.node_history_record(node, event=event)
        self.service._manage_node_history(self.context)
        events = objects.NodeHistory.list(self.context)
        self.assertEqual(6, len(events))

    def test_history_pruning_no_work(self):
        conductor_utils.node_history_record(self.node1, event='meow')
        with mock.patch.object(self.dbapi,
                               'bulk_delete_node_history_records',
                               autospec=True) as mock_delete:
            self.service._manage_node_history(self.context)
            mock_delete.assert_not_called()
        events = objects.NodeHistory.list(self.context)
        self.assertEqual(1, len(events))

    def test_history_pruning_not_other_conductor(self):
        another_conductor = obj_utils.create_test_conductor(self.context)
        node = obj_utils.get_test_node(self.context,
                                       driver='fake-hardware',
                                       id=33,
                                       uuid=uuidutils.generate_uuid(),
                                       conductor_affinity=another_conductor.id)
        # create node so it can be queried
        node.create()
        for i in range(0, 3):
            conductor_utils.node_history_record(node, event='meow%s' % i)
        with mock.patch.object(self.dbapi,
                               'bulk_delete_node_history_records',
                               autospec=True) as mock_delete:
            self.service._manage_node_history(self.context)
            mock_delete.assert_not_called()
        events = objects.NodeHistory.list(self.context)
        self.assertEqual(3, len(events))
        self.assertEqual('meow0', events[0].event)

    def test_history_is_pruned_to_config_with_days(self):
        CONF.set_override('node_history_cleanup_batch_count', 15,
                          group='conductor')
        CONF.set_override('node_history_minimum_days', 1,
                          group='conductor')
        CONF.set_override('node_history_max_entries', 1, group='conductor')

        old_date = datetime.datetime.now() - datetime.timedelta(days=7)
        # Creates 18 entries
        for node in self.nodes:
            for event in ['oldone', 'oldtwo', 'oldthree']:
                objects.NodeHistory(created_at=old_date,
                                    event=event,
                                    node_id=node.id).create()
            for event in ['one', 'two', 'three']:
                conductor_utils.node_history_record(node, event=event)

        # 9 retained due to days, 3 to config
        self.service._manage_node_history(self.context)
        events = objects.NodeHistory.list(self.context)
        self.assertEqual(12, len(events))
        events = objects.NodeHistory.list_by_node_id(self.context, 10)
        self.assertEqual(4, len(events))
        self.assertEqual('oldthree', events[0].event)
        self.assertEqual('one', events[1].event)
        self.assertEqual('two', events[2].event)
        self.assertEqual('three', events[3].event)
        events = objects.NodeHistory.list_by_node_id(self.context, 11)
        self.assertEqual(4, len(events))
        self.assertEqual('oldthree', events[0].event)
        self.assertEqual('one', events[1].event)
        self.assertEqual('two', events[2].event)
        self.assertEqual('three', events[3].event)
        events = objects.NodeHistory.list_by_node_id(self.context, 12)
        self.assertEqual(4, len(events))
        self.assertEqual('oldthree', events[0].event)
        self.assertEqual('one', events[1].event)
        self.assertEqual('two', events[2].event)
        self.assertEqual('three', events[3].event)


class ConcurrentActionLimitTestCase(mgr_utils.ServiceSetUpMixin,
                                    db_base.DbTestCase):

    def setUp(self):
        super(ConcurrentActionLimitTestCase, self).setUp()
        self._start_service()
        self.node1 = obj_utils.get_test_node(
            self.context,
            driver='fake-hardware',
            id=110,
            uuid=uuidutils.generate_uuid())
        self.node2 = obj_utils.get_test_node(
            self.context,
            driver='fake-hardware',
            id=111,
            uuid=uuidutils.generate_uuid())
        self.node3 = obj_utils.get_test_node(
            self.context,
            driver='fake-hardware',
            id=112,
            uuid=uuidutils.generate_uuid())
        self.node4 = obj_utils.get_test_node(
            self.context,
            driver='fake-hardware',
            id=113,
            uuid=uuidutils.generate_uuid())
        # Create the nodes, as the tasks need to operate across tables.
        self.node1.create()
        self.node2.create()
        self.node3.create()
        self.node4.create()

    def test_concurrent_action_limit_deploy(self):
        self.node1.provision_state = states.DEPLOYING
        self.node2.provision_state = states.DEPLOYWAIT
        self.node1.save()
        self.node2.save()
        CONF.set_override('max_concurrent_deploy', 2, group='conductor')
        self.assertRaises(
            exception.ConcurrentActionLimit,
            self.service._concurrent_action_limit,
            'provisioning')
        self.service._concurrent_action_limit('unprovisioning')
        self.service._concurrent_action_limit('cleaning')
        CONF.set_override('max_concurrent_deploy', 3, group='conductor')
        self.service._concurrent_action_limit('provisioning')

    def test_concurrent_action_limit_cleaning(self):
        self.node1.provision_state = states.DELETING
        self.node2.provision_state = states.CLEANING
        self.node3.provision_state = states.CLEANWAIT
        self.node1.save()
        self.node2.save()
        self.node3.save()

        CONF.set_override('max_concurrent_clean', 3, group='conductor')
        self.assertRaises(
            exception.ConcurrentActionLimit,
            self.service._concurrent_action_limit,
            'cleaning')
        self.assertRaises(
            exception.ConcurrentActionLimit,
            self.service._concurrent_action_limit,
            'unprovisioning')
        self.service._concurrent_action_limit('provisioning')
        CONF.set_override('max_concurrent_clean', 4, group='conductor')
        self.service._concurrent_action_limit('cleaning')
        self.service._concurrent_action_limit('unprovisioning')

    def test_concurrent_action_limit_per_conductor_group_deploy(self):
        # Set up nodes in different conductor groups
        self.node1.provision_state = states.DEPLOYING
        self.node1.conductor_group = 'group-a'
        self.node2.provision_state = states.DEPLOYWAIT
        self.node2.conductor_group = 'group-a'
        self.node3.provision_state = states.DEPLOYING
        self.node3.conductor_group = 'group-b'
        self.node1.save()
        self.node2.save()
        self.node3.save()

        # Enable per-conductor-group limiting
        CONF.set_override('max_concurrent_per_conductor_group', True,
                          group='conductor')
        CONF.set_override('conductor_group', 'group-a', group='conductor')
        CONF.set_override('max_concurrent_deploy', 2, group='conductor')

        # Should hit limit for group-a (2 nodes deploying)
        self.assertRaises(
            exception.ConcurrentActionLimit,
            self.service._concurrent_action_limit,
            'provisioning')

        # Switch to group-b, should not hit limit (only 1 node deploying)
        CONF.set_override('conductor_group', 'group-b', group='conductor')
        self.service._concurrent_action_limit('provisioning')

        # Disable per-conductor-group limiting, should hit global limit
        CONF.set_override('max_concurrent_per_conductor_group', False,
                          group='conductor')
        CONF.set_override('max_concurrent_deploy', 3, group='conductor')
        self.assertRaises(
            exception.ConcurrentActionLimit,
            self.service._concurrent_action_limit,
            'provisioning')

    def test_concurrent_action_limit_per_conductor_group_cleaning(self):
        # Set up nodes in different conductor groups
        self.node1.provision_state = states.CLEANING
        self.node1.conductor_group = 'group-a'
        self.node2.provision_state = states.CLEANWAIT
        self.node2.conductor_group = 'group-a'
        self.node3.provision_state = states.DELETING
        self.node3.conductor_group = 'group-b'
        self.node1.save()
        self.node2.save()
        self.node3.save()

        # Enable per-conductor-group limiting
        CONF.set_override('max_concurrent_per_conductor_group', True,
                          group='conductor')
        CONF.set_override('conductor_group', 'group-a', group='conductor')
        CONF.set_override('max_concurrent_clean', 2, group='conductor')

        # Should hit limit for group-a (2 nodes in cleaning states)
        self.assertRaises(
            exception.ConcurrentActionLimit,
            self.service._concurrent_action_limit,
            'cleaning')

        # Switch to group-b, should not hit limit (only 1 node)
        CONF.set_override('conductor_group', 'group-b', group='conductor')
        self.service._concurrent_action_limit('cleaning')


@mgr_utils.mock_record_keepalive
class ContinueInspectionTestCase(mgr_utils.ServiceSetUpMixin,
                                 db_base.DbTestCase):

    @mock.patch.object(manager.ConductorManager, '_spawn_worker',
                       autospec=True)
    def test_continue_ok(self, mock_spawn):
        node = obj_utils.create_test_node(self.context,
                                          provision_state=states.INSPECTWAIT)
        self.service.continue_inspection(self.context, node.id,
                                         {"test": "inventory"},
                                         ["plugin data"])
        node.refresh()
        self.assertEqual(states.INSPECTING, node.provision_state)
        mock_spawn.assert_called_once_with(
            self.service, inspection.continue_inspection, mock.ANY,
            {"test": "inventory"}, ["plugin data"])

    @mock.patch.object(manager.ConductorManager, '_spawn_worker',
                       autospec=True)
    def test_continue_with_discovery(self, mock_spawn):
        CONF.set_override('enabled', True, group='auto_discovery')
        node = obj_utils.create_test_node(self.context,
                                          provision_state=states.ENROLL)
        self.service.continue_inspection(self.context, node.id,
                                         {"test": "inventory"},
                                         ["plugin data"])
        node.refresh()
        self.assertEqual(states.ENROLL, node.provision_state)
        mock_spawn.assert_called_once_with(
            self.service, inspection.continue_inspection, mock.ANY,
            {"test": "inventory"}, ["plugin data"])

    def test_wrong_state(self):
        for state in (states.ENROLL, states.AVAILABLE, states.ACTIVE):
            node = obj_utils.create_test_node(self.context,
                                              uuid=uuidutils.generate_uuid(),
                                              provision_state=state)
            exc = self.assertRaises(messaging.rpc.ExpectedException,
                                    self.service.continue_inspection,
                                    self.context, node.id,
                                    {"test": "inventory"},
                                    ["plugin data"])
            self.assertEqual(exception.NotFound, exc.exc_info[0])
            node.refresh()
            self.assertEqual(state, node.provision_state)


@mock.patch.object(
    task_manager.TaskManager, 'spawn_after',
    lambda self, _spawn, func, *args, **kwargs: func(*args, **kwargs))
@mgr_utils.mock_record_keepalive
class VirtualMediaTestCase(mgr_utils.ServiceSetUpMixin, db_base.DbTestCase):

    def setUp(self):
        super().setUp()
        self.config(enabled_hardware_types=['redfish'],
                    enabled_power_interfaces=['redfish'],
                    enabled_boot_interfaces=['redfish-virtual-media'],
                    enabled_management_interfaces=['redfish'],
                    enabled_inspect_interfaces=['redfish'],
                    enabled_bios_interfaces=['redfish'])
        self.node = obj_utils.create_test_node(
            self.context, driver='redfish', driver_info=INFO_DICT,
            provision_state=states.ACTIVE)

    @mock.patch.object(image_utils, 'ISOImageCache', autospec=True)
    @mock.patch.object(redfish_mgmt.RedfishManagement, 'validate',
                       autospec=True)
    @mock.patch.object(manager, 'do_attach_virtual_media',
                       autospec=True)
    def test_attach_virtual_media_local(self, mock_attach, mock_validate,
                                        mock_cache):
        CONF.set_override('use_swift', 'false', group='redfish')
        self.service.attach_virtual_media(self.context, self.node.id,
                                          boot_devices.CDROM,
                                          'https://url')
        mock_validate.assert_called_once_with(mock.ANY, mock.ANY)
        mock_attach.assert_called_once_with(
            mock.ANY, device_type=boot_devices.CDROM,
            image_url='https://url', image_download_source='local')
        self.node.refresh()
        self.assertIsNone(self.node.last_error)

    @mock.patch.object(redfish_mgmt.RedfishManagement, 'validate',
                       autospec=True)
    @mock.patch.object(manager, 'do_attach_virtual_media', autospec=True)
    def test_attach_virtual_media_http(self, mock_attach, mock_validate):
        self.service.attach_virtual_media(self.context, self.node.id,
                                          boot_devices.CDROM,
                                          'https://url',
                                          image_download_source='http')
        mock_validate.assert_called_once_with(mock.ANY, mock.ANY)
        mock_attach.assert_called_once_with(
            mock.ANY, device_type=boot_devices.CDROM,
            image_url='https://url', image_download_source='http')
        self.node.refresh()
        self.assertIsNone(self.node.last_error)

    @mock.patch.object(redfish_mgmt.RedfishManagement,
                       'attach_virtual_media', autospec=True)
    @mock.patch.object(image_utils, 'cleanup_remote_image', autospec=True)
    @mock.patch.object(image_utils, 'prepare_remote_image', autospec=True)
    def test_do_attach_virtual_media(self, mock_prepare_image,
                                     mock_cleanup_image, mock_attach):
        with task_manager.acquire(self.context, self.node.id) as task:
            manager.do_attach_virtual_media(task, boot_devices.CDROM,
                                            "https://url", "local")
            file_name = f"cdrom-{self.node.uuid}.iso"
            mock_prepare_image.assert_called_once_with(
                task, "https://url", file_name=file_name,
                download_source="local")
            mock_cleanup_image.assert_called_once_with(task, file_name)
            mock_attach.assert_called_once_with(
                task.driver.management, task, device_type=boot_devices.CDROM,
                image_url=mock_prepare_image.return_value)

    @mock.patch.object(redfish_mgmt.RedfishManagement,
                       'attach_virtual_media', autospec=True)
    @mock.patch.object(image_utils, 'cleanup_remote_image', autospec=True)
    @mock.patch.object(image_utils, 'prepare_remote_image', autospec=True)
    def test_do_attach_virtual_media_fails_on_prepare(self, mock_prepare_image,
                                                      mock_cleanup_image,
                                                      mock_attach):
        mock_prepare_image.side_effect = exception.InvalidImageRef
        with task_manager.acquire(self.context, self.node.id) as task:
            manager.do_attach_virtual_media(task, boot_devices.CDROM,
                                            "https://url", "local")
            file_name = f"cdrom-{self.node.uuid}.iso"
            mock_prepare_image.assert_called_once_with(
                task, "https://url", file_name=file_name,
                download_source="local")
            mock_cleanup_image.assert_called_once_with(task, file_name)
            mock_attach.assert_not_called()
        self.node.refresh()
        self.assertIn("Could not attach device cdrom", self.node.last_error)
        self.assertIn("Invalid image href", self.node.last_error)

    @mock.patch.object(redfish_mgmt.RedfishManagement,
                       'attach_virtual_media', autospec=True)
    @mock.patch.object(image_utils, 'cleanup_remote_image', autospec=True)
    @mock.patch.object(image_utils, 'prepare_remote_image', autospec=True)
    def test_do_attach_virtual_media_fails_on_attach(self, mock_prepare_image,
                                                     mock_cleanup_image,
                                                     mock_attach):
        mock_attach.side_effect = exception.UnsupportedDriverExtension
        with task_manager.acquire(self.context, self.node.id) as task:
            manager.do_attach_virtual_media(task, boot_devices.CDROM,
                                            "https://url", "local")
            file_name = f"cdrom-{self.node.uuid}.iso"
            mock_prepare_image.assert_called_once_with(
                task, "https://url", file_name=file_name,
                download_source="local")
            mock_attach.assert_called_once_with(
                task.driver.management, task, device_type=boot_devices.CDROM,
                image_url=mock_prepare_image.return_value)
        self.node.refresh()
        self.assertIn("Could not attach device cdrom", self.node.last_error)
        self.assertIn("disabled or not implemented", self.node.last_error)

    @mock.patch.object(redfish_mgmt.RedfishManagement, 'validate',
                       autospec=True)
    def test_attach_virtual_media_power_failure(self, mock_validate):
        CONF.set_override('use_swift', 'false', group='redfish')
        self.node.maintenance = True
        self.node.fault = faults.POWER_FAILURE
        self.node.save()
        self.assertRaises(
            exception.TemporaryFailure,
            self.service.attach_virtual_media, self.context, self.node.id,
            boot_devices.CDROM, 'https://url')
        mock_validate.assert_called_once_with(mock.ANY, mock.ANY)

    @mock.patch.object(redfish_mgmt.RedfishManagement, 'validate',
                       autospec=True)
    def test_detach_virtual_media_power_failure(self, mock_validate):
        CONF.set_override('use_swift', 'false', group='redfish')
        self.node.maintenance = True
        self.node.fault = faults.POWER_FAILURE
        self.node.save()
        self.assertRaises(
            exception.TemporaryFailure,
            self.service.detach_virtual_media, self.context, self.node.id,
            boot_devices.CDROM)
        mock_validate.assert_called_once_with(mock.ANY, mock.ANY)

    @mock.patch.object(redfish_mgmt.RedfishManagement,
                       'get_virtual_media', autospec=True)
    @mock.patch.object(redfish_mgmt.RedfishManagement, 'validate',
                       autospec=True)
    def test_get_virtual_media(self, mock_validate, mock_get):
        mock_get.return_value = [{'media_types': ['CD'], 'inserted': False,
                                  'image': ''}]
        result = self.service.get_virtual_media(self.context, self.node.id)
        mock_validate.assert_called_once_with(mock.ANY, mock.ANY)
        mock_get.assert_called_once_with(mock.ANY, mock.ANY)
        self.assertEqual(mock_get.return_value, result)

    @mock.patch.object(redfish_mgmt.RedfishManagement,
                       'get_virtual_media', autospec=True)
    @mock.patch.object(redfish_mgmt.RedfishManagement, 'validate',
                       autospec=True)
    def test_get_virtual_media_node_locked(self, mock_validate, mock_get):
        """get_virtual_media uses a shared lock.

        Succeeds even when the node is reserved by another operation.
        """
        self.node.reservation = 'fake-reserv'
        self.node.save()
        mock_get.return_value = [{'media_types': ['CD'], 'inserted': False,
                                  'image': ''}]
        result = self.service.get_virtual_media(self.context, self.node.id)
        mock_validate.assert_called_once_with(mock.ANY, mock.ANY)
        mock_get.assert_called_once_with(mock.ANY, mock.ANY)
        self.assertEqual(mock_get.return_value, result)


class ConductorCleanupTestCase(mgr_utils.ServiceSetUpMixin,
                               db_base.DbTestCase):
    """Test stale conductors cleanup."""
    def test_cleanup_stale_conductors(self):
        """Test the periodic cleanup of stale conductors."""
        self._start_service()

        self.conductor1 = self.dbapi.register_conductor({
            'hostname': 'conductor-1',
            'drivers': ['fake'],
            'conductor_group': 'group1'
        })
        self.conductor2 = self.dbapi.register_conductor({
            'hostname': 'conductor-2',
            'drivers': ['fake'],
            'conductor_group': 'group2'
        })

        self.dbapi.touch_conductor('conductor-1', online=False)

        with mock.patch.object(self.dbapi, 'get_offline_conductors',
                               return_value=['conductor-1'], autospec=True):
            self.service._cleanup_stale_conductors(self.context)

        self.assertRaises(exception.ConductorNotFound,
                          self.dbapi.get_conductor, 'conductor-1')

        conductor2 = self.dbapi.get_conductor('conductor-2')
        self.assertEqual('conductor-2', conductor2.hostname)

    def test_cleanup_stale_conductors_disabled(self):
        """Test that cleanup is disabled when configured to do so."""
        self._start_service()

        conductors = []
        for i in range(5):
            self.dbapi.register_conductor({
                'hostname': 'conductor-%d' % i,
                'drivers': ['fake'],
                'conductor_group': 'group%d' % i
            })
            self.dbapi.touch_conductor('conductor-%d' % i, online=False)
            conductors.append('conductor-%d' % i)

        self.config(conductor_cleanup_batch_size=2, group='conductor')

        with mock.patch.object(self.dbapi, 'get_offline_conductors',
                               return_value=conductors, autospec=True):
            with mock.patch.object(self.dbapi, 'delete_conductor',
                                   autospec=True) as mock_delete:
                self.service._cleanup_stale_conductors(self.context)

                self.assertEqual(2, mock_delete.call_count)

    def _run_cleanup_stale_conductors_and_check_warning(
            self, heartbeat_timeout, conductor_cleanup_timeout,
            expected_warning_substring=None):
        self._start_service()
        self.config(heartbeat_timeout=heartbeat_timeout, group='conductor')
        self.config(conductor_cleanup_timeout=conductor_cleanup_timeout,
                    group='conductor')
        self.config(conductor_cleanup_batch_size=2, group='conductor')
        with mock.patch.object(self.dbapi, 'get_offline_conductors',
                               return_value=[], autospec=True):
            with mock.patch('ironic.conductor.manager.LOG',
                            autospec=True) as mock_log:
                self.service.cleanup_stale_conductors(self.context)
                if expected_warning_substring is None:
                    mock_log.warning.assert_not_called()
                else:
                    mock_log.warning.assert_called_once()
                    self.assertIn(expected_warning_substring,
                                  mock_log.warning.call_args[0][0])

    def test_cleanup_stale_conductors_configs_validation(self):
        test_cases = [
            (60, 180, None),  # valid: exactly 3x
            (60, 200, None),  # valid: greater than 3x
            (0, 180, 'heartbeat_timeout is invalid'),
            (60, 120, 'conductor_cleanup_timeout (120) must be at least 3x'),
        ]
        for heartbeat_timeout, cleanup_timeout, expected_warning in test_cases:
            self._run_cleanup_stale_conductors_and_check_warning(
                heartbeat_timeout=heartbeat_timeout,
                conductor_cleanup_timeout=cleanup_timeout,
                expected_warning_substring=expected_warning
            )
