conman:  React to mwifiex firmware resets.

When the mwifiex firmware resets, its interfaces are recreated (and
hotplug touches a file in /tmp/interfaces).  This causes
wpa_supplicant to get disconnected.  Simply calling "wifi setclient"
again is a no-op, because /bin/wifi compares the new wpa_supplicant
config to the existing one, and exits if there is no change (and
wpa_supplicant is running).  There is a --force-restart /bin/wifi
option that overrides this behavior.

In practice, after an mwifiex reset, we sometimes see conman try to
re-run "wifi setclient" only for /bin/wifi to treat it as a no-op.
This causes conman to attempt to reprovision, which delays rejoining
the WLAN.

This commit watches the files touched by hotplug with inotify, and
when they are touched, adds --force-restart to the next wifi setclient
call.

Bug: 34040473

Change-Id: I0f6e0c7606e1b2569ac75f6cf53adf15b37653d4
diff --git a/conman/connection_manager.py b/conman/connection_manager.py
index c9a4db1..39db198 100755
--- a/conman/connection_manager.py
+++ b/conman/connection_manager.py
@@ -41,6 +41,7 @@
 HOSTNAME = socket.gethostname()
 TMP_HOSTS = '/tmp/hosts'
 CWMP_PATH = '/tmp/cwmp'
+INTERFACE_PATH = '/tmp/interface'
 
 experiment.register('WifiNo2GClient')
 
@@ -186,6 +187,9 @@
     self.wifi.set_gateway_ip(None)
     self.wifi.set_subnet(None)
     command = self.WIFI_SETCLIENT + ['--ssid', self.ssid, '--band', self.band]
+    if self.wifi.recently_reset:
+      command += ['--force-restart']
+      self.wifi.recently_reset = False
     env = dict(os.environ)
     if self.passphrase:
       env['WIFI_CLIENT_PSK'] = self.passphrase
@@ -303,6 +307,8 @@
                  pyinotify.IN_CLOSE_WRITE | pyinotify.IN_MOVED_TO)
     wm.add_watch(self._moca_tmp_dir,
                  pyinotify.IN_CLOSE_WRITE | pyinotify.IN_MOVED_TO)
+    wm.add_watch(INTERFACE_PATH,
+                 pyinotify.IN_CLOSE_WRITE | pyinotify.IN_MOVED_TO)
     self.notifier = pyinotify.Notifier(wm, FileChangeHandler(self), timeout=0)
 
     # If the ethernet file doesn't exist for any reason when conman starts,
@@ -772,6 +778,10 @@
         if had_moca != has_moca:
           self.ifplugd_action('moca0', has_moca)
 
+    elif path == INTERFACE_PATH:
+      ifc = self.interface_by_name(filename)
+      ifc.recently_reset = True
+
   def interface_by_name(self, interface_name):
     for ifc in self.interfaces():
       if ifc.name == interface_name:
diff --git a/conman/connection_manager_test.py b/conman/connection_manager_test.py
index bec59f7..17d7a92 100755
--- a/conman/connection_manager_test.py
+++ b/conman/connection_manager_test.py
@@ -260,8 +260,10 @@
         wpa_control_interface = tempfile.mkdtemp()
         subprocess.mock('wifi', 'wpa_path', wpa_control_interface)
         connection_manager.CWMP_PATH = tempfile.mkdtemp()
+        connection_manager.INTERFACE_PATH = tempfile.mkdtemp()
         subprocess.set_conman_paths(tmp_dir, config_dir,
-                                    connection_manager.CWMP_PATH)
+                                    connection_manager.CWMP_PATH,
+                                    connection_manager.INTERFACE_PATH)
 
         for band, access_point in wlan_configs.iteritems():
           subprocess.mock('cwmp', band, ssid='initial ssid', psk='initial psk',
@@ -1119,5 +1121,24 @@
   wvtest.WVPASSEQ(1, count_setclient_calls())
 
 
+@test_common.wvtest
+@connection_manager_test(WIFI_SHOW_OUTPUT_MARVELL8897)
+def test_regression_b34040473(c):
+  ssid = 'my ssid'
+  psk = 'my passphrase'
+  band = '5'
+
+  subprocess.mock('cwmp', band, ssid=ssid, psk=psk, write_now=True)
+  subprocess.mock('wifi', 'remote_ap', band=band, ssid=ssid, psk=psk,
+                  bssid='00:00:00:00:00:00')
+  c.run_once()
+  wvtest.WVPASS(c.client_up(band))
+
+  subprocess.mock('wifi', 'mwifiex_reset', c.wifi_for_band(band).name, band)
+  wvtest.WVFAIL(c.client_up(band))
+  c.run_once()
+  wvtest.WVPASS(c.client_up(band))
+
+
 if __name__ == '__main__':
   wvtest.wvtest_main()
diff --git a/conman/interface.py b/conman/interface.py
index fbef47e..7639520 100755
--- a/conman/interface.py
+++ b/conman/interface.py
@@ -427,6 +427,10 @@
   def __init__(self, *args, **kwargs):
     self.bands = kwargs.pop('bands', [])
     super(Wifi, self).__init__(*args, **kwargs)
+    # When the interface is reset (e.g. due to a firmware crash), we need to
+    # force-restart wpa_supplicant next time we run "wifi setclient", even if
+    # the config hasn't changed.
+    self.recently_reset = False
 
   @property
   def wpa_supplicant(self):
diff --git a/conman/test/fake_python/subprocess/__init__.py b/conman/test/fake_python/subprocess/__init__.py
index ff8ad0a..f0a8e7a 100644
--- a/conman/test/fake_python/subprocess/__init__.py
+++ b/conman/test/fake_python/subprocess/__init__.py
@@ -28,6 +28,7 @@
     'upload_logs_and_wait': 'upload-logs-and-wait',
     'wifi': None,
     'wpa_cli': None,
+    'hotplug': None,
 }
 _COMMANDS = {v or k: importlib.import_module('.' + k, __name__)
              for k, v in _COMMAND_NAMES.iteritems()}
@@ -116,7 +117,8 @@
       reload(command)
 
 
-def set_conman_paths(tmp_path=None, config_path=None, cwmp_path=None):
+def set_conman_paths(tmp_path=None, config_path=None, cwmp_path=None,
+                     interface_path=None):
   for command in ('run-dhclient', '/etc/ifplugd/ifplugd.action'):
     _COMMANDS[command].CONMAN_PATH = tmp_path
 
@@ -126,6 +128,9 @@
   for command in ('cwmp',):
     _COMMANDS[command].CWMP_PATH = cwmp_path
 
+  for command in ('hotplug',):
+    _COMMANDS[command].INTERFACE_PATH = interface_path
+
   # Make sure <tmp_path>/interfaces exists.
   tmp_interfaces_path = os.path.join(tmp_path, 'interfaces')
   if not os.path.exists(tmp_interfaces_path):
diff --git a/conman/test/fake_python/subprocess/hotplug.py b/conman/test/fake_python/subprocess/hotplug.py
new file mode 100644
index 0000000..99342e6
--- /dev/null
+++ b/conman/test/fake_python/subprocess/hotplug.py
@@ -0,0 +1,16 @@
+"""Fake /sbin/hotplug implemenation."""
+
+import logging
+import os
+
+logger = logging.getLogger('subprocess.hotplug')
+
+
+INTERFACE_PATH = None
+
+
+def call(unused_command, env):
+  if env['SUBSYSTEM'] == 'net' and env['ACTION'] == 'add':
+    interface = env['INTERFACE']
+    logger.debug('Simulating creation of %s', interface)
+    open(os.path.join(INTERFACE_PATH, interface), 'w')
diff --git a/conman/test/fake_python/subprocess/wifi.py b/conman/test/fake_python/subprocess/wifi.py
index 900b908..3f5cdc9 100644
--- a/conman/test/fake_python/subprocess/wifi.py
+++ b/conman/test/fake_python/subprocess/wifi.py
@@ -8,6 +8,7 @@
 
 import connection_check
 import get_quantenna_interfaces
+import hotplug
 import ifplugd_action
 import ifup
 import qcsapi
@@ -71,6 +72,7 @@
                    'vendor_ies', 'connection_check_result', 'hidden')
     for attr in self._attrs:
       setattr(self, attr, kwargs.get(attr, None))
+    self.stuck = False
 
   def scan_str(self):
     security_strs = {
@@ -136,7 +138,8 @@
 
   band = _get_flag(args, ('-b', '--band'))
   bssid = _get_flag(args, ('--bssid',))
-  ssid = _get_flag(args, ('S', '--ssid',))
+  ssid = _get_flag(args, ('-S', '--ssid',))
+  force_restart = '--force-restart' in args
 
   if band not in INTERFACE_FOR_BAND:
     raise ValueError('No interface for band %r' % band)
@@ -166,7 +169,7 @@
     return 1, 'Wrong PSK, got %r, expected %r' % (psk, ap.psk)
 
   _setclient_success(interface_name, ssid, bssid, psk, interface.driver, ap,
-                     band)
+                     band, force_restart)
 
   return 0, ''
 
@@ -197,10 +200,22 @@
   CLIENT_ASSOCIATIONS[interface_name] = None
 
 
-def _setclient_success(interface_name, ssid, bssid, psk, driver, ap, band):
-  if CLIENT_ASSOCIATIONS.get(interface_name, None):
+def _setclient_success(interface_name, ssid, bssid, psk, driver, ap, band,
+                       force_restart):
+  same_ap = False
+  current_association = CLIENT_ASSOCIATIONS.get(interface_name, None)
+  if current_association:
+    same_ap = current_association == ap
+  else:
     _disconnected_event(band)
   if driver == 'cfg80211':
+    if same_ap and not force_restart:
+      return
+
+    # Make sure the association is unstuck after a firmware reset.
+    if current_association:
+      current_association.stuck = False
+
     # Make sure the wpa_supplicant socket exists.
     open(os.path.join(WPA_PATH, interface_name), 'w')
 
@@ -264,8 +279,8 @@
 
     CLIENT_ASSOCIATIONS[interface_name] = None
 
-  # Call ifplugd.action for the interface going down (wifi/quantenna.py does this
-  # manually).
+  # Call ifplugd.action for the interface going down (wifi/quantenna.py does
+  # this manually).
   ifplugd_action.call(interface_name, 'down')
 
   return 0, ''
@@ -296,7 +311,8 @@
 
 
 def _show(unused_args, **unused_kwargs):
-  return 0, '\n\n'.join(WIFI_SHOW_TPL.format(band=band, **interface._asdict()) if interface
+  return 0, '\n\n'.join(WIFI_SHOW_TPL.format(band=band, **interface._asdict())
+                        if interface
                         else WIFI_SHOW_NO_RADIO_TPL.format(band)
                         for band, interface in INTERFACE_FOR_BAND.iteritems())
 
@@ -329,3 +345,11 @@
     _disconnected_event(args[0])
   elif command == 'kill_wpa_supplicant':
     _kill_wpa_supplicant(args[0])
+  elif command == 'mwifiex_reset':
+    interface_name, band = args[0:2]
+    print CLIENT_ASSOCIATIONS
+    association = CLIENT_ASSOCIATIONS.get(interface_name, None)
+    association.stuck = True
+    wpa_cli.mock(interface_name, wpa_state='SCANNING')
+    hotplug.call([], env={'SUBSYSTEM': 'net', 'ACTION': 'add',
+                          'INTERFACE': args[0]})