From f0386264a363c82078510a11ca680e64247466e3 Mon Sep 17 00:00:00 2001 From: Steve Date: Thu, 13 Oct 2016 14:49:06 -0500 Subject: [PATCH] Wifi setup usability (#370) * Enhancing the Wifi setup process: * When no intenet is detected, Mycroft now instructs user to connect cable or tells how to start wifi setup * Wifi setup talks user through connection process * Setup will auto-shutdown after 5 minutes * Completion screen now goes to cerberus.mycroft.ai to allow registration immediately (TODO: custom url/landing page to that will already have the pairing code) * Changing the wording of the prompt message and slowing down the rate at which the password is spoken. * fixed pep8 error * fixed mroe pep8 in listener.py * -webkit-user-select: text; * * Fixed iOS issues with password input * Added SSID wrapping for unusually long network names * Fixed need for left/right scrolling on small phone screens (e.g. iPhone 4/5) * iOS devices now automatically open the screen after connecting to the MYCROFT network * Tweaked Wifi setup instruction wording to make it easier to understand * Fixed a potential bug with the flag used to stop the connection monitor * Removed some temporary debugging code being used to track a very specific issue with odd network names. * Various small changes for the code review. Mainly deleting some dead code and comment changes. * Made the "not connected" message more Mimic-friendly by using the word 'browse' instead of 'login'. * Increased the size of the password characters. They felt tiny at 13px on previously on an iPad. * - Added an auto-check for unit connection during the first 20 seconds when a unit is turned on. If no connection to the internet is found, the user is told how to get connected. - Added calls to 'ping' to help rebuild the ARP table we previously cleared in the test for lost connection - Tweaked some terminology spoken to be more Mimic-friendly and understandable. - Added automatic announcements every 45 seconds of the next step in the process. - Added automatic shutdown after 5 minutes - Added announcement when the process is complete - Added documentation and comments throughout - Made things more "pythonic". Switched functions from camelCase to python_style_names. Also used some underscore prefixes for private methods that are really just implementation helpers. * - The initial check for internet connectivity now happens 5 seconds after the system comes up instead of 20. - Also added a space to tweak the pronunciation of an announcement * Changes to the wifi setup portal - Added a Cancel Setup button (shuts down the process) - The Register Me button only appears once the browser can reach cerberus.mycroft.ai. This prevents following the link while phone is still connected to the temporary access point or not connected back to the real network. * Update version.txt --- mycroft/client/enclosure/version.txt | 2 +- mycroft/client/speech/listener.py | 8 +- mycroft/client/wifisetup/main.py | 287 +++++++++++++++++++-- mycroft/client/wifisetup/web/css/style.css | 39 ++- mycroft/client/wifisetup/web/index.html | 14 +- mycroft/client/wifisetup/web/js/WS.js | 6 + mycroft/client/wifisetup/web/js/main.js | 126 ++++++--- 7 files changed, 421 insertions(+), 61 deletions(-) diff --git a/mycroft/client/enclosure/version.txt b/mycroft/client/enclosure/version.txt index 7ac4e5e38f1e..71d6a66eda4c 100644 --- a/mycroft/client/enclosure/version.txt +++ b/mycroft/client/enclosure/version.txt @@ -1 +1 @@ -0.1.13 +0.1.14 diff --git a/mycroft/client/speech/listener.py b/mycroft/client/speech/listener.py index 5c7ce291d137..c81bdf97aa33 100644 --- a/mycroft/client/speech/listener.py +++ b/mycroft/client/speech/listener.py @@ -154,7 +154,7 @@ def runnable(): logger.error("AccessDenied from Cerberus proxy.") self.__speak( "Your device is not registered yet. To start pairing, " - "login at cerberus dot mycroft dot A.I") + "browse to cerberus dot mycroft dot A.I") utterances.append("pair my device") except Exception as e: logger.error("Unexpected exception: {0}".format(e)) @@ -191,7 +191,11 @@ def transcribe(self, audio_segments): else: raise sr.UnknownValueError else: # TODO: Localization - self.__speak("This device is not connected to the Internet") + # TODO: Enclosure virtualization (might not have a button) + self.__speak("This device is not connected to the Internet." + "Either plug in a network cable or hold the button" + " on top for two seconds, then select wifi from the " + "menu") class RecognizerLoopState(object): diff --git a/mycroft/client/wifisetup/main.py b/mycroft/client/wifisetup/main.py index c6942767aa6a..8815caa867bf 100644 --- a/mycroft/client/wifisetup/main.py +++ b/mycroft/client/wifisetup/main.py @@ -14,16 +14,34 @@ # # You should have received a copy of the GNU General Public License # along with Mycroft Core. If not, see . + +""" +This module implements a mechanism that allows the wifi connection of +a Linux system to be selected by end users. This is achieved by: + * creating a websocket for communication between the pieces of this + mechanism + * temporarilly creating a virtual access point + * directing the end user to connect to that access point with another device + (phone or tablet or laptop) + * having them open a captive portal in that device's web browser + * selecting the desired wifi within that browser + * configuring this device based on that selection +""" + import sys +import traceback +import threading from SimpleHTTPServer import SimpleHTTPRequestHandler from SocketServer import TCPServer from shutil import copyfile from subprocess import Popen, PIPE from threading import Thread from time import sleep +import urlparse import os -from os.path import join, dirname, realpath +import time +from os.path import dirname, realpath from pyric import pyw from wifi import Cell @@ -31,15 +49,26 @@ from mycroft.configuration import ConfigurationManager from mycroft.messagebus.client.ws import WebsocketClient from mycroft.messagebus.message import Message -from mycroft.util import str2bool +from mycroft.util import str2bool, connected from mycroft.util.log import getLogger -__author__ = 'aatchison' +__author__ = 'aatchison and penrods' LOG = getLogger("WiFiClient") +SCRIPT_DIR = dirname(realpath(__file__)) + + +def cli_no_output(*args): + ''' Invoke a command line and return result ''' + LOG.info("Command: %s" % list(args)) + proc = Popen(args=args, stdout=PIPE, stderr=PIPE) + stdout, stderr = proc.communicate() + return {'code': proc.returncode, 'stdout': stdout, 'stderr': stderr} + def cli(*args): + ''' Invoke a command line, then log and return result ''' LOG.info("Command: %s" % list(args)) proc = Popen(args=args, stdout=PIPE, stderr=PIPE) stdout, stderr = proc.communicate() @@ -61,17 +90,62 @@ def sysctrl(*args): return cli('systemctl', *args) -class WebServer(Thread): - DIR = dirname(realpath(__file__)) +class CaptiveHTTPRequestHandler(SimpleHTTPRequestHandler): + ''' Serve a single website, 303 redirecting all other requests to it ''' + def do_HEAD(self): + LOG.info("do_HEAD being called....") + if not self.redirect(): + SimpleHTTPRequestHandler.do_HEAD(self) + + def do_GET(self): + LOG.info("do_GET being called....") + if not self.redirect(): + SimpleHTTPRequestHandler.do_GET(self) + def redirect(self): + try: + LOG.info("***********************") + LOG.info("** HTTP Request ***") + LOG.info("***********************") + LOG.info("Requesting: "+self.path) + LOG.info("REMOTE_ADDR:"+self.client_address[0]) + LOG.info("SERVER_NAME:"+self.server.server_address[0]) + LOG.info("SERVER_PORT:"+str(self.server.server_address[1])) + LOG.info("SERVER_PROTOCOL:"+self.request_version) + LOG.info("HEADERS...") + LOG.info(self.headers) + LOG.info("***********************") + + # path = self.translate_path(self.path) + if "mycroft.ai" in self.headers['host']: + LOG.info("No redirect") + return False + else: + LOG.info("303 redirect to http://start.mycroft.ai") + self.send_response(303) + self.send_header("Location", "http://start.mycroft.ai") + self.end_headers() + return True + except: + tb = traceback.format_exc() + LOG.info("exception caught") + LOG.info(tb) + return False + + +class WebServer(Thread): + ''' Web server for devices connected to the temporary access point ''' def __init__(self, host, port): super(WebServer, self).__init__() self.daemon = True - self.server = TCPServer((host, port), SimpleHTTPRequestHandler) + LOG.info("Creating TCPServer...") + self.server = TCPServer((host, port), CaptiveHTTPRequestHandler) + LOG.info("Created TCPServer") def run(self): LOG.info("Starting Web Server at %s:%s" % self.server.server_address) - os.chdir(join(self.DIR, 'web')) + LOG.info("Serving from: %s" % os.path.join(SCRIPT_DIR, 'web')) + os.chdir(os.path.join(SCRIPT_DIR, 'web')) self.server.serve_forever() LOG.info("Web Server stopped!") @@ -89,9 +163,10 @@ class AccessPoint: def __init__(self, wiface): self.wiface = wiface self.iface = 'p2p-wlan0-0' - self.ip = '172.24.1.1' - self.ip_start = '172.24.1.50' - self.ip_end = '172.24.1.150' + self.subnet = '172.24.1' + self.ip = self.subnet+'.1' + self.ip_start = self.subnet+'.50' + self.ip_end = self.subnet+'.150' self.password = None def up(self): @@ -135,38 +210,173 @@ def save(self): class WiFi: - NAME = "WiFiClient" - def __init__(self): self.iface = pyw.winterfaces()[0] self.ap = AccessPoint(self.iface) self.server = None self.client = WebsocketClient() self.enclosure = EnclosureAPI(self.client) - self.config = ConfigurationManager.get().get(self.NAME) self.init_events() - self.first_setup() + self.conn_monitor = None + self.conn_monitor_stop = threading.Event() def init_events(self): + ''' + Register handlers for various websocket events used + to communicate with outside systems. + ''' + + # This event is generated by an outside mechanism. On a + # Holmes unit this comes from the Enclosure's menu item + # being selected. self.client.on('mycroft.wifi.start', self.start) + + # These events are generated by Javascript in the captive + # portal. self.client.on('mycroft.wifi.stop', self.stop) self.client.on('mycroft.wifi.scan', self.scan) self.client.on('mycroft.wifi.connect', self.connect) - def first_setup(self): - if str2bool(self.config.get('setup')): - self.start() - def start(self, event=None): + ''' + Fire up the MYCROFT access point for the user to connect to + with a phone or computer. + ''' LOG.info("Starting access point...") - self.client.emit(Message("speak", metadata={ - 'utterance': "Initializing wireless setup mode."})) + + # Fire up our access point self.ap.up() if not self.server: + LOG.info("Creating web server...") self.server = WebServer(self.ap.ip, 80) + LOG.info("Starting web server...") self.server.start() - self.enclosure.mouth_text(self.ap.password) + LOG.info("Created web server.") + LOG.info("Access point started!\n%s" % self.ap.__dict__) + self._start_connection_monitor() + + def _connection_prompt(self, prefix): + # let the user know to connect to it... + passwordSpelled = ", ".join(self.ap.password) + self._speak_and_show( + prefix+" Use your mobile device or computer to " + "connect to the wifi network " + "'MYCROFT'; Then enter the uppercase " + "password "+passwordSpelled, + self.ap.password) + + def _speak_and_show(self, speak, show): + ''' Communicate with the user throughout the process ''' + self.client.emit(Message("speak", metadata={'utterance': speak})) + if show is None: + return + + # TODO: This sleep should not be necessary, but without it the + # text to be displayed by enclosure.mouth_text() gets + # wiped out immediately when the utterance above is + # begins processing. + # Remove the sleep once this behavior is corrected. + sleep(0.25) + self.enclosure.mouth_text(show) + + def _start_connection_monitor(self): + LOG.info("Starting monitor thread...\n") + if self.conn_monitor is not None: + LOG.info("Killing old thread...\n") + self.conn_monitor_stop.set() + self.conn_monitor_stop.wait() + + self.conn_monitor = threading.Thread( + target=self._do_connection_monitor, + args={}) + self.conn_monitor.daemon = True + self.conn_monitor.start() + LOG.info("Monitor thread setup complete.\n") + + def _stop_connection_monitor(self): + ''' Set flag that will let monitoring thread close ''' + self.conn_monitor_stop.set() + + def _do_connection_monitor(self): + LOG.info("Invoked monitor thread...\n") + mtimeLast = os.path.getmtime('/var/lib/misc/dnsmasq.leases') + bHasConnected = False + cARPFailures = 0 + timeStarted = time.time() + timeLastAnnounced = 0 # force first announcement to now + self.conn_monitor_stop.clear() + + while not self.conn_monitor_stop.isSet(): + # do our monitoring... + mtime = os.path.getmtime('/var/lib/misc/dnsmasq.leases') + if mtimeLast != mtime: + # Something changed in the dnsmasq lease file - + # presumably a (re)new lease + bHasConnected = True + cARPFailures = 0 + mtimeLast = mtime + timeStarted = time.time() # reset start time after connection + timeLastAnnounced = time.time()-45 # announce how to connect + + if time.time()-timeStarted > 60*5: + # After 5 minutes, shut down the access point + LOG.info("Auto-shutdown of access point after 5 minutes") + self.stop() + continue + + if time.time()-timeLastAnnounced >= 45: + if bHasConnected: + self._speak_and_show( + "Now you can open your browser and go to start dot " + "mycroft dot A I, then follow the instructions given " + " there", + "start.mycroft.ai") + else: + self._connection_prompt("Allow me to walk you through the " + " wifi setup process; ") + timeLastAnnounced = time.time() + + if bHasConnected: + # Flush the ARP entries associated with our access point + # This will require all network hardware to re-register + # with the ARP tables if still present. + if cARPFailures == 0: + res = cli_no_output('ip', '-s', '-s', 'neigh', 'flush', + self.ap.subnet+'.0/24') + # Give ARP system time to re-register hardware + sleep(5) + + # now look at the hardware that has responded, if no entry + # shows up on our access point after 2*5=10 seconds, the user + # has disconnected + if not self._is_ARP_filled(): + cARPFailures += 1 + if cARPFailures > 2: + self._connection_prompt("Connection lost,") + bHasConnected = False + else: + cARPFailures = 0 + sleep(5) # wait a bit to prevent thread from hogging CPU + + LOG.info("Exiting monitor thread...\n") + self.conn_monitor_stop.clear() + + def _is_ARP_filled(self): + res = cli_no_output('/usr/sbin/arp', '-n') + out = str(res.get("stdout")) + if out: + # Parse output, skipping header + for o in out.split("\n")[1:]: + if o[0:len(self.ap.subnet)] == self.ap.subnet: + if "(incomplete)" in o: + # ping the IP to get the ARP table entry reloaded + ip_disconnected = o.split(" ")[0] + cli_no_output('/bin/ping', '-c', '1', '-W', 3, + ip_disconnected) + else: + return True # something on subnet is connected! + return False def scan(self, event=None): LOG.info("Scanning wifi connections...") @@ -178,6 +388,8 @@ def scan(self, event=None): ssid = cell.ssid quality = self.get_quality(cell.quality) + # If there are duplicate network IDs (e.g. repeaters) only + # report the strongest signal if networks.__contains__(ssid): update = networks.get(ssid).get("quality") < quality if update and ssid: @@ -218,12 +430,17 @@ def connect(self, event=None): connected = self.get_connected(ssid) if connected: wpa(self.iface, 'save_config') - # ConfigurationManager.set(self.NAME, 'setup', False, True) self.client.emit(Message("mycroft.wifi.connected", {'connected': connected})) LOG.info("Connection status for %s = %s" % (ssid, connected)) + if connected: + self.client.emit(Message("speak", metadata={ + 'utterance': "Thank you, I'm now connected to the " + "internet and ready for use"})) + # TODO: emit something that triggers a pairing check + def disconnect(self): status = self.get_status() nid = status.get("id") @@ -254,6 +471,7 @@ def is_connected(self, ssid, status=None): def stop(self, event=None): LOG.info("Stopping access point...") + self._stop_connection_monitor() self.ap.down() if self.server: self.server.server.shutdown() @@ -262,8 +480,33 @@ def stop(self, event=None): self.server = None LOG.info("Access point stopped!") + def _do_net_check(self): + # give system 5 seconds to resolve network or get plugged in + sleep(5) + + LOG.info("Checking internet connection again") + if not connected() and self.conn_monitor is None: + # TODO: Enclosure/localization + self._speak_and_show( + "This device is not connected to the Internet. Either plug " + "in a network cable or hold the button on top for two " + "seconds, then select wifi from the menu", None) + def run(self): try: + # When the system first boots up, check for a valid internet + # connection. + LOG.info("Checking internet connection") + if not connected(): + LOG.info("No connection initially, waiting 20...") + self.net_check = threading.Thread( + target=self._do_net_check, + args={}) + self.net_check.daemon = True + self.net_check.start() + else: + LOG.info("Connection found!") + self.client.run_forever() except Exception as e: LOG.error("Error: {0}".format(e)) diff --git a/mycroft/client/wifisetup/web/css/style.css b/mycroft/client/wifisetup/web/css/style.css index f92511553251..9e5e1c0d4712 100644 --- a/mycroft/client/wifisetup/web/css/style.css +++ b/mycroft/client/wifisetup/web/css/style.css @@ -2,15 +2,32 @@ font-family: sans-serif; user-select: none; -moz-user-select: none; - -webkit-user-select: none; + -webkit-user-select: text; -ms-user-select: none; } +/* Prevent ridiculously long network names from screwing up the layout + by forcing them to wrap. */ +.ssid { + max-width: 280px; +} + +/* A bug in Safari (as of 10-10-2016) breaks password input when + user-select:none is used. This prevents that bug by allowing + text selection for all input fields (which is probably expected + by users anyway). */ +input { + user-select: text; + -moz-user-select: text; + -webkit-user-select: text; + -ms-user-select: text; +} + .panel { border-radius: 15px; box-shadow: 0 0 10px #AAA; max-width: 500px; - min-width: 320px; + min-width: 300px; } .panel .title { @@ -104,6 +121,7 @@ height: 26px; border: 1px solid #DDD; width: calc(100% - 160px); + font-size: 16px; } .panel .body li .error-item span { @@ -223,6 +241,23 @@ span.connected { text-decoration: none; } +#centered { + margin-right:auto; + margin-left:auto; +} +#footer { + position:fixed; + bottom:10px; + width:100%; +} +#cancelBtn { + background: white; + z-index: 999; + color: #ff6666; + padding: 7px; /* 10px of .button -3px to account for the thick border */ + border: 3px solid #ff6666; +} + .alert { width: 100%; text-align: center; diff --git a/mycroft/client/wifisetup/web/index.html b/mycroft/client/wifisetup/web/index.html index ba5b065096f9..5f84a92f027d 100644 --- a/mycroft/client/wifisetup/web/index.html +++ b/mycroft/client/wifisetup/web/index.html @@ -26,14 +26,14 @@
-
Loading...
+
LOADING
-
Connecting...
+
CONNECTING
@@ -43,10 +43,14 @@
CONNECTED
I'm connected to the internet now. Feels goood - Final step, register me so you can add new skills to my repertoire. - REGISTER ME + If I am registered with your account, I'm ready to roll. If not, go ahead and register me at cerberus.mycroft.ai. + REGISTER ME
+

+ CANCEL SETUP +

+ - \ No newline at end of file + diff --git a/mycroft/client/wifisetup/web/js/WS.js b/mycroft/client/wifisetup/web/js/WS.js index 7e005f86af79..29048ca94942 100644 --- a/mycroft/client/wifisetup/web/js/WS.js +++ b/mycroft/client/wifisetup/web/js/WS.js @@ -41,6 +41,12 @@ var WS = { })); }, + close: function () { + this.ws.close(); + this.wsConnected = false; + this.ws = null; + }, + addMessageListener: function (type, callback) { this.listeners[type] = this.listeners[type] || []; this.listeners[type].push(callback); diff --git a/mycroft/client/wifisetup/web/js/main.js b/mycroft/client/wifisetup/web/js/main.js index 9cf9fe2eaf04..8f6ba015b9df 100644 --- a/mycroft/client/wifisetup/web/js/main.js +++ b/mycroft/client/wifisetup/web/js/main.js @@ -14,9 +14,10 @@ function getImagePath(strength) { function showPanel(id) { var panels = document.querySelectorAll(".panel"); - Object.keys(panels).forEach(function (panel) { - panels[panel].classList.add("hide"); - }); + + for (var i=0; i < panels.length; i++) + panels[i].classList.add("hide"); + document.querySelector("#" + id).classList.remove("hide"); } @@ -31,9 +32,25 @@ var WifiSetup = { onConnected: function (data) { if (data.connected) { + // NOTE: Once we send the "mycroft.wifi.stop", the unit will + // be shutting down the wifi access point. So the device + // hosting the browser session is probably being disconnected + // and hopefully automatically reconnecting to the internet. + // + // Until the reconnect happens, the user cannot actually + // follow the link to http://cerberus.mycroft.ai to register + // their device. That is part of why we are doing this 2 sec + // delay. + // WS.send("mycroft.wifi.stop"); + WS.close(); + setTimeout(function () { + var btnCancel = document.querySelector("#cancelBtn"); + btnCancel.classList.add("hide"); + showPanel("success"); + startPing(); }, 2000); } else { showPanel("list-panel"); @@ -74,6 +91,7 @@ var WifiSetup = { imgSignal = document.createElement("img"); listItem.className = "list-item show"; span.textContent = network.ssid; + span.className = "ssid"; imgSignal.src = getImagePath(network.quality); imgSignal.className = "wifi"; listItem.appendChild(span); @@ -113,31 +131,31 @@ var WifiSetup = { renderConnectItem: function (li) { var connect = li.querySelector(".connect-item"); - if (connect) { - li.querySelector(".list-item").classList.remove("show"); - connect.classList.add("show"); - return; - } - connect = document.createElement("div"); - var imgArrow = document.createElement("img"); - var label = document.createElement("label"); - connect.className = "connect-item"; - imgArrow.src = "img/next.png"; - imgArrow.addEventListener("click", this.clickConnect.bind(this)); - connect.appendChild(label); - if (this.selectedNetword.encrypted) { - passwordInput = document.createElement("input"); - label.textContent = "Password: "; - passwordInput.type = "password"; - connect.appendChild(passwordInput); - } else { - label.className = "public"; - label.textContent = this.selectedNetword.ssid; - } - connect.appendChild(imgArrow); - li.appendChild(connect); + if (!connect) { + connect = document.createElement("div"); + var imgArrow = document.createElement("img"); + var label = document.createElement("label"); + connect.className = "connect-item"; + imgArrow.src = "img/next.png"; + imgArrow.addEventListener("click", this.clickConnect.bind(this)); + connect.appendChild(label); + if (this.selectedNetword.encrypted) { + connect.passwordInput = document.createElement("input"); + label.textContent = "Password: "; + connect.passwordInput.type = "password"; + connect.appendChild(connect.passwordInput); + } else { + label.className = "public"; + label.textContent = this.selectedNetword.ssid; + } + connect.appendChild(imgArrow); + + li.appendChild(connect); + } li.querySelector(".list-item").classList.remove("show"); connect.classList.add("show"); + if ('passwordInput' in connect) + connect.passwordInput.focus(); }, renderErrorItem: function (li) { @@ -170,6 +188,7 @@ var WifiSetup = { sendScan: function () { showPanel("loading"); + document.querySelector("#cancelBtn").classList.remove("hide"); WS.send("mycroft.wifi.scan"); } , @@ -195,19 +214,68 @@ var WifiSetup = { } , + cancelSetup: function () { + WS.send("mycroft.wifi.stop"); + WS.close(); + } + , + init: function () { this.setListeners(); showPanel("home"); document.querySelector("#connectBtn").addEventListener("click", this.sendScan); document.querySelector("#registerBtn").addEventListener("click", function () { setTimeout(function() { - location.href="https://mycroft.ai"; + location.href="https://cerberus.mycroft.ai"; }, 2000); }); - + document.querySelector("#cancelBtn").addEventListener("click", this.cancelSetup); } + }; + +function startPing() { + ping("cerberus.mycroft.ai", + function(status,e) { + if (status == 'responded') { + // Un-hide the register button once we detect an + // active internet connection. + document.querySelector("#registerBtn").classList.remove("hide"); + } + else + setTimeout(function() { startPing(); }, 1000); + }); +} + +function ping(domain, callback) { + if (!this.inUse) { + this.status = 'unchecked'; + this.inUse = true; + this.callback = callback; + this.ip = domain; + var _that = this; + this.img = new Image(); + this.img.onload = function () { + _that.inUse = false; + _that.callback('responded'); + + }; + this.img.onerror = function (e) { + if (_that.inUse) { + _that.inUse = false; + _that.callback('responded', e); + } + + }; + this.start = new Date().getTime(); + this.img.src = "http://" + domain; + this.timer = setTimeout(function () { + if (_that.inUse) { + _that.inUse = false; + _that.callback('timeout'); + } + }, 1500); } - ; +} window.addEventListener("load", function () { WS.connect();