[Huge] Util Refactor, Dashboard Improvements, Hass.io Auth API, Better Validation Errors, Conditions, Custom Platforms, Substitutions (#234)

* Implement custom sensor platform * Update * Ethernet * Lint * Fix * Login page * Rename cookie secret * Update manifest * Update cookie check logic * Favicon * Fix * Favicon manifest * Fix * Fix * Fix * Use hostname * Message * Temporary commit for screenshot * Automatic board selection * Undo temporary commit * Update esphomeyaml-edge * In-dashboard editing and hosting files locally * Update esphomeyaml-edge * Better ANSI color escaping * Message * Lint * Download Efficiency * Fix gitlab * Fix * Rename extra_libraries to libraries * Add example * Update README.md * Update README.md * Update README.md * HassIO -> Hass.io * Updates * Add update available notice * Update * Fix substitutions * Better error message * Re-do dashboard ANSI colors * Only include FastLED if user says so * Autoscroll logs * Remove old checks * Use safer RedirectText * Improvements * Fix * Use enviornment variable * Use http://hassio/host/info * Fix conditions * Update platformio versions * Revert "Use enviornment variable" This reverts commit 7f038eb5d26df72f76ea9ae76774e2cec1fd7f59. * Fix * README update * Temp * Better invalid config messages * Platformio debug * Improve error messages * Debug * Remove debug * Multi Conf * Update * Better paths * Remove unused * Fixes * Lint * lib_ignore * Try fix platformio colors * Fix dashboard scrolling * Revert * Lint * Revert
2026-06-13 16:13:32 +02:00 · 2018-12-05 21:22:06 +01:00
parent 2b88c987da
commit 7c7032c59e
192 changed files with 6156 additions and 2845 deletions
@@ -2,41 +2,52 @@
 from __future__ import print_function

 import codecs
+import collections
 import hmac
 import json
 import logging
+import multiprocessing
 import os
-import random
 import subprocess
+import threading

-from esphomeyaml.const import CONF_ESPHOMEYAML, CONF_BUILD_PATH
-from esphomeyaml.core import ESPHomeYAMLError
-from esphomeyaml import const, core, __main__
+import tornado
+import tornado.concurrent
+import tornado.gen
+import tornado.ioloop
+import tornado.iostream
+from tornado.log import access_log
+import tornado.process
+import tornado.web
+import tornado.websocket
+
+from esphomeyaml import const
 from esphomeyaml.__main__ import get_serial_ports
-from esphomeyaml.helpers import relative_path
+from esphomeyaml.helpers import mkdir_p, run_system_command
+from esphomeyaml.storage_json import EsphomeyamlStorageJSON, StorageJSON, \
+    esphomeyaml_storage_path, ext_storage_path
 from esphomeyaml.util import shlex_quote

-try:
-    import tornado
-    import tornado.gen
-    import tornado.ioloop
-    import tornado.iostream
-    import tornado.process
-    import tornado.web
-    import tornado.websocket
-    import tornado.concurrent
-except ImportError as err:
-    tornado = None
+# pylint: disable=unused-import, wrong-import-order
+from typing import Optional  # noqa

 _LOGGER = logging.getLogger(__name__)
 CONFIG_DIR = ''
-PASSWORD = ''
+PASSWORD_DIGEST = ''
+COOKIE_SECRET = None
+USING_PASSWORD = False
+ON_HASSIO = False
+USING_HASSIO_AUTH = True
+HASSIO_MQTT_CONFIG = None


 # pylint: disable=abstract-method
 class BaseHandler(tornado.web.RequestHandler):
    def is_authenticated(self):
-        return not PASSWORD or self.get_secure_cookie('authenticated') == 'yes'
+        if USING_HASSIO_AUTH or USING_PASSWORD:
+            return self.get_secure_cookie('authenticated') == 'yes'
+
+        return True


 # pylint: disable=abstract-method, arguments-differ
@@ -47,12 +58,13 @@ class EsphomeyamlCommandWebSocket(tornado.websocket.WebSocketHandler):
        self.closed = False

    def on_message(self, message):
-        if PASSWORD and self.get_secure_cookie('authenticated') != 'yes':
-            return
+        if USING_HASSIO_AUTH or USING_PASSWORD:
+            if self.get_secure_cookie('authenticated') != 'yes':
+                return
        if self.proc is not None:
            return
        command = self.build_command(message)
-        _LOGGER.debug(u"WebSocket opened for command %s", [shlex_quote(x) for x in command])
+        _LOGGER.info(u"Running command '%s'", ' '.join(shlex_quote(x) for x in command))
        self.proc = tornado.process.Subprocess(command,
                                               stdout=tornado.process.Subprocess.STREAM,
                                               stderr=subprocess.STDOUT)
@@ -66,13 +78,11 @@ class EsphomeyamlCommandWebSocket(tornado.websocket.WebSocketHandler):
                data = yield self.proc.stdout.read_until_regex('[\n\r]')
            except tornado.iostream.StreamClosedError:
                break
-            if data.endswith('\r') and random.randrange(100) < 90:
-                continue
            try:
-                data = data.replace('\033', '\\033')
+                self.write_message({'event': 'line', 'data': data})
            except UnicodeDecodeError:
-                data = data.encode('ascii', 'backslashreplace')
-            self.write_message({'event': 'line', 'data': data})
+                data = codecs.decode(data, 'utf8', 'replace')
+                self.write_message({'event': 'line', 'data': data})

    def proc_on_exit(self, returncode):
        if not self.closed:
@@ -93,15 +103,14 @@ class EsphomeyamlLogsHandler(EsphomeyamlCommandWebSocket):
    def build_command(self, message):
        js = json.loads(message)
        config_file = CONFIG_DIR + '/' + js['configuration']
-        return ["esphomeyaml", config_file, "logs", '--serial-port', js["port"], '--escape']
+        return ["esphomeyaml", config_file, "logs", '--serial-port', js["port"]]


 class EsphomeyamlRunHandler(EsphomeyamlCommandWebSocket):
    def build_command(self, message):
        js = json.loads(message)
        config_file = os.path.join(CONFIG_DIR, js['configuration'])
-        return ["esphomeyaml", config_file, "run", '--upload-port', js["port"],
-                '--escape', '--use-esptoolpy']
+        return ["esphomeyaml", config_file, "run", '--upload-port', js["port"]]


 class EsphomeyamlCompileHandler(EsphomeyamlCommandWebSocket):
@@ -166,11 +175,8 @@ class WizardRequestHandler(BaseHandler):
            self.redirect('/login')
            return
        kwargs = {k: ''.join(v) for k, v in self.request.arguments.iteritems()}
-        config = wizard.wizard_file(**kwargs)
        destination = os.path.join(CONFIG_DIR, kwargs['name'] + '.yaml')
-        with codecs.open(destination, 'w') as f_handle:
-            f_handle.write(config)
-
+        wizard.wizard_write(path=destination, **kwargs)
        self.redirect('/?begin=True')


@@ -181,13 +187,16 @@ class DownloadBinaryRequestHandler(BaseHandler):
            return

        configuration = self.get_argument('configuration')
-        config_file = os.path.join(CONFIG_DIR, configuration)
-        core.CONFIG_PATH = config_file
-        config = __main__.read_config(core.CONFIG_PATH)
-        build_path = relative_path(config[CONF_ESPHOMEYAML][CONF_BUILD_PATH])
-        path = os.path.join(build_path, '.pioenvs', core.NAME, 'firmware.bin')
+        storage_path = ext_storage_path(CONFIG_DIR, configuration)
+        storage_json = StorageJSON.load(storage_path)
+        if storage_json is None:
+            self.send_error()
+            return
+
+        path = storage_json.firmware_bin_path
        self.set_header('Content-Type', 'application/octet-stream')
-        self.set_header("Content-Disposition", 'attachment; filename="{}.bin"'.format(core.NAME))
+        filename = '{}.bin'.format(storage_json.name)
+        self.set_header("Content-Disposition", 'attachment; filename="{}"'.format(filename))
        with open(path, 'rb') as f:
            while 1:
                data = f.read(16384)  # or some other nice-sized chunk
@@ -197,6 +206,83 @@ class DownloadBinaryRequestHandler(BaseHandler):
        self.finish()


+def _list_yaml_files():
+    files = []
+    for file in os.listdir(CONFIG_DIR):
+        if not file.endswith('.yaml'):
+            continue
+        if file.startswith('.'):
+            continue
+        if file == 'secrets.yaml':
+            continue
+        files.append(file)
+    files.sort()
+    return files
+
+
+def _list_dashboard_entries():
+    files = _list_yaml_files()
+    return [DashboardEntry(file) for file in files]
+
+
+class DashboardEntry(object):
+    def __init__(self, filename):
+        self.filename = filename
+        self._storage = None
+        self._loaded_storage = False
+
+    @property
+    def full_path(self):  # type: () -> str
+        return os.path.join(CONFIG_DIR, self.filename)
+
+    @property
+    def storage(self):  # type: () -> Optional[StorageJSON]
+        if not self._loaded_storage:
+            self._storage = StorageJSON.load(ext_storage_path(CONFIG_DIR, self.filename))
+            self._loaded_storage = True
+        return self._storage
+
+    @property
+    def address(self):
+        if self.storage is None:
+            return None
+        return self.storage.address
+
+    @property
+    def name(self):
+        if self.storage is None:
+            return self.filename[:-len('.yaml')]
+        return self.storage.name
+
+    @property
+    def esp_platform(self):
+        if self.storage is None:
+            return None
+        return self.storage.esp_platform
+
+    @property
+    def board(self):
+        if self.storage is None:
+            return None
+        return self.storage.board
+
+    @property
+    def update_available(self):
+        if self.storage is None:
+            return True
+        return self.update_old != self.update_new
+
+    @property
+    def update_old(self):
+        if self.storage is None:
+            return ''
+        return self.storage.esphomeyaml_version or ''
+
+    @property
+    def update_new(self):
+        return const.__version__
+
+
 class MainRequestHandler(BaseHandler):
    def get(self):
        if not self.is_authenticated():
@@ -204,31 +290,194 @@ class MainRequestHandler(BaseHandler):
            return

        begin = bool(self.get_argument('begin', False))
-        files = sorted([f for f in os.listdir(CONFIG_DIR) if f.endswith('.yaml') and
-                        not f.startswith('.')])
-        full_path_files = [os.path.join(CONFIG_DIR, f) for f in files]
-        self.render("templates/index.html", files=files, full_path_files=full_path_files,
-                    version=const.__version__, begin=begin)
+        entries = _list_dashboard_entries()
+        version = const.__version__
+        docs_link = 'https://beta.esphomelib.com/esphomeyaml/' if 'b' in version else \
+            'https://esphomelib.com/esphomeyaml/'
+        mqtt_config = get_mqtt_config_lazy()
+
+        self.render("templates/index.html", entries=entries,
+                    version=version, begin=begin, docs_link=docs_link, mqtt_config=mqtt_config)
+
+
+def _ping_func(filename, address):
+    if os.name == 'nt':
+        command = ['ping', '-n', '1', address]
+    else:
+        command = ['ping', '-c', '1', address]
+    rc, _, _ = run_system_command(*command)
+    return filename, rc == 0
+
+
+class PingThread(threading.Thread):
+    def run(self):
+        pool = multiprocessing.Pool(processes=8)
+
+        while not STOP_EVENT.is_set():
+            # Only do pings if somebody has the dashboard open
+            PING_REQUEST.wait()
+            PING_REQUEST.clear()
+
+            def callback(ret):
+                PING_RESULT[ret[0]] = ret[1]
+
+            entries = _list_dashboard_entries()
+            queue = collections.deque()
+            for entry in entries:
+                if entry.address is None:
+                    PING_RESULT[entry.filename] = None
+                    continue
+
+                result = pool.apply_async(_ping_func, (entry.filename, entry.address),
+                                          callback=callback)
+                queue.append(result)
+
+            while queue:
+                item = queue[0]
+                if item.ready():
+                    queue.popleft()
+                    continue
+
+                try:
+                    item.get(0.1)
+                except OSError:
+                    # ping not installed
+                    pass
+                except multiprocessing.TimeoutError:
+                    pass
+
+                if STOP_EVENT.is_set():
+                    pool.terminate()
+                    return
+
+
+class PingRequestHandler(BaseHandler):
+    def get(self):
+        if not self.is_authenticated():
+            self.redirect('/login')
+            return
+
+        PING_REQUEST.set()
+        self.write(json.dumps(PING_RESULT))
+
+
+def is_allowed(configuration):
+    return os.path.sep not in configuration
+
+
+class EditRequestHandler(BaseHandler):
+    def get(self):
+        if not self.is_authenticated():
+            self.redirect('/login')
+            return
+        configuration = self.get_argument('configuration')
+        if not is_allowed(configuration):
+            self.set_status(401)
+            return
+
+        with open(os.path.join(CONFIG_DIR, configuration), 'r') as f:
+            content = f.read()
+        self.write(content)
+
+    def post(self):
+        if not self.is_authenticated():
+            self.redirect('/login')
+            return
+        configuration = self.get_argument('configuration')
+        if not is_allowed(configuration):
+            self.set_status(401)
+            return
+
+        with open(os.path.join(CONFIG_DIR, configuration), 'w') as f:
+            f.write(self.request.body)
+        self.set_status(200)
+        return
+
+
+PING_RESULT = {}  # type: dict
+STOP_EVENT = threading.Event()
+PING_REQUEST = threading.Event()


 class LoginHandler(BaseHandler):
    def get(self):
+        if USING_HASSIO_AUTH:
+            self.render_hassio_login()
+            return
        self.write('<html><body><form action="/login" method="post">'
                   'Password: <input type="password" name="password">'
                   '<input type="submit" value="Sign in">'
                   '</form></body></html>')

+    def render_hassio_login(self, error=None):
+        version = const.__version__
+        docs_link = 'https://beta.esphomelib.com/esphomeyaml/' if 'b' in version else \
+            'https://esphomelib.com/esphomeyaml/'
+
+        self.render("templates/login.html", version=version, docs_link=docs_link, error=error)
+
+    def post_hassio_login(self):
+        import requests
+
+        headers = {
+            'X-HASSIO-KEY': os.getenv('HASSIO_TOKEN'),
+        }
+        data = {
+            'username': str(self.get_argument('username', '')),
+            'password': str(self.get_argument('password', ''))
+        }
+        try:
+            req = requests.post('http://hassio/auth', headers=headers, data=data)
+            if req.status_code == 200:
+                self.set_secure_cookie("authenticated", "yes")
+                self.redirect('/')
+                return
+        except Exception as err:  # pylint: disable=broad-except
+            _LOGGER.warn("Error during Hass.io auth request: %s", err)
+            self.set_status(500)
+            self.render_hassio_login(error="Internal server error")
+            return
+        self.set_status(401)
+        self.render_hassio_login(error="Invalid username or password")
+
    def post(self):
+        if USING_HASSIO_AUTH:
+            self.post_hassio_login()
+            return
+
        password = str(self.get_argument("password", ''))
        password = hmac.new(password).digest()
-        if hmac.compare_digest(PASSWORD, password):
+        if hmac.compare_digest(PASSWORD_DIGEST, password):
            self.set_secure_cookie("authenticated", "yes")
        self.redirect("/")


 def make_app(debug=False):
+    def log_function(handler):
+        if handler.get_status() < 400:
+            log_method = access_log.info
+
+            if isinstance(handler, SerialPortRequestHandler) and not debug:
+                return
+            if isinstance(handler, PingRequestHandler) and not debug:
+                return
+        elif handler.get_status() < 500:
+            log_method = access_log.warning
+        else:
+            log_method = access_log.error
+
+        request_time = 1000.0 * handler.request.request_time()
+        # pylint: disable=protected-access
+        log_method("%d %s %.2fms", handler.get_status(),
+                   handler._request_summary(), request_time)
+
+    class StaticFileHandler(tornado.web.StaticFileHandler):
+        def set_extra_headers(self, path):
+            if debug:
+                self.set_header('Cache-Control', 'no-store, no-cache, must-revalidate, max-age=0')
+
    static_path = os.path.join(os.path.dirname(__file__), 'static')
-    return tornado.web.Application([
+    app = tornado.web.Application([
        (r"/", MainRequestHandler),
        (r"/login", LoginHandler),
        (r"/logs", EsphomeyamlLogsHandler),
@@ -238,39 +487,83 @@ def make_app(debug=False):
        (r"/clean-mqtt", EsphomeyamlCleanMqttHandler),
        (r"/clean", EsphomeyamlCleanHandler),
        (r"/hass-config", EsphomeyamlHassConfigHandler),
+        (r"/edit", EditRequestHandler),
        (r"/download.bin", DownloadBinaryRequestHandler),
        (r"/serial-ports", SerialPortRequestHandler),
+        (r"/ping", PingRequestHandler),
        (r"/wizard.html", WizardRequestHandler),
-        (r'/static/(.*)', tornado.web.StaticFileHandler, {'path': static_path}),
-    ], debug=debug, cookie_secret=PASSWORD)
+        (r'/static/(.*)', StaticFileHandler, {'path': static_path}),
+    ], debug=debug, cookie_secret=COOKIE_SECRET, log_function=log_function)
+    return app
+
+
+def _get_mqtt_config_impl():
+    import requests
+
+    headers = {
+        'X-HASSIO-KEY': os.getenv('HASSIO_TOKEN'),
+    }
+
+    mqtt_config = requests.get('http://hassio/services/mqtt', headers=headers).json()['data']
+    info = requests.get('http://hassio/host/info', headers=headers).json()['data']
+    host = '{}.local'.format(info['hostname'])
+    port = mqtt_config['port']
+    if port != 1883:
+        host = '{}:{}'.format(host, port)
+
+    return {
+        'ssl': mqtt_config['ssl'],
+        'host': host,
+        'username': mqtt_config.get('username', ''),
+        'password': mqtt_config.get('password', '')
+    }
+
+
+def get_mqtt_config_lazy():
+    global HASSIO_MQTT_CONFIG
+
+    if not ON_HASSIO:
+        return None
+
+    if HASSIO_MQTT_CONFIG is None:
+        try:
+            HASSIO_MQTT_CONFIG = _get_mqtt_config_impl()
+        except Exception:  # pylint: disable=broad-except
+            pass
+
+    return HASSIO_MQTT_CONFIG


 def start_web_server(args):
    global CONFIG_DIR
-    global PASSWORD
-
-    if tornado is None:
-        raise ESPHomeYAMLError("Attempted to load dashboard, but tornado is not installed! "
-                               "Please run \"pip2 install tornado esptool\" in your terminal.")
+    global PASSWORD_DIGEST
+    global USING_PASSWORD
+    global ON_HASSIO
+    global USING_HASSIO_AUTH
+    global COOKIE_SECRET

    CONFIG_DIR = args.configuration
-    if not os.path.exists(CONFIG_DIR):
-        os.makedirs(CONFIG_DIR)
+    mkdir_p(CONFIG_DIR)
+    mkdir_p(os.path.join(CONFIG_DIR, ".esphomeyaml"))

-    # HassIO options storage
-    PASSWORD = args.password
-    if os.path.isfile('/data/options.json'):
-        with open('/data/options.json') as f:
-            js = json.load(f)
-            PASSWORD = js.get('password') or PASSWORD
+    ON_HASSIO = args.hassio
+    if ON_HASSIO:
+        USING_HASSIO_AUTH = not bool(os.getenv('DISABLE_HA_AUTHENTICATION'))
+        USING_PASSWORD = False
+    else:
+        USING_HASSIO_AUTH = False
+        USING_PASSWORD = args.password

-    if PASSWORD:
-        PASSWORD = hmac.new(str(PASSWORD)).digest()
-        # Use the digest of the password as our cookie secret. This makes sure the cookie
-        # isn't too short. It, of course, enables local hash brute forcing (because the cookie
-        # secret can be brute forced without making requests). But the hashing algorithm used
-        # by tornado is apparently strong enough to make brute forcing even a short string pretty
-        # hard.
+    if USING_PASSWORD:
+        PASSWORD_DIGEST = hmac.new(args.password).digest()
+
+    if USING_HASSIO_AUTH or USING_PASSWORD:
+        path = esphomeyaml_storage_path(CONFIG_DIR)
+        storage = EsphomeyamlStorageJSON.load(path)
+        if storage is None:
+            storage = EsphomeyamlStorageJSON.get_default()
+            storage.save(path)
+        COOKIE_SECRET = storage.cookie_secret

    _LOGGER.info("Starting dashboard web server on port %s and configuration dir %s...",
                 args.port, CONFIG_DIR)
@@ -282,7 +575,12 @@ def start_web_server(args):

        webbrowser.open('localhost:{}'.format(args.port))

+    ping_thread = PingThread()
+    ping_thread.start()
    try:
        tornado.ioloop.IOLoop.current().start()
    except KeyboardInterrupt:
        _LOGGER.info("Shutting down...")
+        STOP_EVENT.set()
+        PING_REQUEST.set()
+        ping_thread.join()