217 lines
9.1 KiB
Python
217 lines
9.1 KiB
Python
import asyncio
|
|
import socket
|
|
import aioping
|
|
from datetime import datetime, timedelta
|
|
|
|
from aiohttp import ClientSession, ClientTimeout
|
|
from injector import inject, singleton
|
|
|
|
from backup.config import Config, Setting, VERSION, _DEFAULTS, PRIVATE
|
|
from backup.exceptions import KnownError
|
|
from backup.util import GlobalInfo, Resolver
|
|
from backup.time import Time
|
|
from backup.worker import Worker
|
|
from backup.logger import getLogger, getHistory
|
|
from backup.ha import HaRequests, HaSource
|
|
from backup.model import Coordinator, DestinationPrecache
|
|
from yarl import URL
|
|
|
|
logger = getLogger(__name__)
|
|
ERROR_LOG_LENGTH = 30
|
|
|
|
|
|
@singleton
|
|
class DebugWorker(Worker):
|
|
@inject
|
|
def __init__(self, time: Time, info: GlobalInfo, config: Config, resolver: Resolver, session: ClientSession, ha: HaRequests, coord: Coordinator, ha_source: HaSource, precache: DestinationPrecache):
|
|
super().__init__("Debug Worker", self.doWork, time, interval=10)
|
|
self.time = time
|
|
self._info = info
|
|
self.config = config
|
|
self.ha = ha
|
|
self.ha_source = ha_source
|
|
self.coord = coord
|
|
|
|
self.last_dns_update = None
|
|
self.dns_info = None
|
|
|
|
self.last_sent_error = None
|
|
self.last_sent_error_time = None
|
|
self._health = None
|
|
self.resolver = resolver
|
|
self.session = session
|
|
self._last_server_check = None
|
|
self._last_server_refresh = timedelta(days=1)
|
|
self._precache = precache
|
|
|
|
async def doWork(self):
|
|
if not self.last_dns_update or self.time.now() > self.last_dns_update + timedelta(hours=12):
|
|
await self.updateDns()
|
|
if not self._last_server_check or self.time.now() > self._last_server_check + self._last_server_refresh:
|
|
await self.updateHealthCheck()
|
|
if self.config.get(Setting.SEND_ERROR_REPORTS):
|
|
try:
|
|
await self.maybeSendErrorReport()
|
|
except Exception:
|
|
pass
|
|
|
|
# Once per day, query the health endpoint of the token server to see who is up.
|
|
# This checks for broadcast messages for all users and also finds which token
|
|
# servers are available.
|
|
async def updateHealthCheck(self):
|
|
headers = {
|
|
'client': self.config.clientIdentifier(),
|
|
'addon_version': VERSION
|
|
}
|
|
self._last_server_check = self.time.now()
|
|
for host in self.config.getTokenServers():
|
|
url = host.with_path("/health")
|
|
try:
|
|
async with self.session.get(url, headers=headers, timeout=ClientTimeout(total=10)) as resp:
|
|
resp.raise_for_status()
|
|
self._health = await resp.json()
|
|
self._last_server_refresh = timedelta(days=1)
|
|
return
|
|
except: # noqa: E722
|
|
# ignore any error and just try a different endpoint
|
|
pass
|
|
|
|
# no good token host could be found, so reset it to the default and check again sooner.
|
|
self._last_server_refresh = timedelta(minutes=1)
|
|
|
|
async def maybeSendErrorReport(self):
|
|
error = self._info._last_error
|
|
if error is not None:
|
|
if isinstance(error, KnownError):
|
|
error = error.code()
|
|
else:
|
|
error = logger.formatException(error)
|
|
if error != self.last_sent_error:
|
|
self.last_sent_error = error
|
|
if error is not None:
|
|
self.last_sent_error_time = self.time.now()
|
|
package = await self.buildErrorReport(error)
|
|
else:
|
|
package = self.buildClearReport()
|
|
logger.info("Sending error report (see settings to disable)")
|
|
headers = {
|
|
'client': self.config.clientIdentifier(),
|
|
'addon_version': VERSION
|
|
}
|
|
url = URL(self.config.get(Setting.AUTHORIZATION_HOST)).with_path("/logerror")
|
|
async with self.session.post(url, headers=headers, json=package):
|
|
pass
|
|
|
|
async def updateDns(self):
|
|
self.last_dns_update = self.time.now()
|
|
try:
|
|
# Resolve google's addresses
|
|
self.dns_info = await self.getPingInfo()
|
|
self._info.setDnsInfo(self.dns_info)
|
|
except Exception as e:
|
|
self.dns_info = logger.formatException(e)
|
|
|
|
async def buildErrorReport(self, error):
|
|
config_special = {}
|
|
for setting in Setting:
|
|
if self.config.get(setting) != _DEFAULTS[setting]:
|
|
if setting in PRIVATE:
|
|
config_special[str(setting)] = "REDACTED"
|
|
else:
|
|
config_special[str(setting)] = self.config.get(setting)
|
|
report = {}
|
|
report['config'] = config_special
|
|
report['time'] = self.formatDate(self.time.now())
|
|
report['start_time'] = self.formatDate(self._info._start_time)
|
|
report['addon_version'] = VERSION
|
|
report['failure_time'] = self.formatDate(self._info._last_failure_time)
|
|
report['failure_count'] = self._info._failures
|
|
report['sync_last_start'] = self.formatDate(self._info._last_sync_start)
|
|
report['sync_count'] = self._info._syncs
|
|
report['sync_success_count'] = self._info._successes
|
|
report['sync_last_success'] = self.formatDate(self._info._last_sync_success)
|
|
report['upload_count'] = self._info._uploads
|
|
report['upload_last_size'] = self._info._last_upload_size
|
|
report['upload_last_attempt'] = self.formatDate(self._info._last_upload)
|
|
report['next_sync'] = self.formatDate(self.coord.nextSyncAttempt())
|
|
report['next_backup'] = self.formatDate(self.coord.nextBackupTime())
|
|
report['next_cache_warm'] = self.formatDate(self._precache.getNextWarmDate())
|
|
report['time_offset'] = self._time.offset.total_seconds()
|
|
|
|
report['debug'] = self._info.debug
|
|
report['version'] = VERSION
|
|
report['error'] = error
|
|
report['client'] = self.config.clientIdentifier()
|
|
|
|
if self.ha_source.isInitialized() and self.ha_source.host_info and self.ha_source.super_info and self.ha_source.ha_info:
|
|
report["super_version"] = self.ha_source.host_info.get('supervisor', "None")
|
|
report["hassos_version"] = self.ha_source.host_info.get('hassos', "None")
|
|
report["docker_version"] = self.ha_source.host_info.get('docker', "None")
|
|
report["machine"] = self.ha_source.host_info.get('machine', "None")
|
|
report["supervisor_channel"] = self.ha_source.host_info.get('channel', "None")
|
|
report["arch"] = self.ha_source.super_info.get('arch', "None")
|
|
report["timezone"] = self.ha_source.super_info.get('timezone', "None")
|
|
report["ha_version"] = self.ha_source.ha_info.get('version', "None")
|
|
else:
|
|
report["super_version"] = "Uninitialized"
|
|
report["arch"] = "Uninitialized"
|
|
report["timezone"] = "Uninitialized"
|
|
report["ha_version"] = "Uninitialized"
|
|
report["backups"] = self.coord.buildBackupMetrics()
|
|
return report
|
|
|
|
async def buildBugReportData(self, error):
|
|
report = await self.buildErrorReport(error)
|
|
report['addon_logs'] = "\n".join(b for a, b in list(getHistory(0, False))[-ERROR_LOG_LENGTH:])
|
|
try:
|
|
report['super_logs'] = "\n".join((await self.ha.getSuperLogs()).split("\n")[-ERROR_LOG_LENGTH:])
|
|
except Exception as e:
|
|
report['super_logs'] = logger.formatException(e)
|
|
try:
|
|
report['core_logs'] = "\n".join((await self.ha.getCoreLogs()).split("\n")[-ERROR_LOG_LENGTH:])
|
|
except Exception as e:
|
|
report['core_logs'] = logger.formatException(e)
|
|
return report
|
|
|
|
def buildClearReport(self):
|
|
duration = self.time.now() - self.last_sent_error_time
|
|
report = {
|
|
'duration': str(duration)
|
|
}
|
|
return report
|
|
|
|
def formatDate(self, date: datetime):
|
|
if date is None:
|
|
return "Never"
|
|
else:
|
|
return date.isoformat()
|
|
|
|
async def getPingInfo(self):
|
|
who = self.config.get(Setting.DRIVE_HOST_NAME)
|
|
ips = await self.resolve(who)
|
|
results = {who: {}}
|
|
tasks = {who: {}}
|
|
for ip in ips:
|
|
results[who][ip] = "Unknown"
|
|
tasks[who][ip] = asyncio.create_task(aioping.ping(ip, timeout=self.config.get(Setting.PING_TIMEOUT)))
|
|
|
|
# ping each server
|
|
for server in tasks.keys():
|
|
for ip in tasks[server].keys():
|
|
try:
|
|
time = await tasks[server][ip]
|
|
results[server][ip] = f"{round(time * 1000, 0)} ms"
|
|
except Exception as e:
|
|
results[server][ip] = str(e)
|
|
return results
|
|
|
|
async def resolve(self, who: str):
|
|
try:
|
|
ret = [who]
|
|
addresses = await self.resolver.resolve(who, 443, socket.AF_INET)
|
|
for address in addresses:
|
|
ret.append(address['host'])
|
|
return ret
|
|
except Exception:
|
|
return [who]
|