New Addon

Google BK

hassio-google-drive-backup/backup/model/__init__.py

@@ -0,0 +1,13 @@
+# flake8: noqa
+from .backupscheme import GenerationalScheme, OldestScheme, GenConfig, BackupScheme
+from .coordinator import Coordinator
+from .model import BackupSource, BackupDestination, Model
+from .syncer import Scyncer
+from .backups import AbstractBackup, Backup
+from .drivebackup import DriveBackup
+from .dummybackup import DummyBackup
+from .dummybackupsource import DummyBackupSource
+from .habackup import HABackup
+from .simulatedsource import SimulatedSource
+from .precache import Precache
+from .destinationprecache import DestinationPrecache

hassio-google-drive-backup/backup/model/backups.py

@@ -0,0 +1,318 @@
+
+from datetime import datetime, timedelta
+from typing import Dict, Optional
+from dateutil.tz import tzutc
+from ..util import Estimator
+
+from ..const import SOURCE_GOOGLE_DRIVE, SOURCE_HA
+from ..logger import getLogger
+from ..config import CreateOptions
+
+logger = getLogger(__name__)
+
+PROP_TYPE = "type"
+PROP_VERSION = "version"
+PROP_PROTECTED = "protected"
+PROP_RETAINED = "retained"
+PROP_NOTE = "note"
+
+DRIVE_KEY_TEXT = "Google Drive's backup metadata"
+HA_KEY_TEXT = "Home Assistant's backup metadata"
+
+
+class AbstractBackup():
+    def __init__(self, name: str, slug: str, source: str, date: str, size: int, version: str, backupType: str, protected: bool, note=None, retained: bool = False, uploadable: bool = False, details={}, pending=False):
+        self._options = None
+        self._name = name
+        self._slug = slug
+        self._source = source
+        self._date = date
+        self._size = size
+        self._retained = retained
+        self._uploadable = uploadable
+        self._details = details
+        self._version = version
+        self._backupType = backupType
+        self._protected = protected
+        self._ignore = False
+        self._note = note
+        self._pending = pending
+
+    def isPending(self):
+        return self._pending
+
+    def setOptions(self, options: CreateOptions):
+        self._options = options
+
+    def getOptions(self) -> CreateOptions:
+        return self._options
+
+    def name(self) -> str:
+        return self._name
+
+    def slug(self) -> str:
+        return self._slug
+
+    def size(self) -> int:
+        return self._size
+
+    def note(self) -> str:
+        return self._note
+
+    def sizeInt(self) -> int:
+        try:
+            return int(self.size())
+        except ValueError:
+            return 0
+
+    def date(self) -> datetime:
+        return self._date
+
+    def source(self) -> str:
+        return self._source
+
+    def retained(self) -> str:
+        return self._retained
+
+    def version(self):
+        return self._version
+
+    def backupType(self):
+        return self._backupType
+
+    def protected(self):
+        return self._protected
+
+    def setRetained(self, retained):
+        self._retained = retained
+
+    def uploadable(self) -> bool:
+        return self._uploadable
+
+    def considerForPurge(self) -> bool:
+        return not self.retained()
+
+    def setUploadable(self, uploadable):
+        self._uploadable = uploadable
+
+    def details(self):
+        return self._details
+
+    def setNote(self, note: str):
+        self._note = note
+
+    def status(self):
+        return None
+
+    def madeByTheAddon(self):
+        return True
+
+    def ignore(self):
+        return self._ignore
+
+    def setIgnore(self, ignore):
+        self._ignore = ignore
+
+
+class Backup(object):
+    """
+    Represents a Home Assistant backup stored on Google Drive, locally in
+    Home Assistant, or a pending backup we expect to see show up later
+    """
+
+    def __init__(self, backup: Optional[AbstractBackup] = None):
+        self.sources: Dict[str, AbstractBackup] = {}
+        self._purgeNext: Dict[str, bool] = {}
+        self._options = None
+        self._status_override = None
+        self._status_override_args = None
+        self._state_detail = None
+        self._upload_source = None
+        self._upload_source_name = None
+        self._upload_fail_info = None
+        if backup is not None:
+            self.addSource(backup)
+
+    def setOptions(self, options):
+        self._options = options
+
+    def getOptions(self):
+        return self._options
+
+    def updatePurge(self, source: str, purge: bool):
+        self._purgeNext[source] = purge
+
+    def addSource(self, backup: AbstractBackup):
+        self.sources[backup.source()] = backup
+        if backup.getOptions() and not self.getOptions():
+            self.setOptions(backup.getOptions())
+
+    def getStatusDetail(self):
+        return self._state_detail
+
+    def setStatusDetail(self, info):
+        self._state_detail = info
+
+    def removeSource(self, source):
+        if source in self.sources:
+            del self.sources[source]
+        if source in self._purgeNext:
+            del self._purgeNext[source]
+
+    def getPurges(self):
+        return self._purgeNext
+
+    def uploadInfo(self):
+        if not self._upload_source:
+            return {}
+        elif self._upload_source.progress() == 100:
+            return {}
+        else:
+            return {
+                'progress': self._upload_source.progress()
+            }
+
+    def getSource(self, source: str):
+        return self.sources.get(source, None)
+
+    def name(self):
+        for backup in self.sources.values():
+            return backup.name()
+        return "error"
+
+    def note(self):
+        longest = None
+        for backup in self.sources.values():
+            if backup.note() is not None and (longest is None or len(backup.note()) > len(longest)):
+                longest = backup.note()
+        return longest
+
+    def slug(self) -> str:
+        for backup in self.sources.values():
+            return backup.slug()
+        return "error"
+
+    def size(self) -> int:
+        for backup in self.sources.values():
+            return backup.size()
+        return 0
+
+    def sizeInt(self) -> int:
+        for backup in self.sources.values():
+            return backup.sizeInt()
+        return 0
+
+    def backupType(self) -> str:
+        for backup in self.sources.values():
+            return backup.backupType()
+        return "error"
+
+    def version(self) -> str:
+        for backup in self.sources.values():
+            if backup.version() is not None:
+                return backup.version()
+        return None
+
+    def details(self):
+        for backup in self.sources.values():
+            if backup.details() is not None:
+                return backup.details()
+        return {}
+
+    def getUploadInfo(self, time):
+        if self._upload_source_name is None:
+            return None
+        ret = {
+            'name': self._upload_source_name
+        }
+        if self._upload_fail_info:
+            ret['failure'] = self._upload_fail_info
+        elif self._upload_source is not None:
+            ret['progress'] = self._upload_source.progress()
+            ret['speed'] = self._upload_source.speed(timedelta(seconds=20))
+            ret['total'] = self._upload_source.position()
+            ret['started'] = time.formatDelta(self._upload_source.startTime())
+        return ret
+
+    def protected(self) -> bool:
+        for backup in self.sources.values():
+            return backup.protected()
+        return False
+
+    def ignore(self) -> bool:
+        for backup in self.sources.values():
+            if not backup.ignore():
+                return False
+        return True
+
+    def date(self) -> datetime:
+        for backup in self.sources.values():
+            return backup.date()
+        return datetime.now(tzutc())
+
+    def sizeString(self) -> str:
+        size_string = self.size()
+        if type(size_string) == str:
+            return size_string
+        return Estimator.asSizeString(size_string)
+
+    def status(self) -> str:
+        # TODO: Drive Specific
+        if self._status_override is not None:
+            return self._status_override.format(*self._status_override_args)
+
+        for backup in self.sources.values():
+            status = backup.status()
+            if status:
+                return status
+
+        inDrive = self.getSource(SOURCE_GOOGLE_DRIVE) is not None
+        inHa = self.getSource(SOURCE_HA) is not None
+
+        if inDrive and inHa:
+            return "Backed Up"
+        if inDrive:
+            return "Drive Only"
+        if inHa:
+            return "HA Only"
+        return "Deleted"
+
+    def isDeleted(self) -> bool:
+        return len(self.sources) == 0
+
+    def overrideStatus(self, format, *args) -> None:
+        self._status_override = format
+        self._status_override_args = args
+
+    def setUploadSource(self, source_name: str, source):
+        self._upload_source = source
+        self._upload_source_name = source_name
+        self._upload_fail_info = None
+
+    def clearUploadSource(self):
+        self._upload_source = None
+        self._upload_source_name = None
+        self._upload_fail_info = None
+
+    def uploadFailure(self, info):
+        self._upload_source = None
+        self._upload_fail_info = info
+
+    def clearStatus(self):
+        self._status_override = None
+        self._status_override_args = None
+
+    def isPending(self):
+        for backup in self.sources.values():
+            if backup.isPending():
+                return True
+        return False
+
+    def __str__(self) -> str:
+        return "<Slug: {0} {1} {2}>".format(self.slug(), " ".join(self.sources), self.date().isoformat())
+
+    def __format__(self, format_spec: str) -> str:
+        return self.__str__()
+
+    def __repr__(self) -> str:
+        return self.__str__()

hassio-google-drive-backup/backup/model/backupscheme.py

@@ -0,0 +1,236 @@
+from abc import ABC, abstractmethod
+from calendar import monthrange
+from datetime import datetime, timedelta, date
+from typing import List, Optional, Sequence, Set, Tuple, Any, Union
+
+from .backups import Backup
+from backup.util import RangeLookup
+from ..time import Time
+from ..config import GenConfig
+from ..logger import getLogger
+logger = getLogger(__name__)
+
+
+class BackupScheme(ABC):
+    def __init__(self):
+        pass
+
+    @abstractmethod
+    def getOldest(self, backups: Sequence[Backup]) -> Tuple[str, Optional[Backup]]:
+        pass
+
+    def handleNaming(self, backups: Sequence[Backup]) -> None:
+        for backup in backups:
+            backup.setStatusDetail(None)
+
+
+class DeleteAfterUploadScheme(BackupScheme):
+    def __init__(self, source: str, destinations: List[str]):
+        self.source = source
+        self.destinations = destinations
+
+    def getOldest(self, backups: List[Backup]):
+        consider = []
+        for backup in backups:
+            uploaded = True
+            if backup.getSource(self.source) is None:
+                # No source, so ignore it
+                uploaded = False
+            for destination in self.destinations:
+                if backup.getSource(destination) is None:
+                    # its not in destination, so ignore it
+                    uploaded = False
+            if uploaded:
+                consider.append(backup)
+
+        # Delete the oldest first
+        return OldestScheme().getOldest(consider)
+
+
+class OldestScheme(BackupScheme):
+    def __init__(self, count=0):
+        self.count = count
+
+    def getOldest(self, backups: Sequence[Backup]) -> Tuple[Any, Union[Backup, None]]:
+        if len(backups) <= self.count:
+            return None, None
+        return "default", min(backups, default=None, key=lambda s: s.date())
+
+    def handleNaming(self, backups: Sequence[Backup]) -> None:
+        for backup in backups:
+            backup.setStatusDetail(None)
+
+
+class Partition(object):
+    def __init__(self, start: datetime, end: datetime, prefer: datetime, time: Time, details=None, delete_only: bool = False):
+        self.start: datetime = start
+        self.end: datetime = end
+        self.prefer: datetime = prefer
+        self.time = time
+        self.details = details
+        self.selected = None
+        self._delete_only_partitions = delete_only
+
+    def select(self, backups: List[Backup]) -> Optional[Backup]:
+        options = list(RangeLookup(backups, lambda s: s.date()).matches(self.start, self.end - timedelta(milliseconds=1)))
+
+        searcher = lambda s: self.day(s.date()) == self.day(self.prefer)
+
+        preferred = list(filter(searcher, options))
+        if len(preferred) > 0:
+            # If there is a backup on the "preferred" day, then use the latest backup on that day
+            self.selected = max(preferred, default=None, key=Backup.date)
+        else:
+            #  Otherwise, use the earliest backup over the valid period.
+            self.selected = min(options, default=None, key=Backup.date)
+        return self.selected
+
+    def delta(self) -> timedelta:
+        return self.end - self.start
+
+    def day(self, date: datetime):
+        # TODO: this conversion isn't time-zone safe, but is ok because we only use it to compare local day to local day.
+        local = self.time.toLocal(date)
+        return datetime(day=local.day, month=local.month, year=local.year)
+
+    # True if the partition exists only to determine why a snapshot is getting deleted.
+    @property
+    def is_delete_only(self):
+        return self._delete_only_partitions
+
+    def __hash__(self):
+        """Overrides the default implementation"""
+        return hash(tuple(sorted(self.__dict__.items())))
+
+
+class GenerationalScheme(BackupScheme):
+    def __init__(self, time: Time, config: GenConfig, count=0):
+        self.count = count
+        self.time: Time = time
+        self.config = config
+
+    def _buildPartitions(self, backups_input):
+        backups: List[Backup] = list(backups_input)
+
+        # build the list of dates we should partition by
+        day_of_week = 3
+        weekday_lookup = {
+            'mon': 0,
+            'tue': 1,
+            'wed': 2,
+            'thu': 3,
+            'fri': 4,
+            'sat': 5,
+            'sun': 6,
+        }
+        if self.config.day_of_week in weekday_lookup:
+            day_of_week = weekday_lookup[self.config.day_of_week]
+
+        last = self.time.toLocal(backups[len(backups) - 1].date())
+        lookups: List[Partition] = []
+        currentDay = self.day(last)
+        if self.config.days > 0:
+            for x in range(0, self.config.days + 1):
+                nextDay = self.day(currentDay, add_days=1)
+                lookups.append(
+                    Partition(currentDay, nextDay, currentDay, self.time, "Day {0} of {1}".format(x + 1, self.config.days), delete_only=(x >= self.config.days)))
+                currentDay = self.day(currentDay, add_days=-1)
+
+        if self.config.weeks > 0:
+            for x in range(0, self.config.weeks + 1):
+                # Start at the first monday preceeding the last backup
+                start = self.time.local(last.year, last.month, last.day)
+                start = self.day(start, add_days=-1 * start.weekday())
+
+                # Move back x weeks
+                start = self.day(start, add_days=-7 * x)
+                end = self.day(start, add_days=7)
+
+                # Only consider backups from that week after the start day
+                # TODO: should this actually "prefer" the day of week but start on monday?
+                start = self.day(start, add_days=day_of_week)
+                lookups.append(Partition(start, end, start, self.time, "Week {0} of {1}".format(x + 1, self.config.weeks), delete_only=(x >= self.config.weeks)))
+
+        if self.config.months > 0:
+            for x in range(0, self.config.months + 1):
+                year_offset = int(x / 12)
+                month_offset = int(x % 12)
+                if last.month - month_offset < 1:
+                    year_offset = year_offset + 1
+                    month_offset = month_offset - 12
+                start = self.time.local(
+                    last.year - year_offset, last.month - month_offset, 1)
+                weekday, days = monthrange(start.year, start.month)
+                end = start + timedelta(days=days)
+                lookups.append(Partition(
+                    start, end, start + timedelta(days=self.config.day_of_month - 1), self.time,
+                    "{0} ({1} of {2} months)".format(start.strftime("%B"), x + 1, self.config.months), delete_only=(x >= self.config.months)))
+
+        if self.config.years > 0:
+            for x in range(0, self.config.years + 1):
+                start = self.time.local(last.year - x, 1, 1)
+                end = self.time.local(last.year - x + 1, 1, 1)
+                lookups.append(Partition(
+                    start, end, start + timedelta(days=self.config.day_of_year - 1), self.time,
+                    "{0} ({1} of {2} years)".format(start.strftime("%Y"), x + 1, self.config.years), delete_only=(x >= self.config.years)))
+
+        # Keep track of which backups are being saved for which time period.
+        for lookup in lookups:
+            lookup.select(backups)
+        return lookups
+
+    def getOldest(self, backups: Sequence[Backup]):
+        if len(backups) == 0:
+            return None, None
+
+        sorted = list(backups)
+        sorted.sort(key=lambda s: s.date())
+
+        partitions = self._buildPartitions(sorted)
+        keepers: Set[Backup] = set()
+        for part in partitions:
+            if part.selected is not None and not part.is_delete_only:
+                keepers.add(part.selected)
+
+        extras = []
+        for backup in sorted:
+            if backup not in keepers:
+                extras.append(backup)
+
+        if self.config.aggressive and len(extras) > 0:
+            match = min(filter(lambda p: p.selected == extras[0], partitions), key=Partition.delta, default=None)
+            if match is not None:
+                return match, extras[0]
+            return "default", extras[0]
+
+        if len(sorted) <= self.count and not self.config.aggressive:
+            return "default", None
+        elif (self.config.aggressive or len(sorted) > self.count) and len(extras) > 0:
+            return "default", min(extras, default=None, key=lambda s: s.date())
+        elif len(sorted) > self.count:
+            # no non-keep is invalid, so delete the oldest keeper
+            return "default", min(keepers, default=None, key=lambda s: s.date())
+        return None, None
+
+    def handleNaming(self, backups: Sequence[Backup]) -> None:
+        sorted = list(backups)
+        sorted.sort(key=lambda s: s.date())
+        for backup in sorted:
+            backup.setStatusDetail(None)
+        # Ignored snapshots should have their label cleared in case
+        # it was added previosuly, but should not get new labels
+        unignored = list(filter(lambda s: not s.ignore(), sorted))
+
+        if len(unignored) == 0:
+            return
+        for part in self._buildPartitions(unignored):
+            if part.selected is not None:
+                if part.selected.getStatusDetail() is None:
+                    part.selected.setStatusDetail([])
+                part.selected.getStatusDetail().append(part.details)
+
+    def day(self, utc_datetime: datetime, add_days=0):
+        local = self.time.toLocal(utc_datetime)
+
+        local_date = date.fromordinal(date(local.year, local.month, local.day).toordinal() + add_days)
+        return self.time.localize(datetime(local_date.year, local_date.month, local_date.day, 0, 0))

hassio-google-drive-backup/backup/model/coordinator.py

@@ -0,0 +1,354 @@
+from asyncio import CancelledError, Task, create_task, wait, Event
+from datetime import timedelta
+from threading import Lock
+from typing import Dict, List
+
+from injector import inject, singleton
+
+from backup.config import Config, Setting, CreateOptions, DurationParser
+from backup.exceptions import (KnownError, LogicError, NoBackup, PleaseWait,
+                               UserCancelledError)
+from backup.util import GlobalInfo, Backoff, Estimator
+from backup.time import Time
+from backup.worker import Trigger
+from backup.logger import getLogger
+from backup.creds.creds import Creds
+from .precache import Precache
+from .model import BackupSource, Model
+from .backups import AbstractBackup, Backup, SOURCE_HA
+from random import Random
+
+logger = getLogger(__name__)
+
+
+@singleton
+class Coordinator(Trigger):
+    @inject
+    def __init__(self, model: Model, time: Time, config: Config, global_info: GlobalInfo, estimator: Estimator):
+        super().__init__()
+        self._model = model
+        self._precache: Precache = None
+        self._time = time
+        self._config = config
+        self._lock: Lock = Lock()
+        self._global_info: GlobalInfo = global_info
+        self._sources: Dict[str, BackupSource] = {
+            self._model.source.name(): self._model.source,
+            self._model.dest.name(): self._model.dest
+        }
+        self._backoff = Backoff(initial=0, base=10, max=config.get(Setting.MAX_BACKOFF_SECONDS))
+        self._estimator = estimator
+        self._busy = False
+        self._sync_task: Task = None
+        self._sync_start = Event()
+        self._sync_wait = Event()
+        self._sync_wait.set()
+        self._random = Random()
+        self._random.seed()
+        self._next_sync_offset = self._random.random()
+        self._global_info.triggerBackupCooldown(timedelta(minutes=self._config.get(Setting.BACKUP_STARTUP_DELAY_MINUTES)))
+        self.trigger()
+
+    def saveCreds(self, creds: Creds):
+        if not self._model.dest.enabled():
+            # Since this is the first time saving credentials (eg the addon was just enabled).  Hold off on
+            # automatic backups for a few minutes to give the user a little while to figure out whats going on.
+            self._global_info.triggerBackupCooldown(timedelta(minutes=self._config.get(Setting.BACKUP_STARTUP_DELAY_MINUTES)))
+
+        self._model.dest.saveCreds(creds)
+        self._global_info.credsSaved()
+
+    def setPrecache(self, precache: Precache):
+        self._precache = precache
+
+    def name(self):
+        return "Coordinator"
+
+    def enabled(self) -> bool:
+        return self._model.enabled()
+
+    def isWaitingForStartup(self):
+        return self._model.waiting_for_startup
+
+    def ignoreStartupDelay(self):
+        self._model.ignore_startup_delay = True
+
+    async def check(self) -> bool:
+        if self._time.now() >= self.nextSyncAttempt():
+            self.reset()
+            return True
+        else:
+            return await super().check()
+
+    async def sync(self):
+        await self._withSoftLock(lambda: self._sync_wrapper())
+
+    def isSyncing(self):
+        task = self._sync_task
+        return task is not None and not task.done()
+
+    def isWorkingThroughUpload(self):
+        return self.isSyncing() and self._model.isWorkingThroughUpload()
+
+    async def waitForSyncToFinish(self):
+        task = self._sync_task
+        if task is not None:
+            await task
+
+    async def cancel(self):
+        task = self._sync_task
+        if task is not None and not task.done():
+            task.cancel()
+            self.clearCaches()
+            await wait([task])
+
+    def nextSyncAttempt(self):
+        if self._global_info._last_error is not None:
+            # we had an error last
+            failure = self._global_info._last_failure_time
+            if failure is None:
+                return self._time.now() - timedelta(minutes=1)
+            return failure + timedelta(seconds=self._backoff.peek())
+        else:
+            scheduled = self._global_info._last_success
+            if scheduled is None:
+                scheduled = self._time.now() - timedelta(minutes=1)
+            else:
+                scheduled += timedelta(seconds=self.nextSyncCheckOffset())
+            next_backup = self.nextBackupTime()
+            if next_backup is None:
+                return scheduled
+            else:
+                return min(self.nextBackupTime(), scheduled)
+
+    def nextSyncCheckOffset(self):
+        """Determines how long we shoudl wait from the last check the refresh the cache of backups from Google Drive and Home Assistant"""
+        # If we always sync MAX_SYNC_INTERVAL_SECONDS secodns after the last
+        # check, then the addon in aggregate puts a really high strain on google
+        # on every hour and the addon's auth servers need to be provisioned for
+        # a big peak, which is epxensive.  Instead we add some randomness to the time interval.
+        randomness_max = self._config.get(Setting.MAX_SYNC_INTERVAL_SECONDS) * self._config.get(Setting.DEFAULT_SYNC_INTERVAL_VARIATION)
+        non_randomness = self._config.get(Setting.MAX_SYNC_INTERVAL_SECONDS) - randomness_max
+
+        # The offset should be stable between syncs, which gets controlled by updating _next_sync_offset on each good sync
+        return self._next_sync_offset * randomness_max + non_randomness
+
+    def nextBackupTime(self, include_pending=True):
+        return self._buildModel().nextBackup(self._time.now(), include_pending)
+
+    def buildBackupMetrics(self):
+        info = {}
+        for source in self._sources:
+            source_class = self._sources[source]
+            source_info = {
+                'backups': 0,
+                'retained': 0,
+                'deletable': 0,
+                'name': source,
+                'title': source_class.title(),
+                'latest': None,
+                'max': source_class.maxCount(),
+                'enabled': source_class.enabled(),
+                'icon': source_class.icon(),
+                'ignored': 0,
+                'detail': source_class.detail()
+            }
+            size = 0
+            ignored_size = 0
+            latest = None
+            for backup in self.backups():
+                data: AbstractBackup = backup.getSource(source)
+                if data is None:
+                    continue
+                if data.ignore() and backup.ignore():
+                    source_info['ignored'] += 1
+                if backup.ignore():
+                    ignored_size += backup.size()
+                    continue
+                source_info['backups'] += 1
+                if data.retained():
+                    source_info['retained'] += 1
+                else:
+                    source_info['deletable'] += 1
+                if latest is None or data.date() > latest:
+                    latest = data.date()
+                size += int(data.sizeInt())
+            if latest is not None:
+                source_info['latest'] = self._time.asRfc3339String(latest)
+            source_info['size'] = Estimator.asSizeString(size)
+            source_info['ignored_size'] = Estimator.asSizeString(ignored_size)
+            free_space = source_class.freeSpace()
+            if free_space is not None:
+                source_info['free_space'] = Estimator.asSizeString(free_space)
+            info[source] = source_info
+        return info
+
+    async def _sync_wrapper(self):
+        self._sync_task = create_task(
+            self._sync(), name="Internal sync worker")
+        await wait([self._sync_task])
+
+    async def _sync(self):
+        try:
+            self._sync_start.set()
+            await self._sync_wait.wait()
+            logger.info("Syncing Backups")
+            self._global_info.sync()
+            self._estimator.refresh()
+            await self._buildModel().sync(self._time.now())
+            self._next_sync_offset = self._random.random()
+            self._global_info.success()
+            self._backoff.reset()
+            self._global_info.setSkipSpaceCheckOnce(False)
+        except BaseException as e:
+            self.handleError(e)
+        finally:
+            if self._precache:
+                # Any sync should invalidate the precache regardless of the outcome
+                # so the next sync uses fresh data
+                self.clearCaches()
+            self._updateFreshness()
+
+    def handleError(self, e):
+        if isinstance(e, CancelledError):
+            e = UserCancelledError()
+        if isinstance(e, KnownError):
+            known: KnownError = e
+            logger.error(known.message())
+            if known.retrySoon():
+                self._backoff.backoff(e)
+            else:
+                self._backoff.maxOut()
+        else:
+            logger.printException(e)
+            self._backoff.backoff(e)
+        self._global_info.failed(e)
+
+        text = DurationParser().format(timedelta(seconds=self._backoff.peek()))
+        logger.info("I'll try again in {0}".format(text))
+
+    def backups(self) -> List[Backup]:
+        ret = list(self._model.backups.values())
+        ret.sort(key=lambda s: s.date())
+        return ret
+
+    async def uploadBackups(self, slug):
+        await self._withSoftLock(lambda: self._uploadBackup(slug))
+
+    async def _uploadBackup(self, slug):
+        self.clearCaches()
+        backup = self._ensureBackup(self._model.dest.name(), slug)
+        backup_dest = backup.getSource(self._model.dest.name())
+        backup_source = backup.getSource(self._model.source.name())
+        if backup_source:
+            raise LogicError("This backup already exists in Home Assistant")
+        if not backup_dest:
+            # Unreachable?
+            raise LogicError("This backup isn't in Google Drive")
+        created = await self._model.source.save(backup, await self._model.dest.read(backup))
+        backup.addSource(created)
+        self._updateFreshness()
+
+    async def startBackup(self, options: CreateOptions):
+        return await self._withSoftLock(lambda: self._startBackup(options))
+
+    async def _startBackup(self, options: CreateOptions):
+        self.clearCaches()
+        self._estimator.refresh()
+        self._estimator.checkSpace(self.backups())
+        created = await self._buildModel().source.create(options)
+        backup = Backup(created)
+        self._model.backups[backup.slug()] = backup
+        self._updateFreshness()
+        self._estimator.refresh()
+        return backup
+
+    def getBackup(self, slug):
+        return self._ensureBackup(None, slug)
+
+    async def download(self, slug):
+        self.clearCaches()
+        backup = self._ensureBackup(None, slug)
+        for source in self._sources.values():
+            if not source.enabled():
+                continue
+            if backup.getSource(source.name()):
+                return await source.read(backup)
+        raise NoBackup()
+
+    async def retain(self, sources: Dict[str, bool], slug: str):
+        self.clearCaches()
+        for source in sources:
+            backup = self._ensureBackup(source, slug)
+            await self._ensureSource(source).retain(backup, sources[source])
+        self._updateFreshness()
+
+    async def note(self, note: str, slug: str):
+        self.clearCaches()
+        backup = self._ensureBackup(None, slug)
+        for source in backup.sources.keys():
+            await self._ensureSource(source).note(backup, note)
+
+    async def delete(self, sources, slug):
+        await self._withSoftLock(lambda: self._delete(sources, slug))
+
+    async def ignore(self, slug: str, ignore: bool):
+        await self._withSoftLock(lambda: self._ignore(slug, ignore))
+
+    async def _delete(self, sources, slug):
+        self.clearCaches()
+        for source in sources:
+            backup = self._ensureBackup(source, slug)
+            await self._ensureSource(source).delete(backup)
+            if backup.isDeleted():
+                del self._model.backups[slug]
+        self._updateFreshness()
+
+    async def _ignore(self, slug: str, ignore: bool):
+        self.clearCaches()
+        backup = self._ensureBackup(SOURCE_HA, slug)
+        await self._ensureSource(SOURCE_HA).ignore(backup, ignore)
+
+    def _ensureBackup(self, source: str = None, slug=None) -> Backup:
+        backup = self._buildModel().backups.get(slug)
+        if not backup:
+            raise NoBackup()
+        if not source:
+            return backup
+        if not source:
+            return backup
+        if not backup.getSource(source):
+            raise NoBackup()
+        return backup
+
+    def _ensureSource(self, source):
+        ret = self._sources.get(source)
+        if ret and ret.enabled():
+            return ret
+        raise LogicError()
+
+    def _buildModel(self) -> Model:
+        self._model.reinitialize(self._precache)
+        return self._model
+
+    def _updateFreshness(self):
+        purges = self._buildModel().getNextPurges()
+        for backup in self._model.backups.values():
+            for source in purges:
+                if backup.getSource(source):
+                    backup.updatePurge(source, backup == purges[source])
+
+    def clearCaches(self):
+        if self._precache:
+            self._precache.clear()
+
+    async def _withSoftLock(self, callable):
+        with self._lock:
+            if self._busy:
+                raise PleaseWait()
+            self._busy = True
+        try:
+            return await callable()
+        finally:
+            with self._lock:
+                self._busy = False

hassio-google-drive-backup/backup/model/destinationprecache.py

@@ -0,0 +1,81 @@
+from .coordinator import Coordinator
+from backup.worker import Worker
+from injector import inject, singleton
+from backup.time import Time
+from backup.logger import getLogger
+from backup.config import Config, Setting
+from .model import BackupDestination
+from .precache import Precache
+from random import Random
+from datetime import datetime, timedelta
+from dataclasses import dataclass
+from typing import Any, Dict
+from logging import DEBUG
+
+logger = getLogger(__name__)
+
+
+@dataclass
+class CacheItem:
+    """Class for keeping track of an item in inventory."""
+    valid_until: datetime
+    data: Any
+
+
+@singleton
+class DestinationPrecache(Worker, Precache):
+    @inject
+    def __init__(self, coord: Coordinator, time: Time, dest: BackupDestination, config: Config):
+        super().__init__("Traffic Smoothing Cache", self.checkForSmoothing, time, 60)
+        self._config = config
+        self._coord = coord
+        self._dest = dest
+        self._offset = Random().random()
+        self._cache: Dict[str, CacheItem] = {}
+        self._last_error: datetime = None
+
+    async def checkForSmoothing(self):
+        if self._config.get(Setting.CACHE_WARMUP_MAX_SECONDS) == 0:
+            # disable cache warmup
+            return
+        try:
+            self._coord.setPrecache(self)
+            nextSync = self._coord.nextSyncAttempt()
+            now = self._time.now()
+            if nextSync <= now:
+                # No reason to warm the cache if we should sync right now anyway
+                return
+            if self.cached(self._dest.name(), now):
+                # A value is already cached, so don't do anything
+                return
+            if now >= self.getNextWarmDate():
+                # Warm the cache
+                logger.debug("Preemptively retrieving and caching info from the backup destination to avoid peak demand")
+                data = await self._dest.get()
+                validity = nextSync + timedelta(minutes=1)
+                self._cache[self._dest.name()] = CacheItem(validity, data)
+                self._offset = Random().random()
+        except Exception as e:
+            # Any error should make us avoid precaching for a solid day.
+            logger.debug("Unable to precache data from backup destination")
+            logger.printException(e, level=DEBUG)
+            self._offset = Random().random()
+            if self._config.get(Setting.CACHE_WARMUP_ERROR_TIMEOUT_SECONDS) != 0:
+                self._last_error = self._time.now()
+
+    def getNextWarmDate(self):
+        warm_date = self._coord.nextSyncAttempt() - timedelta(seconds=self._config.get(Setting.CACHE_WARMUP_MAX_SECONDS) * self._offset)
+        if self._last_error:
+            return max(warm_date, self._last_error + timedelta(self._config.get(Setting.CACHE_WARMUP_ERROR_TIMEOUT_SECONDS)))
+        return warm_date
+
+    def cached(self, source: str, date: datetime) -> Any:
+        cached = self._cache.get(source)
+        if cached and cached.valid_until >= date:
+            return cached.data
+        return None
+
+    def clear(self):
+        """Clears any precached data"""
+        self._cache = {}
+        self._offset = Random().random()

hassio-google-drive-backup/backup/model/drivebackup.py

@@ -0,0 +1,74 @@
+from .backups import AbstractBackup
+from typing import Any, Dict
+
+from ..const import SOURCE_GOOGLE_DRIVE, NECESSARY_PROP_KEY_SLUG, NECESSARY_PROP_KEY_DATE, NECESSARY_PROP_KEY_NAME, PROP_NOTE
+from ..exceptions import ensureKey
+from ..config import BoolValidator
+from ..time import Time
+from ..logger import getLogger
+
+logger = getLogger(__name__)
+
+PROP_TYPE = "type"
+PROP_VERSION = "version"
+PROP_PROTECTED = "protected"
+PROP_RETAINED = "retained"
+DRIVE_KEY_TEXT = "Google Drive's backup metadata"
+
+
+class DriveBackup(AbstractBackup):
+
+    """
+    Represents a Home Assistant backup stored on Google Drive
+    """
+
+    def __init__(self, data: Dict[Any, Any]):
+        props = ensureKey('appProperties', data, DRIVE_KEY_TEXT)
+        retained = BoolValidator.strToBool(props.get(PROP_RETAINED, "False"))
+        if NECESSARY_PROP_KEY_NAME in props:
+            backup_name = ensureKey(NECESSARY_PROP_KEY_NAME, props, DRIVE_KEY_TEXT)
+        else:
+            backup_name = data['name'].replace(".tar", "")
+        super().__init__(
+            name=backup_name,
+            slug=ensureKey(NECESSARY_PROP_KEY_SLUG, props, DRIVE_KEY_TEXT),
+            date=Time.parse(
+                ensureKey(NECESSARY_PROP_KEY_DATE, props, DRIVE_KEY_TEXT)),
+            size=int(ensureKey("size", data, DRIVE_KEY_TEXT)),
+            source=SOURCE_GOOGLE_DRIVE,
+            backupType=props.get(PROP_TYPE, "?"),
+            version=props.get(PROP_VERSION, None),
+            protected=BoolValidator.strToBool(props.get(PROP_PROTECTED, "?")),
+            retained=retained,
+            uploadable=False,
+            details=None,
+            note=props.get(PROP_NOTE, None),
+            pending=False)
+        self._drive_data = data
+        self._id = ensureKey('id', data, DRIVE_KEY_TEXT)
+
+    def id(self) -> str:
+        return self._id
+
+    def canDeleteDirectly(self) -> str:
+        caps = self._drive_data.get("capabilities", {})
+        if caps.get('canDelete', False):
+            return True
+
+        # check if the item is in a shared drive
+        sharedId = self._drive_data.get("driveId")
+        if sharedId and len(sharedId) > 0 and caps.get("canTrash", False):
+            # Its in a shared drive and trashable, so trash won't exhaust quota
+            return False
+
+        # We aren't certain we can trash or delete, so just make a try at deleting.
+        return True
+
+    def __str__(self) -> str:
+        return "<Drive: {0} Name: {1} Id: {2}>".format(self.slug(), self.name(), self.id())
+
+    def __format__(self, format_spec: str) -> str:
+        return self.__str__()
+
+    def __repr__(self) -> str:
+        return self.__str__()

hassio-google-drive-backup/backup/model/dummybackup.py

@@ -0,0 +1,26 @@
+from .backups import Backup
+from .dummybackupsource import DummyBackupSource
+from ..logger import getLogger
+
+logger = getLogger(__name__)
+
+
+class DummyBackup(Backup):
+    def __init__(self, name, date, source, slug, size=0, ignore=False, note=None):
+        super().__init__(None)
+        self._size = size
+        self._ignore = ignore
+        self._note = note
+        self.addSource(DummyBackupSource(name, date, source, slug))
+
+    def size(self):
+        return self._size
+
+    def ignore(self):
+        return self._ignore
+
+    def note(self):
+        if self._note is not None:
+            return self._note
+        else:
+            return super().note()

hassio-google-drive-backup/backup/model/dummybackupsource.py

@@ -0,0 +1,20 @@
+from .backups import AbstractBackup
+from ..logger import getLogger
+
+logger = getLogger(__name__)
+
+
+class DummyBackupSource(AbstractBackup):
+    def __init__(self, name, date, source, slug, retain=False):
+        super().__init__(
+            name=name,
+            slug=slug,
+            date=date,
+            size=0,
+            source=source,
+            backupType="dummy",
+            version="dummy_version",
+            protected=True,
+            retained=retain,
+            uploadable=True,
+            details={})

hassio-google-drive-backup/backup/model/habackup.py

@@ -0,0 +1,72 @@
+from typing import Any, Dict
+
+from backup.const import SOURCE_HA
+from backup.exceptions import ensureKey
+from backup.time import Time
+from .backups import AbstractBackup
+from backup.logger import getLogger
+from backup.util import DataCache, KEY_I_MADE_THIS, KEY_IGNORE, KEY_NOTE
+from backup.config import Config, Setting
+
+logger = getLogger(__name__)
+
+HA_KEY_TEXT = "Home Assistant's backup metadata"
+
+
+class HABackup(AbstractBackup):
+    """
+    Represents a Home Assistant backup stored locally in Home Assistant
+    """
+
+    def __init__(self, data: Dict[str, Any], data_cache: DataCache, config: Config, retained=False):
+        super().__init__(
+            name=ensureKey('name', data, HA_KEY_TEXT),
+            slug=ensureKey('slug', data, HA_KEY_TEXT),
+            date=Time.parse(ensureKey('date', data, HA_KEY_TEXT)),
+            size=float(ensureKey("size", data, HA_KEY_TEXT)) * 1024 * 1024,
+            source=SOURCE_HA,
+            backupType=ensureKey('type', data, HA_KEY_TEXT),
+            version=ensureKey('homeassistant', data, HA_KEY_TEXT),
+            protected=ensureKey('protected', data, HA_KEY_TEXT),
+            retained=retained,
+            uploadable=True,
+            details=data,
+            pending=False)
+        self._data_cache = data_cache
+        self._config = config
+
+    def madeByTheAddon(self):
+        return self._data_cache.backup(self.slug()).get(KEY_I_MADE_THIS, False)
+
+    def note(self):
+        parent = super().note()
+        if parent is None:
+            return self._data_cache.backup(self.slug()).get(KEY_NOTE, None)
+        else:
+            return parent
+
+    def ignore(self):
+        override = self._data_cache.backup(self.slug()).get(KEY_IGNORE, None)
+        if override is not None:
+            return override
+        if self.madeByTheAddon():
+            return False
+        if self._config.get(Setting.IGNORE_OTHER_BACKUPS):
+            return True
+        archive_count = len(self.details().get("addons", [])) + len(self.details().get("folders", []))
+        if self.details().get("homeassistant", None) is not None:
+            # Supervisor backup query API doesn't quite match the create API, if the HA config folder
+            # is present in a backup then the Home Assistant version is present in its details
+            archive_count += 1
+        if archive_count == 1 and self._config.get(Setting.IGNORE_UPGRADE_BACKUPS):
+            return True
+        return super().ignore()
+
+    def __str__(self) -> str:
+        return "<HA: {0} Name: {1} {2}>".format(self.slug(), self.name(), self.date().isoformat())
+
+    def __format__(self, format_spec: str) -> str:
+        return self.__str__()
+
+    def __repr__(self) -> str:
+        return self.__str__()

hassio-google-drive-backup/backup/model/model.py

@@ -0,0 +1,397 @@
+from datetime import datetime, timedelta, date
+from io import IOBase
+from typing import Dict, Generic, List, Optional, Tuple, TypeVar
+
+from injector import inject, singleton
+
+from .backupscheme import GenerationalScheme, OldestScheme, DeleteAfterUploadScheme
+from backup.config import Config, Setting, CreateOptions
+from backup.exceptions import DeleteMutlipleBackupsError, SimulatedError
+from backup.util import GlobalInfo, Estimator, DataCache
+from .backups import AbstractBackup, Backup
+from .dummybackup import DummyBackup
+from .precache import Precache
+from backup.time import Time
+from backup.worker import Trigger
+from backup.logger import getLogger
+
+logger = getLogger(__name__)
+
+T = TypeVar('T')
+
+
+class BackupSource(Trigger, Generic[T]):
+    def __init__(self):
+        super().__init__()
+        pass
+
+    def name(self) -> str:
+        return "Unnamed"
+
+    def title(self) -> str:
+        return "Default"
+
+    def enabled(self) -> bool:
+        return True
+
+    def needsConfiguration(self) -> bool:
+        return not self.enabled()
+
+    def upload(self) -> bool:
+        return True
+
+    def icon(self) -> str:
+        return "sd_card"
+
+    def freeSpace(self):
+        return None
+
+    async def create(self, options: CreateOptions) -> T:
+        pass
+
+    async def get(self) -> Dict[str, T]:
+        pass
+
+    async def delete(self, backup: T):
+        pass
+
+    async def ignore(self, backup: T, ignore: bool):
+        pass
+
+    async def save(self, backup: AbstractBackup, bytes: IOBase) -> T:
+        pass
+
+    async def read(self, backup: T) -> IOBase:
+        pass
+
+    async def retain(self, backup: T, retain: bool) -> None:
+        pass
+
+    async def note(self, backup, note: str) -> None:
+        pass
+
+    def maxCount(self) -> None:
+        return 0
+
+    def postSync(self) -> None:
+        return
+
+    def detail(self) -> str:
+        return ""
+
+    def isDestination(self) -> bool:
+        return False
+
+    # Gets called after reading state but before any changes are made
+    # to check for additional errors.
+    def checkBeforeChanges(self) -> None:
+        pass
+
+
+class BackupDestination(BackupSource):
+    def isWorking(self):
+        return False
+
+    @property
+    def might_be_oob_creds(self) -> bool:
+        return False
+
+    def isDestination(self) -> bool:
+        return True
+
+
+@singleton
+class Model():
+    @inject
+    def __init__(self, config: Config, time: Time, source: BackupSource, dest: BackupDestination, info: GlobalInfo, estimator: Estimator, data_cache: DataCache):
+        self.config: Config = config
+        self.time = time
+        self.precache: Precache = None
+        self.source: BackupSource = source
+        self.dest: BackupDestination = dest
+        self.reinitialize()
+        self.backups: Dict[str, Backup] = {}
+        self.firstSync = True
+        self.info = info
+        self.simulate_error = None
+        self.estimator = estimator
+        self.waiting_for_startup = False
+        self.ignore_startup_delay = False
+        self._data_cache = data_cache
+
+    def enabled(self):
+        if self.source.needsConfiguration():
+            return False
+        if self.dest.needsConfiguration():
+            return False
+        return True
+
+    def allSources(self):
+        return [self.source, self.dest]
+
+    def reinitialize(self, precache: Precache = None):
+        self.precache = precache
+        self._time_of_day: Optional[Tuple[int, int]] = self._parseTimeOfDay()
+
+        # SOMEDAY: this should be cached in config and regenerated on config updates, not here
+        self.generational_config = self.config.getGenerationalConfig()
+
+    def getTimeOfDay(self):
+        return self._time_of_day
+
+    def _nextBackup(self, now: datetime, last_backup: Optional[datetime]) -> Optional[datetime]:
+        timeofDay = self.getTimeOfDay()
+        if self.config.get(Setting.DAYS_BETWEEN_BACKUPS) <= 0:
+            next = None
+        elif self.dest.needsConfiguration():
+            next = None
+        elif not last_backup:
+            # this isn't the cleanest logic, but the idea here is that if there are no backups,
+            # then the backups shoudl be made right when the addon starts up.
+            next = self.info.start_time
+        elif not timeofDay:
+            next = last_backup + timedelta(days=self.config.get(Setting.DAYS_BETWEEN_BACKUPS))
+        else:
+            newest_local: datetime = self.time.toLocal(last_backup)
+            time_that_day_local = self.time.localize(datetime(newest_local.year, newest_local.month, newest_local.day, timeofDay[0], timeofDay[1]))
+            if newest_local < time_that_day_local:
+                # Latest backup is before the backup time for that day
+                next = self.time.toUtc(time_that_day_local)
+            else:
+                # return the next backup after the delta
+                next_date = date.fromordinal(int(date(newest_local.year, newest_local.month, newest_local.day).toordinal() + self.config.get(Setting.DAYS_BETWEEN_BACKUPS)))
+                next_datetime_local = self.time.localize(datetime(next_date.year, next_date.month, next_date.day, timeofDay[0], timeofDay[1]))
+                next = self.time.toUtc(next_datetime_local)
+
+        if next is None:
+            self.waiting_for_startup = False
+            return None
+
+        # Don't backup X minutes after startup, since that can put an unreasonable amount of strain on
+        # the system while booting up.
+        cooldown_minimum = self.info.backupCooldownTime()
+        if next <= now and now < cooldown_minimum and not self.ignore_startup_delay:
+            self.waiting_for_startup = True
+            return cooldown_minimum
+        elif self.ignore_startup_delay:
+            self.waiting_for_startup = False
+            return next
+        elif cooldown_minimum > next:
+            self.waiting_for_startup = cooldown_minimum > now
+            return cooldown_minimum
+        else:
+            self.waiting_for_startup = False
+            return next
+
+    def nextBackup(self, now: datetime, include_pending=True):
+        latest = max(filter(lambda s: not s.ignore() and (not s.isPending() or include_pending), self.backups.values()),
+                     default=None, key=lambda s: s.date())
+        if latest:
+            latest = latest.date()
+        return self._nextBackup(now, latest)
+
+    async def sync(self, now: datetime):
+        if self.simulate_error is not None:
+            if self.simulate_error.startswith("test"):
+                raise Exception(self.simulate_error)
+            else:
+                raise SimulatedError(self.simulate_error)
+        await self._syncBackups([self.source, self.dest], now)
+
+        self.source.checkBeforeChanges()
+        self.dest.checkBeforeChanges()
+
+        if not self.dest.needsConfiguration():
+            if self.source.enabled():
+                await self._purge(self.source)
+            if self.dest.enabled():
+                await self._purge(self.dest)
+
+        # Delete any "ignored" backups that have expired
+        if (self.config.get(Setting.IGNORE_OTHER_BACKUPS) or self.config.get(Setting.IGNORE_UPGRADE_BACKUPS)) and self.config.get(Setting.DELETE_IGNORED_AFTER_DAYS) > 0:
+            cutoff = now - timedelta(days=self.config.get(Setting.DELETE_IGNORED_AFTER_DAYS))
+            delete = []
+            for backup in self.backups.values():
+                if backup.ignore() and backup.date() < cutoff:
+                    delete.append(backup)
+            for backup in delete:
+                await self.deleteBackup(backup, self.source)
+
+        self._handleBackupDetails()
+        next_backup = self.nextBackup(now)
+        if next_backup and now >= next_backup and self.source.enabled() and not self.dest.needsConfiguration():
+            if self.config.get(Setting.DELETE_BEFORE_NEW_BACKUP):
+                await self._purge(self.source, pre_purge=True)
+            await self.createBackup(CreateOptions(now, self.config.get(Setting.BACKUP_NAME)))
+            await self._purge(self.source)
+            self._handleBackupDetails()
+
+        if self.dest.enabled() and self.dest.upload():
+            # get the backups we should upload
+            uploads = []
+            for backup in self.backups.values():
+                if backup.getSource(self.source.name()) is not None and backup.getSource(self.source.name()).uploadable() and backup.getSource(self.dest.name()) is None and not backup.ignore():
+                    uploads.append(backup)
+            uploads.sort(key=lambda s: s.date())
+            uploads.reverse()
+            for upload in uploads:
+                # only upload if doing so won't result in it being deleted next
+                dummy = DummyBackup(
+                    "", upload.date(), self.dest.name(), "dummy_slug_name")
+                proposed = list(self.backups.values())
+                proposed.append(dummy)
+                if self._nextPurge(self.dest, proposed)[1] != dummy:
+                    if self.config.get(Setting.DELETE_BEFORE_NEW_BACKUP):
+                        await self._purge(self.dest, pre_purge=True)
+                    upload.addSource(await self.dest.save(upload, await self.source.read(upload)))
+                    await self._purge(self.dest)
+                    self._handleBackupDetails()
+                else:
+                    break
+            if self.config.get(Setting.DELETE_AFTER_UPLOAD):
+                await self._purge(self.source)
+        self._handleBackupDetails()
+        self.source.postSync()
+        self.dest.postSync()
+        self._data_cache.saveIfDirty()
+
+    def isWorkingThroughUpload(self):
+        return self.dest.isWorking()
+
+    async def createBackup(self, options):
+        if not self.source.enabled():
+            return
+
+        self.estimator.refresh()
+        self.estimator.checkSpace(list(self.backups.values()))
+        created = await self.source.create(options)
+        backup = Backup(created)
+        self.backups[backup.slug()] = backup
+
+    async def deleteBackup(self, backup, source):
+        if not backup.getSource(source.name()):
+            return
+        slug = backup.slug()
+        await source.delete(backup)
+        backup.removeSource(source.name())
+        if backup.isDeleted():
+            del self.backups[slug]
+
+    def getNextPurges(self):
+        purges = {}
+        for source in [self.source, self.dest]:
+            purges[source.name()] = self._nextPurge(
+                source, self.backups.values(), findNext=True)[1]
+        return purges
+
+    def _parseTimeOfDay(self) -> Optional[Tuple[int, int]]:
+        from_config = self.config.get(Setting.BACKUP_TIME_OF_DAY)
+        if len(from_config) == 0:
+            return None
+        parts = from_config.split(":")
+        if len(parts) != 2:
+            return None
+        try:
+            hour: int = int(parts[0])
+            minute: int = int(parts[1])
+            if hour < 0 or minute < 0 or hour > 23 or minute > 59:
+                return None
+            return (hour, minute)
+        except ValueError:
+            # Parse error
+            return None
+
+    async def _syncBackups(self, sources: List[BackupSource], now: datetime):
+        for source in sources:
+            if source.enabled():
+                # check if we have the results from this source precached
+                from_source: Dict[str, AbstractBackup] = None
+                if self.precache is not None:
+                    from_source = self.precache.cached(source.name(), now)
+                if not from_source:
+                    from_source = await source.get()
+            else:
+                from_source: Dict[str, AbstractBackup] = {}
+            for backup in from_source.values():
+                if backup.slug() not in self.backups:
+                    self.backups[backup.slug()] = Backup(backup)
+                else:
+                    self.backups[backup.slug()].addSource(backup)
+            for backup in list(self.backups.values()):
+                if backup.slug() not in from_source:
+                    slug = backup.slug()
+                    backup.removeSource(source.name())
+                    if backup.isDeleted():
+                        del self.backups[slug]
+        self.firstSync = False
+
+    def _buildDeleteScheme(self, source, findNext=False):
+        count = source.maxCount()
+        if findNext:
+            count -= 1
+        if source == self.source and self.config.get(Setting.DELETE_AFTER_UPLOAD):
+            return DeleteAfterUploadScheme(source.name(), [self.dest.name()])
+        elif self.generational_config:
+            return GenerationalScheme(
+                self.time, self.generational_config, count=count)
+        else:
+            return OldestScheme(count=count)
+
+    def _buildNamingScheme(self):
+        source = max(filter(BackupSource.enabled, self.allSources()), key=BackupSource.maxCount)
+        return self._buildDeleteScheme(source)
+
+    def _handleBackupDetails(self):
+        self._buildNamingScheme().handleNaming(self.backups.values())
+
+    def _nextPurge(self, source: BackupSource, backups, findNext=False):
+        """
+        Given a list of backups, decides if one should be purged.
+        """
+        if not source.enabled() or len(backups) == 0:
+            return None, None
+        if source.maxCount() == 0 and source.isDestination():
+            # When maxCount is zero for a destination, we should never delete from it.
+            return None, None
+        if source.maxCount() == 0 and not self.config.get(Setting.DELETE_AFTER_UPLOAD):
+            return None, None
+
+        scheme = self._buildDeleteScheme(source, findNext=findNext)
+        consider_purging = []
+        for backup in backups:
+            source_backup = backup.getSource(source.name())
+            if source_backup is not None and source_backup.considerForPurge() and not backup.ignore():
+                consider_purging.append(backup)
+        if len(consider_purging) == 0:
+            return None, None
+        return scheme.getOldest(consider_purging)
+
+    async def _purge(self, source: BackupSource, pre_purge=False):
+        while True:
+            purge = self._getPurgeList(source, pre_purge)
+
+            reasons = set(map(lambda p: p[1], purge))
+            if len(purge) <= 0:
+                return
+            if len(purge) != len(reasons) and (self.config.get(Setting.CONFIRM_MULTIPLE_DELETES) and not self.info.isPermitMultipleDeletes()):
+                raise DeleteMutlipleBackupsError(self._getPurgeStats())
+            await self.deleteBackup(purge[0][0], source)
+
+    def _getPurgeStats(self):
+        ret = {}
+        for source in [self.source, self.dest]:
+            ret[source.name()] = len(self._getPurgeList(source))
+        return ret
+
+    def _getPurgeList(self, source: BackupSource, pre_purge=False):
+        if not source.enabled():
+            return []
+        candidates = list(self.backups.values())
+        purges = []
+        while True:
+            reason, next_purge = self._nextPurge(source, candidates, findNext=pre_purge)
+            if next_purge is None:
+                return purges
+            else:
+                purges.append((next_purge, reason))
+                candidates.remove(next_purge)

hassio-google-drive-backup/backup/model/precache.py

@@ -0,0 +1,15 @@
+from abc import ABC, abstractmethod
+from typing import Any
+from datetime import datetime
+
+
+class Precache(ABC):
+    @abstractmethod
+    def cached(self, source: str, date: datetime) -> Any:
+        """For a given source and datetime, returns valid precached results if they're available"""
+        pass
+
+    @abstractmethod
+    def clear(self):
+        """Clears any cached results stored"""
+        pass

hassio-google-drive-backup/backup/model/simulatedsource.py

@@ -0,0 +1,129 @@
+from .model import CreateOptions, BackupDestination
+from .backups import Backup
+from .dummybackupsource import DummyBackupSource
+from typing import Dict
+from io import IOBase
+from ..ha import BackupName
+from ..logger import getLogger
+
+logger = getLogger(__name__)
+
+
+class SimulatedSource(BackupDestination):
+    def __init__(self, name, is_destination=False):
+        self._name = name
+        self.current: Dict[str, DummyBackupSource] = {}
+        self.saved = []
+        self.deleted = []
+        self.created = []
+        self._enabled = True
+        self._upload = True
+        self.index = 0
+        self.max = 0
+        self.backup_name = BackupName()
+        self.host_info = {}
+        self.backup_type = "Full"
+        self.working = False
+        self.needConfig = None
+        self.is_destination = is_destination
+
+    def isDestination(self):
+        return self.is_destination
+
+    def setEnabled(self, value):
+        self._enabled = value
+        return self
+
+    def needsConfiguration(self) -> bool:
+        if self.needConfig is not None:
+            return self.needConfig
+        return super().needsConfiguration()
+
+    def setNeedsConfiguration(self, value: bool):
+        self.needConfig = value
+
+    def setUpload(self, value):
+        self._upload = value
+        return self
+
+    def upload(self):
+        return self._upload
+
+    def setMax(self, count):
+        self.max = count
+        return self
+
+    def isWorking(self):
+        return self.working
+
+    def setIsWorking(self, value):
+        self.working = value
+
+    def maxCount(self) -> None:
+        return self.max
+
+    def insert(self, name, date, slug=None, retain=False):
+        if slug is None:
+            slug = name
+        new_backup = DummyBackupSource(
+            name,
+            date,
+            self._name,
+            slug)
+        self.current[new_backup.slug()] = new_backup
+        return new_backup
+
+    def name(self) -> str:
+        return self._name
+
+    def enabled(self) -> bool:
+        return self._enabled
+
+    def nameSetup(self, type, host_info):
+        self.backup_type = type
+        self.host_info = host_info
+
+    async def create(self, options: CreateOptions) -> DummyBackupSource:
+        assert self.enabled
+        new_backup = DummyBackupSource(
+            self.backup_name.resolve(
+                self.backup_type, options.name_template, options.when, self.host_info),
+            options.when,
+            self._name,
+            "{0}slug{1}".format(self._name, self.index))
+        self.index += 1
+        self.current[new_backup.slug()] = new_backup
+        self.created.append(new_backup)
+        return new_backup
+
+    async def get(self) -> Dict[str, DummyBackupSource]:
+        assert self.enabled
+        return self.current
+
+    async def delete(self, backup: Backup):
+        assert self.enabled
+        assert backup.getSource(self._name) is not None
+        assert backup.getSource(self._name).source() is self._name
+        assert backup.slug() in self.current
+        slug = backup.slug()
+        self.deleted.append(backup.getSource(self._name))
+        backup.removeSource(self._name)
+        del self.current[slug]
+
+    async def save(self, backup: Backup, bytes: IOBase = None) -> DummyBackupSource:
+        assert self.enabled
+        assert backup.slug() not in self.current
+        new_backup = DummyBackupSource(
+            backup.name(), backup.date(), self._name, backup.slug())
+        backup.addSource(new_backup)
+        self.current[new_backup.slug()] = new_backup
+        self.saved.append(new_backup)
+        return new_backup
+
+    async def read(self, backup: DummyBackupSource) -> IOBase:
+        assert self.enabled
+        return None
+
+    async def retain(self, backup: DummyBackupSource, retain: bool) -> None:
+        assert self.enabled
+        backup.getSource(self.name()).setRetained(retain)

hassio-google-drive-backup/backup/model/syncer.py

@@ -0,0 +1,36 @@
+from typing import List
+
+from injector import inject, singleton
+
+from .coordinator import Coordinator
+from backup.time import Time
+from backup.worker import Worker, Trigger
+from backup.logger import getLogger
+from backup.exceptions import PleaseWait
+
+logger = getLogger(__name__)
+
+
+@singleton
+class Scyncer(Worker):
+    @inject
+    def __init__(self, time: Time, coord: Coordinator, triggers: List[Trigger]):
+        super().__init__("Sync Worker", self.checkforSync, time, 0.5)
+        self.coord = coord
+        self.triggers: List[Trigger] = triggers
+        self._time = time
+
+    async def checkforSync(self):
+        try:
+            doSync = False
+            for trigger in self.triggers:
+                if await trigger.check():
+                    logger.debug("Sync requested by " + str(trigger.name()))
+                    doSync = True
+            if doSync:
+                while self.coord.isSyncing():
+                    await self._time.sleepAsync(3)
+                await self.coord.sync()
+        except PleaseWait:
+            # Ignore this, since it means a sync already started (unavilable race condition)
+            pass