New Addon

Google BK

hassio-google-drive-backup/backup/drive/__init__.py

@@ -0,0 +1,5 @@
+# flake8: noqa
+from .driverequests import DriveRequests, RETRY_SESSION_ATTEMPTS, UPLOAD_SESSION_EXPIRATION_DURATION, URL_START_UPLOAD, OOB_CRED_CUTOFF
+from .drivesource import DriveSource, SOURCE_GOOGLE_DRIVE
+from .folderfinder import FolderFinder
+from .authcodequery import AuthCodeQuery

hassio-google-drive-backup/backup/drive/authcodequery.py

@@ -0,0 +1,107 @@
+from datetime import datetime, timedelta
+
+from backup.config import Config, Setting
+from backup.time import Time
+from backup.exceptions import GoogleCredGenerateError, KnownError, LogicError, ensureKey
+from aiohttp import ClientSession
+from injector import inject
+from .driverequests import DriveRequester
+from backup.logger import getLogger
+from backup.creds import Creds
+import asyncio
+
+logger = getLogger(__name__)
+SCOPE = 'https://www.googleapis.com/auth/drive.file'
+
+
+class AuthCodeQuery:
+    @inject
+    def __init__(self, config: Config, session: ClientSession, time: Time, drive: DriveRequester):
+        self.session = session
+        self.config = config
+        self.drive = drive
+        self.time = time
+        self.client_id: str = None
+        self.client_secret: str = None
+        self.device_code: str = None
+        self.verification_url: str = None
+        self.user_code: str = None
+        self.check_interval: timedelta = timedelta(seconds=5)
+        self.expiration: datetime = time.now()
+        self.last_check = time.now()
+
+    async def requestCredentials(self, client_id: str, client_secret: str):
+        self.client_id = client_id
+        self.client_secret = client_secret
+        request_data = {
+            'client_id': self.client_id,
+            'scope': SCOPE
+        }
+        resp = await self.session.post(self.config.get(Setting.DRIVE_DEVICE_CODE_URL), data=request_data, timeout=30)
+        if resp.status != 200:
+            raise GoogleCredGenerateError(f"Google responded with error status HTTP {resp.status}.  Please verify your credentials are set up correctly.")
+        data = await resp.json()
+        self.device_code = str(ensureKey("device_code", data, "Google's authorization request"))
+        self.verification_url = str(ensureKey("verification_url", data, "Google's authorization request"))
+        self.user_code = str(ensureKey("user_code", data, "Google's authorization request"))
+        self.expiration = self.time.now() + timedelta(seconds=int(ensureKey("expires_in", data, "Google's authorization request")))
+        self.check_interval = timedelta(seconds=int(ensureKey("interval", data, "Google's authorization request")))
+
+    async def waitForPermission(self) -> Creds:
+        if not self.device_code:
+            raise LogicError("Please call requestCredentials() first")
+        error_count = 0
+        data = {
+            'client_id': self.client_id,
+            'client_secret': self.client_secret,
+            'device_code': self.device_code,
+            'grant_type': 'urn:ietf:params:oauth:grant-type:device_code'
+        }
+        while self.expiration > self.time.now():
+            start = self.time.now()
+            resp = None
+            try:
+                resp = await self.session.post(self.config.get(Setting.DRIVE_TOKEN_URL), data=data, timeout=self.check_interval.total_seconds())
+                try:
+                    reply = await resp.json()
+                except Exception:
+                    reply = {}
+                if resp.status == 403:
+                    if reply.get("error", "") == "slow_down":
+                        # google wants us to chill out, so do that
+                        await asyncio.sleep(self.check_interval.total_seconds())
+                    else:
+                        # Google says no
+                        logger.error(f"Getting credentials from Google failed with HTTP 403 and error: {reply.get('error', 'unspecified')}")
+                        raise GoogleCredGenerateError("Google refused the request to connect your account, either because you rejected it or they were set up incorrectly.")
+                elif resp.status == 428:
+                    # Google says PEBKAC
+                    logger.info(f"Waiting for you to authenticate with Google at {self.verification_url}")
+                elif resp.status / 100 != 2:
+                    # Mysterious error
+                    logger.error(f"Getting credentials from Google failed with HTTP {resp.status} and error: {reply.get('error', 'unspecified')}")
+                    raise GoogleCredGenerateError("Failed unexpectedly while trying to reach Google.  See the add-on logs for details.")
+                else:
+                    # got the token, return it
+                    return Creds.load(self.time, reply, id=self.client_id, secret=self.client_secret)
+            except KnownError:
+                raise
+            except Exception as e:
+                logger.error("Error while trying to retrieve credentials from Google")
+                logger.printException(e)
+
+                # Allowing 10 errors is arbitrary, but prevents us from just erroring out forever in the background
+                error_count += 1
+                if error_count > 10:
+                    raise GoogleCredGenerateError("Failed unexpectedly too many times while attempting to reach Google.  See the logs for details.")
+            finally:
+                if resp is not None:
+                    resp.release()
+
+            # Make sure we never query more than google says we should
+            remainder = self.check_interval - (self.time.now() - start)
+            if remainder > timedelta(seconds=0):
+                await asyncio.sleep(remainder.total_seconds())
+
+        logger.error("Getting credentials from Google expired, please try again")
+        raise GoogleCredGenerateError("Credentials expired while waiting for you to authorize with Google")

hassio-google-drive-backup/backup/drive/driverequests.py

@@ -0,0 +1,373 @@
+import io
+import math
+import re
+from typing import Any, Dict, Optional
+from urllib.parse import urlencode
+from datetime import datetime, timedelta
+
+from aiohttp import ClientSession, ClientTimeout, ClientResponse
+from aiohttp.client_exceptions import ClientResponseError, ServerTimeoutError
+from injector import inject, singleton
+
+from ..util import AsyncHttpGetter
+from ..config import Config, Setting
+from ..exceptions import (GoogleCredentialsExpired,
+                          GoogleSessionError, LogicError,
+                          ProtocolError, ensureKey, KnownTransient, GoogleTimeoutError, GoogleUnexpectedError)
+from backup.util import Backoff
+from backup.file import JsonFileSaver
+from ..time import Time
+from ..logger import getLogger
+from backup.creds import Creds, Exchanger, DriveRequester
+from datetime import timezone
+
+logger = getLogger(__name__)
+
+MIME_TYPE = "application/tar"
+FOLDER_MIME_TYPE = 'application/vnd.google-apps.folder'
+FOLDER_NAME = 'Home Assistant Backups'
+DRIVE_VERSION = "v3"
+DRIVE_SERVICE = "drive"
+
+SELECT_FIELDS = "id,name,appProperties,size,trashed,mimeType,modifiedTime,capabilities,parents,driveId"
+THUMBNAIL_MIME_TYPE = "image/png"
+QUERY_FIELDS = "nextPageToken,files(" + SELECT_FIELDS + ")"
+CREATE_FIELDS = SELECT_FIELDS
+URL_FILES = "/drive/v3/files/"
+URL_ABOUT = "/drive/v3/about"
+URL_START_UPLOAD = "/upload/drive/v3/files/?uploadType=resumable&supportsAllDrives=true"
+PAGE_SIZE = 100
+CHUNK_SIZE = 5 * 262144
+RANGE_RE = re.compile("^bytes=0-\\d+$")
+
+BASE_CHUNK_SIZE = 256 * 1024  # Google's api requires uploading chunks in multiples of 256kb
+
+# During upload, chunks get sized to complete upload after 10s so we can give status updates on progress.
+CHUNK_UPLOAD_TARGET_SECONDS = 10
+
+# don't attempt to resume a session with than this many times consistant failures, just in case something is broken on Google's
+# end so we don't retry the same broken session forever.  Because the addon eventually backs off to doing 1 attempt/hour, this will
+# cause uploads to fail and start over after about 4 days.  This gets reset every time a chunk successfully uploads.
+# God be with you if your upload takes that long.
+RETRY_SESSION_ATTEMPTS = 100
+
+# Google claims that an upload session becomes invalid after 7 days.  I have not verified this, but probably better to call it
+# after 6 and restart the session.
+UPLOAD_SESSION_EXPIRATION_DURATION = timedelta(days=6)
+
+
+RATE_LIMIT_EXCEEDED = 403
+TOO_MANY_REQUESTS = 429
+
+
+# Defines the retry strategy for calls made to Drive
+# max # of time to retry and call to Drive
+DRIVE_MAX_RETRIES: int = 5
+# The initial backoff for drive retries.
+DRIVE_RETRY_INITIAL_SECONDS: int = 2
+# How uch longer to wait for each Drive service call (Exponential backoff)
+DRIVE_EXPONENTIAL_BACKOFF: int = 2
+
+OOB_CRED_CUTOFF = datetime(2022, 3, 16, tzinfo=timezone.utc)
+
+
+@singleton
+class DriveRequests():
+    @inject
+    def __init__(self, config: Config, time: Time, drive: DriveRequester, session: ClientSession, exchanger: Exchanger):
+        self.session = session
+        self.config = config
+        self.time = time
+        self.drive = drive
+        self.creds: Optional[Creds] = None
+        self.exchanger: Exchanger = exchanger
+
+        # Between attempts to upload, we keep track of the info needed to resume a resumable upload.
+        self.last_attempt_metadata = None
+        self.last_attempt_location = None
+        self.last_attempt_count = 0
+        self.last_attempt_start_time = None
+        self.tryLoadCredentials()
+
+    async def _getHeaders(self):
+        return {
+            "Authorization": "Bearer " + await self.getToken(),
+            "Client-Identifier": self.config.clientIdentifier()
+        }
+
+    @property
+    def might_be_oob_creds(self):
+        """Attempts to determine if the user might be using custom creds affected by google's OOB cred deprecation"""
+        if not self.isCustomCreds():
+            return False
+        if self.creds.original_expiration is None:
+            # These creds must be old, so assume they're affected
+            return True
+        try:
+            return self.creds.original_expiration < OOB_CRED_CUTOFF
+        except:  # noqa: E722
+            # Regardless of why this happens, assume they need to check
+            return True
+
+    def isCustomCreds(self):
+        return self.creds is not None and self.creds.id != self.config.get(Setting.DEFAULT_DRIVE_CLIENT_ID)
+
+    def _getAuthHeaders(self):
+        return {
+            "Client-Identifier": self.config.clientIdentifier()
+        }
+
+    def enabled(self):
+        return self.creds is not None and self.config.get(Setting.ENABLE_DRIVE_UPLOAD)
+
+    def _enabledCheck(self):
+        if not self.enabled():
+            raise LogicError(
+                "Attempt to use Google Drive before credentials are configured")
+
+    def tryLoadCredentials(self):
+        path = self.config.get(Setting.CREDENTIALS_FILE_PATH)
+        if JsonFileSaver.exists(path):
+            try:
+                self.creds = Creds.load(self.time, JsonFileSaver.read(path))
+            except Exception:
+                pass
+
+    def saveCredentials(self, creds: Creds):
+        path = self.config.get(Setting.CREDENTIALS_FILE_PATH)
+        if not creds:
+            if JsonFileSaver.exists(path):
+                JsonFileSaver.delete(path)
+                self.creds = None
+            return
+        JsonFileSaver.write(path, creds.serialize())
+        self.tryLoadCredentials()
+
+    async def getToken(self, refresh=False):
+        if self.creds and not self.creds.is_expired and not refresh:
+            return self.creds.access_token
+
+        # refresh the credentials
+        logger.debug("Requesting refreshed Google Drive credentials")
+        self.creds = await self.exchanger.refresh(self.creds)
+        return self.creds.access_token
+
+    async def refreshToken(self):
+        await self.getToken(refresh=True)
+
+    async def get(self, id):
+        q = {
+            "fields": SELECT_FIELDS,
+            "supportsAllDrives": "true"
+        }
+        async with await self.retryRequest("GET", URL_FILES + id + "/?" + urlencode(q)) as response:
+            return await response.json()
+
+    async def download(self, id, size):
+        ret = AsyncHttpGetter(self.config.get(Setting.DRIVE_URL) + URL_FILES + id + "/?alt=media&supportsAllDrives=true",
+                              await self._getHeaders(),
+                              self.session,
+                              size=size,
+                              timeoutFactory=GoogleTimeoutError.factory,
+                              otherErrorFactory=GoogleUnexpectedError.factory,
+                              timeout=ClientTimeout(
+                                  sock_connect=self.config.get(Setting.DOWNLOAD_TIMEOUT_SECONDS),
+                                  sock_read=self.config.get(Setting.DOWNLOAD_TIMEOUT_SECONDS)),
+                              time=self.time)
+        return ret
+
+    async def query(self, query):
+        # SOMEDAY: Add a test for page size, test server support is needed too for continuation tokens
+        continuation = None
+        while True:
+            q = {
+                "q": query,
+                "fields": QUERY_FIELDS,
+                "pageSize": self.config.get(Setting.GOOGLE_DRIVE_PAGE_SIZE),
+                "supportsAllDrives": "true",
+                "includeItemsFromAllDrives": "true",
+                "corpora": "allDrives"
+            }
+            if continuation:
+                q["pageToken"] = continuation
+            async with await self.retryRequest("GET", URL_FILES + "?" + urlencode(q)) as response:
+                data = await response.json()
+                for item in data['files']:
+                    yield item
+                if "nextPageToken" not in data or len(data['nextPageToken']) <= 0:
+                    break
+                else:
+                    continuation = data['nextPageToken']
+
+    async def update(self, id, update_metadata):
+        async with await self.retryRequest("PATCH", URL_FILES + id + "/?supportsAllDrives=true", json=update_metadata):
+            pass
+
+    async def delete(self, id):
+        async with await self.retryRequest("DELETE", URL_FILES + id + "/?supportsAllDrives=true"):
+            pass
+
+    async def getAboutInfo(self):
+        q = {"fields": 'storageQuota,user'}
+        async with await self.retryRequest("GET", URL_ABOUT + "?" + urlencode(q)) as resp:
+            return await resp.json()
+
+    async def create(self, stream, metadata, mime_type):
+        # Upload logic is complicated. See https://developers.google.com/drive/api/v3/manage-uploads#resumable
+        total_size = stream.size()
+        location = None
+        if metadata == self.last_attempt_metadata and self.last_attempt_location is not None and self.last_attempt_count < RETRY_SESSION_ATTEMPTS and self.time.now() < self.last_attempt_start_time + UPLOAD_SESSION_EXPIRATION_DURATION:
+            logger.debug(
+                "Attempting to resume a previously failed upload where we left off")
+            self.last_attempt_count += 1
+            # Attempt to resume from a partially completed upload.
+            headers = {
+                "Content-Length": "0",
+                "Content-Range": "bytes */{0}".format(total_size)
+            }
+            try:
+                async with await self.retryRequest("PUT", self.last_attempt_location, headers=headers, patch_url=False) as initial:
+                    if initial.status == 308:
+                        # We can resume the upload, check where it left off
+                        if 'Range' in initial.headers:
+                            position = int(initial.headers["Range"][len("bytes=0-"):])
+                            stream.position(position + 1)
+                        else:
+                            # No range header in the response means no bytes have been uploaded yet.
+                            stream.position(0)
+                        logger.debug("Resuming upload at byte {0} of {1}".format(
+                            stream.position(), total_size))
+                        location = self.last_attempt_location
+                    else:
+                        logger.debug("Drive returned status code {0}, so we'll have to start the upload over again.".format(
+                            initial.status))
+            except ClientResponseError as e:
+                if e.status == 410:
+                    # Drive doesn't recognize the resume token, so we'll just have to start over.
+                    logger.debug("Drive upload session wasn't recognized, restarting upload from the beginning.")
+                    location = None
+                else:
+                    raise
+
+        if location is None:
+            # There is no session resume, so start a new one.
+            logger.debug("Starting a new upload session with Google Drive")
+            headers = {
+                "X-Upload-Content-Type": mime_type,
+                "X-Upload-Content-Length": str(total_size),
+            }
+            async with await self.retryRequest("POST", URL_START_UPLOAD, headers=headers, json=metadata) as initial:
+                # Google returns a url in the header "Location", which is where subsequent requests to upload
+                # the backup's bytes should be sent.  Logic below handles uploading the file bytes in chunks.
+                location = ensureKey(
+                    'Location', initial.headers, "Google Drive's Upload headers")
+                self.last_attempt_count = 0
+                stream.position(0)
+
+        # Keep track of the location in case the upload fails and we want to resume where we left off.
+        # "metadata" is a durable fingerprint that uniquely identifies a backup, so we can use it to identify a
+        # resumable partial upload in future retrys.
+        self.last_attempt_location = location
+        self.last_attempt_metadata = metadata
+        self.last_attempt_start_time = self.time.now()
+
+        # Always start with the minimum chunk size and work up from there in case the last attempt
+        # failed due to connectivity errors or ... whatever.
+        current_chunk_size = BASE_CHUNK_SIZE
+        while True:
+            start = stream.position()
+            data = await stream.read(current_chunk_size)
+            chunk_size = len(data.getbuffer())
+            if chunk_size == 0:
+                raise LogicError(
+                    "Backup file stream ended prematurely while uploading to Google Drive")
+            headers = {
+                "Content-Length": str(chunk_size),
+                "Content-Range": "bytes {0}-{1}/{2}".format(start, start + chunk_size - 1, total_size)
+            }
+            startTime = self.time.now()
+            logger.debug("Sending {0} bytes to Google Drive".format(current_chunk_size))
+            try:
+                async with await self.retryRequest("PUT", location, headers=headers, data=data, patch_url=False) as partial:
+                    # Base the next chunk size on how long it took to send the last chunk.
+                    current_chunk_size = self._getNextChunkSize(
+                        current_chunk_size, (self.time.now() - startTime).total_seconds())
+
+                    # any time a chunk gets uploaded, reset the retry counter.  This lets very flaky connections
+                    # complete eventually after enough retrying.
+                    self.last_attempt_count = 1
+                    yield float(start + chunk_size) / float(total_size)
+                    if partial.status == 200 or partial.status == 201:
+                        # Upload completed, return the object json
+                        self.last_attempt_location = None
+                        self.last_attempt_metadata = None
+                        yield await self.get((await partial.json())['id'])
+                        break
+                    elif partial.status == 308:
+                        # Upload partially complete, seek to the new requested position
+                        range_bytes = ensureKey(
+                            "Range", partial.headers, "Google Drive's upload response headers")
+                        if not RANGE_RE.match(range_bytes):
+                            raise ProtocolError(
+                                "Range", partial.headers, "Google Drive's upload response headers")
+                        position = int(partial.headers["Range"][len("bytes=0-"):])
+                        stream.position(position + 1)
+                    else:
+                        partial.raise_for_status()
+            except ClientResponseError as e:
+                if math.floor(e.status / 100) == 4:
+                    # clear the cached session location URI, since a 4XX error
+                    # always means the upload session is no good anymore (AFAIK)
+                    self.last_attempt_location = None
+                    self.last_attempt_metadata = None
+
+                if e.status == 404:
+                    raise GoogleSessionError()
+                else:
+                    raise e
+
+    def _getNextChunkSize(self, last_chunk_size, last_chunk_seconds):
+        max = BASE_CHUNK_SIZE * math.floor(self.config.get(Setting.MAXIMUM_UPLOAD_CHUNK_BYTES) / BASE_CHUNK_SIZE)
+        if max < BASE_CHUNK_SIZE:
+            max = BASE_CHUNK_SIZE
+        if last_chunk_seconds <= 0:
+            return max
+        next_chunk = CHUNK_UPLOAD_TARGET_SECONDS * last_chunk_size / last_chunk_seconds
+        if next_chunk >= max:
+            return max
+        if next_chunk < BASE_CHUNK_SIZE:
+            return BASE_CHUNK_SIZE
+        return math.floor(next_chunk / BASE_CHUNK_SIZE) * BASE_CHUNK_SIZE
+
+    async def createFolder(self, metadata):
+        async with await self.retryRequest("POST", URL_FILES + "?supportsAllDrives=true", json=metadata) as resp:
+            return await resp.json()
+
+    async def retryRequest(self, method, url, auth_headers: Optional[Dict[str, str]] = None, headers: Optional[Dict[str, str]] = None, json: Optional[Dict[str, Any]] = None, data: Any = None, cred_retry: bool = True, patch_url: bool = True) -> ClientResponse:
+        backoff = Backoff(base=DRIVE_RETRY_INITIAL_SECONDS, attempts=DRIVE_MAX_RETRIES)
+        if patch_url:
+            url = self.config.get(Setting.DRIVE_URL) + url
+        while True:
+            headers_to_use = await self._getHeaders()
+            if headers:
+                headers_to_use.update(headers)
+            if self.config.get(Setting.TRACE_REQUESTS):
+                logger.trace("Making Google Drive request: " + url)
+            try:
+                data_to_use = data
+                if isinstance(data_to_use, io.BytesIO):
+                    # This is a pretty low-down dirty hack, but it works and lets us reuse the byte stream.
+                    # aiohttp complains if you pass it a large byte object
+                    data_to_use = io.BytesIO(data_to_use.getbuffer())
+                    data_to_use.seek(0)
+                return await self.drive.request(method, url, headers=headers_to_use, json=json, data=data_to_use)
+            except GoogleCredentialsExpired:
+                # Get fresh credentials, then retry right away.
+                logger.debug("Google Drive credentials have expired.  We'll retry with new ones.")
+                await self.refreshToken()
+            except KnownTransient as e:
+                backoff.backoff(e)
+                logger.error("{0}: we'll retry in {1} seconds".format(e.message(), backoff.peek()))
+                await self.time.sleepAsync(backoff.peek())
+            except ServerTimeoutError:
+                raise GoogleTimeoutError()

hassio-google-drive-backup/backup/drive/drivesource.py

@@ -0,0 +1,282 @@
+from datetime import datetime
+from io import IOBase
+from asyncio import Event
+from typing import Dict
+
+from aiohttp import ClientSession
+from aiohttp.client_exceptions import ClientResponseError
+from injector import inject, singleton
+
+from ..util import AsyncHttpGetter, GlobalInfo
+from ..config import Config, Setting, CreateOptions
+from ..const import SOURCE_GOOGLE_DRIVE
+from ..exceptions import (BackupFolderInaccessible,
+                          ExistingBackupFolderError,
+                          GoogleDrivePermissionDenied, LogicError)
+from ..model.backups import (PROP_NOTE, PROP_PROTECTED, PROP_RETAINED, PROP_TYPE, PROP_VERSION)
+from ..time import Time
+from .driverequests import DriveRequests
+from .folderfinder import FolderFinder
+from .thumbnail import THUMBNAIL_IMAGE
+from ..model import BackupDestination, DriveBackup, Backup
+from ..logger import getLogger
+from ..creds.creds import Creds
+from backup.const import NECESSARY_OLD_BACKUP_NAME, NECESSARY_OLD_BACKUP_PLURAL_NAME, NECESSARY_PROP_KEY_SLUG, NECESSARY_PROP_KEY_DATE, NECESSARY_PROP_KEY_NAME
+
+logger = getLogger(__name__)
+
+MIME_TYPE = "application/tar"
+THUMBNAIL_MIME_TYPE = "image/png"
+FOLDER_MIME_TYPE = 'application/vnd.google-apps.folder'
+FOLDER_NAME = 'Home Assistant Backups'
+FOLDER_CACHE_SECONDS = 30
+DRIVE_MAX_PROPERTY_LENGTH = 120
+
+
+@singleton
+class DriveSource(BackupDestination):
+    # SOMEDAY: read backups all in one big batch request, then sort the folder and child addons from that.  Would need to add test verifying the "current" backup directory is used instead of the "latest"
+    @inject
+    def __init__(self, config: Config, time: Time, drive_requests: DriveRequests, info: GlobalInfo, session: ClientSession, folderfinder: FolderFinder):
+        super().__init__()
+        self.session = session
+        self.config = config
+        self.drivebackend: DriveRequests = drive_requests
+        self.time = time
+        self.folder_finder = folderfinder
+        self._info = info
+        self._uploadedAtLeastOneChunk = False
+        self._drive_info = None
+        self._cred_trigger = Event()
+
+    def saveCreds(self, creds: Creds) -> None:
+        logger.info("Saving new Google Drive credentials")
+        self.drivebackend.saveCredentials(creds)
+        self.trigger()
+        self._cred_trigger.set()
+
+    async def debug_wait_for_credentials(self):
+        await self._cred_trigger.wait()
+        self._cred_trigger.clear()
+
+    def isCustomCreds(self):
+        return self.drivebackend.isCustomCreds()
+
+    @property
+    def might_be_oob_creds(self) -> bool:
+        return self.drivebackend.might_be_oob_creds
+
+    def name(self) -> str:
+        return SOURCE_GOOGLE_DRIVE
+
+    def title(self) -> str:
+        return "Google Drive"
+
+    def maxCount(self) -> None:
+        return self.config.get(Setting.MAX_BACKUPS_IN_GOOGLE_DRIVE)
+
+    def upload(self) -> bool:
+        return self.config.get(Setting.ENABLE_DRIVE_UPLOAD)
+
+    def enabled(self) -> bool:
+        return self.drivebackend.enabled()
+
+    def needsConfiguration(self) -> bool:
+        if not self.config.get(Setting.ENABLE_DRIVE_UPLOAD):
+            return False
+        return super().needsConfiguration()
+
+    def freeSpace(self):
+        if self._drive_info and self._drive_info.get("storageQuota") is not None and not self.folder_finder.currentIsSharedDrive():
+            info = self._drive_info.get("storageQuota")
+            if 'limit' in info and 'usage' in info:
+                return int(info.get("limit")) - int((info.get("usage")))
+        return super().freeSpace()
+
+    async def create(self, options: CreateOptions) -> DriveBackup:
+        raise LogicError("Backups can't be created in Drive")
+
+    def checkBeforeChanges(self):
+        existing = self.folder_finder.getExisting()
+        if existing:
+            raise ExistingBackupFolderError(
+                existing.get('id'), existing.get('name'))
+
+    def icon(self) -> str:
+        return "google-drive"
+
+    def isWorking(self):
+        return self._uploadedAtLeastOneChunk
+
+    def detail(self):
+        if self._drive_info and 'user' in self._drive_info and 'emailAddress' in self._drive_info['user']:
+            return f'{self._drive_info["user"]["emailAddress"]}'
+        else:
+            return super().detail()
+
+    async def get(self, allow_retry=True) -> Dict[str, DriveBackup]:
+        parent = await self.getFolderId()
+        try:
+            self._drive_info = await self.drivebackend.getAboutInfo()
+        except Exception as e:
+            # This is just used to get the remaining space in Drive, which is a
+            # nice to have.  Just log the error to debug if we can't get it
+            logger.debug("Unable to retrieve Google Drive storage info: " + str(e))
+        backups: Dict[str, DriveBackup] = {}
+        try:
+            async for child in self.drivebackend.query("'{}' in parents".format(parent)):
+                properties = child.get('appProperties')
+                if properties and NECESSARY_PROP_KEY_DATE in properties and NECESSARY_PROP_KEY_SLUG in properties and not child['trashed']:
+                    backup = DriveBackup(child)
+                    backups[backup.slug()] = backup
+        except ClientResponseError as e:
+            if e.status == 404:
+                # IIUC, 404 on create can only mean that the parent id isn't valid anymore.
+                if not self.config.get(Setting.SPECIFY_BACKUP_FOLDER) and allow_retry:
+                    self.folder_finder.deCache()
+                    await self.folder_finder.create()
+                    return await self.get(False)
+                raise BackupFolderInaccessible(parent)
+            raise e
+        except GoogleDrivePermissionDenied:
+            # This should always mean we lost permission on the backup folder, but at least it still exists.
+            if not self.config.get(Setting.SPECIFY_BACKUP_FOLDER) and allow_retry:
+                self.folder_finder.deCache()
+                await self.folder_finder.create()
+                return await self.get(False)
+            raise BackupFolderInaccessible(parent)
+        return backups
+
+    async def delete(self, backup: Backup):
+        item = self._validateBackup(backup)
+        if item.canDeleteDirectly():
+            logger.info("Deleting '{}' From Google Drive".format(item.name()))
+            await self.drivebackend.delete(item.id())
+        else:
+            logger.info("Trashing '{}' in Google Drive".format(item.name()))
+            await self.drivebackend.update(item.id(), {"trashed": True})
+        backup.removeSource(self.name())
+
+    async def save(self, backup: Backup, source: AsyncHttpGetter) -> DriveBackup:
+        retain = backup.getOptions() and backup.getOptions().retain_sources.get(self.name(), False)
+        parent_id = await self.getFolderId()
+        if backup.note() is not None:
+            desc = backup.note()
+        else:
+            desc = 'A Home Assistant backup file uploaded by Home Assistant Google Drive Backup'
+        file_metadata = {
+            'name': str(backup.name()) + ".tar",
+            'parents': [parent_id],
+            'description': desc,
+            'appProperties': {
+                NECESSARY_PROP_KEY_SLUG: backup.slug(),
+                NECESSARY_PROP_KEY_DATE: str(backup.date()),
+                PROP_TYPE: str(backup.backupType()),
+                PROP_VERSION: str(backup.version()),
+                PROP_PROTECTED: str(backup.protected()),
+                PROP_RETAINED: str(retain),
+            },
+            'contentHints': {
+                'indexableText': 'Home Assistant hassio ' + NECESSARY_OLD_BACKUP_NAME + ' ' + NECESSARY_OLD_BACKUP_PLURAL_NAME + ' backup backups home assistant ' + desc,
+                'thumbnail': {
+                    'image': THUMBNAIL_IMAGE,
+                    'mimeType': THUMBNAIL_MIME_TYPE
+                }
+            },
+            'createdTime': self._timeToRfc3339String(backup.date()),
+            'modifiedTime': self._timeToRfc3339String(backup.date())
+        }
+
+        if backup.note() is not None:
+            file_metadata['appProperties'][PROP_NOTE] = self.truncateAppProperty(PROP_NOTE, backup.note())
+        file_metadata['appProperties'][NECESSARY_PROP_KEY_NAME] = self.truncateAppProperty(NECESSARY_PROP_KEY_NAME, str(backup.name()))
+
+        async with source:
+            try:
+                logger.info("Uploading '{}' to Google Drive".format(
+                    backup.name()))
+                size = source.size()
+                self._info.upload(size)
+                backup.overrideStatus("Uploading {0}%", source)
+                backup.setUploadSource(self.title(), source)
+                async for progress in self.drivebackend.create(source, file_metadata, MIME_TYPE):
+                    self._uploadedAtLeastOneChunk = True
+                    if isinstance(progress, float):
+                        logger.debug("Uploading {1} {0:.2f}%".format(
+                            progress * 100, backup.name()))
+                    else:
+                        return DriveBackup(progress)
+                raise LogicError(
+                    "Google Drive backup upload didn't return a completed item before exiting")
+            except ClientResponseError as e:
+                if e.status == 404:
+                    # IIUC, 404 on create can only mean that the parent id isn't valid anymore.
+                    raise BackupFolderInaccessible(parent_id)
+                raise e
+            except GoogleDrivePermissionDenied:
+                # This should always mean we lost permission on the backup folder, since we could have only just
+                # created the backup item on this request.
+                raise BackupFolderInaccessible(parent_id)
+            finally:
+                backup.clearUploadSource()
+                self._uploadedAtLeastOneChunk = False
+                backup.clearStatus()
+
+    def truncateAppProperty(self, key: str, value: str):
+        # Annoylingly, Drive properties can be a maximum of 124 bytes, in len(key + value) UTF8 encoded.
+        # https://developers.google.com/drive/api/guides/properties
+        # Is the extra indexing REALLY that expensive? Thats like some 1990's mainframe limitation.
+        # Make sure we stay well under that limit
+        if value is None:
+            return value
+        permitted = ""
+        current = 0
+        while current < len(value) and len(str(key + permitted + value[current]).encode('utf-8')) < DRIVE_MAX_PROPERTY_LENGTH:
+            permitted += value[current]
+            current += 1
+        return permitted
+
+    async def read(self, backup: Backup) -> IOBase:
+        item = self._validateBackup(backup)
+        return await self.drivebackend.download(item.id(), item.size())
+
+    async def retain(self, backup: Backup, retain: bool) -> None:
+        item = self._validateBackup(backup)
+        if item.retained() == retain:
+            return
+        file_metadata: Dict[str, str] = {
+            'appProperties': {
+                PROP_RETAINED: str(retain),
+            },
+        }
+        await self.drivebackend.update(item.id(), file_metadata)
+        item.setRetained(retain)
+
+    async def note(self, backup, note: str) -> None:
+        item = self._validateBackup(backup)
+        truncated = self.truncateAppProperty(PROP_NOTE, note)
+        file_metadata: Dict[str, str] = {
+            'appProperties': {
+                PROP_NOTE: truncated,
+            },
+            'description': note,
+        }
+        logger.debug(f"Adding a note to drive backup '{item.name()}'")
+        await self.drivebackend.update(item.id(), file_metadata)
+        item.setNote(truncated)
+
+    async def getFolderId(self):
+        return await self.folder_finder.get()
+
+    def _validateBackup(self, backup: Backup) -> DriveBackup:
+        drive_item: DriveBackup = backup.getSource(self.name())
+        if not drive_item:
+            raise LogicError(
+                "Requested to do something with a backup from Google Drive, but the backup has no Google Drive source")
+        return drive_item
+
+    def _timeToRfc3339String(self, time: datetime) -> str:
+        return time.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    async def _get(self, id):
+        return await self.drivebackend.get(id)

hassio-google-drive-backup/backup/drive/folderfinder.py

@@ -0,0 +1,204 @@
+from datetime import timedelta
+from typing import Any, Dict
+from backup.file import File
+from aiohttp.client_exceptions import ClientResponseError
+from injector import inject, singleton
+
+from ..config import Config, Setting
+from ..exceptions import (BackupFolderInaccessible, BackupFolderMissingError,
+                          GoogleDrivePermissionDenied, LogInToGoogleDriveError)
+from ..time import Time
+from .driverequests import DriveRequests
+from ..logger import getLogger
+
+logger = getLogger(__name__)
+
+FOLDER_MIME_TYPE = 'application/vnd.google-apps.folder'
+FOLDER_NAME = 'Home Assistant Backups'
+FOLDER_CACHE_SECONDS = 60 * 31  # 31 minutes
+
+
+@singleton
+class FolderFinder():
+    @inject
+    def __init__(self, config: Config, time: Time, drive_requests: DriveRequests):
+        self.config = config
+        self.drivebackend: DriveRequests = drive_requests
+        self.time = time
+
+        # The cached folder id
+        self._folderId = None
+
+        # When the fodler id was last cached
+        self._folder_queryied_last = None
+
+        # These get set when an existing folder is found and should cause the UI to
+        # prompt for what to do about it.
+        self._existing_folder = None
+        self._use_existing = None
+        self._folder_details = None
+
+    def resolveExisting(self, val):
+        if self._existing_folder:
+            self._use_existing = val
+        else:
+            self._use_existing = None
+
+    def _isSharedDrive(self, folder):
+        driveId = folder.get("driveId", None)
+        return driveId and len(driveId) > 0
+
+    def currentIsSharedDrive(self):
+        return self._folder_details and self._isSharedDrive(self._folder_details)
+
+    async def get(self):
+        if self._existing_folder and self._use_existing is not None:
+            if self._use_existing:
+                await self.save(self._existing_folder)
+            else:
+                await self.create()
+            self._use_existing = None
+        if not self._folder_queryied_last or self._folder_queryied_last + timedelta(seconds=FOLDER_CACHE_SECONDS) < self.time.now():
+            try:
+                self._folderId = await self._readFolderId()
+            except (BackupFolderMissingError, BackupFolderInaccessible):
+                if not self.config.get(Setting.SPECIFY_BACKUP_FOLDER):
+                    # Search for a folder, they may have created one before
+                    self._existing_folder = await self._search()
+                    if self._existing_folder:
+                        self._folderId = self._existing_folder.get('id')
+                    else:
+                        # Create folder, since no other folder is available
+                        await self.create()
+                else:
+                    raise
+            self._folder_queryied_last = self.time.now()
+        return self._folderId
+
+    def getExisting(self):
+        return self._existing_folder
+
+    async def save(self, folder: Any) -> str:
+        if not isinstance(folder, str):
+            self._folder_details = folder
+            folder = folder.get('id')
+        else:
+            self._folder_details = None
+        logger.info("Saving backup folder: " + folder)
+        File.write(self.config.get(Setting.FOLDER_FILE_PATH), folder)
+        self._folderId = folder
+        self._folder_queryied_last = self.time.now()
+        self._existing_folder = None
+
+    def reset(self):
+        if File.exists(self.config.get(Setting.FOLDER_FILE_PATH)):
+            File.delete(self.config.get(Setting.FOLDER_FILE_PATH))
+        self._folderId = None
+        self._folder_queryied_last = None
+        self._existing_folder = None
+
+    def getCachedFolder(self):
+        return self._folderId
+
+    def deCache(self):
+        self._folderId = None
+        self._folder_queryied_last = None
+
+    async def _readFolderId(self) -> str:
+        # First, check if we cached the drive folder
+        if not File.exists(self.config.get(Setting.FOLDER_FILE_PATH)):
+            raise BackupFolderMissingError()
+        else:
+            folder_id: str = File.read(self.config.get(Setting.FOLDER_FILE_PATH)).strip()
+            if await self._verify(folder_id):
+                return folder_id
+            else:
+                raise BackupFolderInaccessible(folder_id)
+
+    async def _search(self) -> str:
+        folders = []
+
+        try:
+            async for child in self.drivebackend.query("mimeType='" + FOLDER_MIME_TYPE + "'"):
+                if self._isValidFolder(child):
+                    folders.append(child)
+        except ClientResponseError as e:
+            # 404 means the folder doesn't exist (maybe it got moved?)
+            if e.status == 404:
+                "Make Error"
+                raise LogInToGoogleDriveError()
+            else:
+                raise e
+
+        if len(folders) == 0:
+            return None
+
+        folders.sort(key=lambda c: Time.parse(c.get("modifiedTime")))
+        # Found a folder, which means we're probably using the add-on from a
+        # previous (or duplicate) installation.  Record and return the id but don't
+        # persist it until the user chooses to do so.
+        folder = folders[len(folders) - 1]
+        logger.info("Found " + folder.get('name'))
+        return folder
+
+    async def _verify(self, id):
+        if self.drivebackend.isCustomCreds():
+            # If the user is using custom creds and specifying the backup folder, then chances are the
+            # app doesn't have permission to access the parent folder directly.  Ironically, we can still
+            # query for children and add/remove backups.  Not a huge deal, just
+            # means we can't verify the folder still exists, isn't trashed, etc.  Just let it be valid
+            # and handle potential errors elsewhere.
+            return True
+        # Query drive for the folder to make sure it still exists and we have the right permission on it.
+        try:
+            folder = await self.drivebackend.get(id)
+            if not self._isValidFolder(folder):
+                logger.info("Provided backup folder {0} is invalid".format(id))
+                return False
+            self._folder_details = folder
+            return True
+        except ClientResponseError as e:
+            if e.status == 404:
+                # 404 means the folder doesn't exist (maybe it got moved?) but can also mean that we
+                # just don't have permission to see the folder.   Often we can still upload into it, so just
+                # let it pass without further verification and let other error handling (on upload) identify problems.
+                return True
+            else:
+                raise e
+        except GoogleDrivePermissionDenied:
+            # Lost permission on the backup folder
+            return False
+
+    def _isValidFolder(self, folder) -> bool:
+        try:
+            caps = folder.get('capabilities')
+            if folder.get('trashed'):
+                return False
+            elif not caps['canAddChildren']:
+                return False
+            elif not caps['canListChildren']:
+                return False
+            elif not caps.get('canDeleteChildren', False) and not caps.get('canRemoveChildren', False):
+                if self._isSharedDrive(folder) and caps.get("canTrashChildren", False):
+                    # Allow folders in shared drives if you can still trash items inside it.
+                    return True
+                return False
+            elif folder.get("mimeType") != FOLDER_MIME_TYPE:
+                return False
+        except Exception:
+            return False
+        return True
+
+    async def create(self) -> str:
+        logger.info('Creating folder "{}" in "My Drive"'.format(FOLDER_NAME))
+        file_metadata: Dict[str, str] = {
+            'name': FOLDER_NAME,
+            'mimeType': FOLDER_MIME_TYPE,
+            'appProperties': {
+                "backup_folder": "true",
+            },
+        }
+        folder = await self.drivebackend.createFolder(file_metadata)
+        self._folder_details = folder
+        await self.save(folder)
+        return folder.get('id')