New Addon
Google BK
This commit is contained in:
@@ -0,0 +1,5 @@
|
||||
# flake8: noqa
|
||||
from .exchanger import Exchanger
|
||||
from .creds import Creds, KEY_TOKEN_EXPIRY, KEY_ACCESS_TOKEN, KEY_CLIENT_ID, KEY_CLIENT_SECRET
|
||||
from .driverequester import DriveRequester
|
||||
MANUAL_CODE_REDIRECT_URI: str = "urn:ietf:wg:oauth:2.0:oob"
|
||||
@@ -0,0 +1,90 @@
|
||||
from ..exceptions import ensureKey
|
||||
from ..time import Time
|
||||
from typing import Optional
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
KEY_REFRESH_TOKEN = 'refresh_token'
|
||||
KEY_CLIENT_ID = 'client_id'
|
||||
KEY_CLIENT_SECRET = 'client_secret'
|
||||
KEY_EXPIRES_IN = 'expires_in'
|
||||
KEY_TOKEN_EXPIRY = 'token_expiry'
|
||||
KEY_ACCESS_TOKEN = 'access_token'
|
||||
|
||||
|
||||
class Creds():
|
||||
def __init__(self, time: Time, id: str, expiration: datetime,
|
||||
access_token: str, refresh_token: str,
|
||||
secret: Optional[str] = None, original_expiration: datetime = None):
|
||||
self._id = id
|
||||
self.time: Time = time
|
||||
self._secret = secret
|
||||
self._access_token = access_token
|
||||
self._refresh_token = refresh_token
|
||||
self._expiration = expiration
|
||||
self._original_expiration = original_expiration
|
||||
|
||||
@property
|
||||
def id(self):
|
||||
return self._id
|
||||
|
||||
@property
|
||||
def secret(self):
|
||||
return self._secret
|
||||
|
||||
@property
|
||||
def refresh_token(self):
|
||||
return self._refresh_token
|
||||
|
||||
@property
|
||||
def access_token(self):
|
||||
return self._access_token
|
||||
|
||||
@property
|
||||
def expiration(self):
|
||||
if self._expiration is None:
|
||||
return self.time.now()
|
||||
return self._expiration
|
||||
|
||||
@property
|
||||
def original_expiration(self) -> datetime:
|
||||
return self._original_expiration
|
||||
|
||||
@property
|
||||
def is_expired(self):
|
||||
return self.time.now() >= self.expiration
|
||||
|
||||
def serialize(self, include_secret=True):
|
||||
ret = {
|
||||
"client_id": self.id
|
||||
}
|
||||
if self.secret is not None and include_secret:
|
||||
ret[KEY_CLIENT_SECRET] = self.secret
|
||||
if self.refresh_token is not None:
|
||||
ret[KEY_REFRESH_TOKEN] = self.refresh_token
|
||||
if self.access_token is not None:
|
||||
ret[KEY_ACCESS_TOKEN] = self.access_token
|
||||
if self.expiration is not None:
|
||||
ret[KEY_TOKEN_EXPIRY] = self.time.asRfc3339String(self.expiration)
|
||||
return ret
|
||||
|
||||
@classmethod
|
||||
def load(cls, time: Time, data, id=None, secret=None, original_expiration=None):
|
||||
if id is None:
|
||||
id = ensureKey(KEY_CLIENT_ID, data, "credentials")
|
||||
if secret is None and KEY_CLIENT_SECRET in data:
|
||||
secret = data[KEY_CLIENT_SECRET]
|
||||
refresh = ensureKey(KEY_REFRESH_TOKEN, data, "credentials")
|
||||
access = ensureKey(KEY_ACCESS_TOKEN, data, "credentials")
|
||||
expires = None
|
||||
try:
|
||||
if KEY_TOKEN_EXPIRY in data:
|
||||
expires = time.parse(data[KEY_TOKEN_EXPIRY])
|
||||
if original_expiration is None:
|
||||
original_expiration = expires
|
||||
elif KEY_EXPIRES_IN in data:
|
||||
expires = time.now() + timedelta(seconds=int(data[KEY_EXPIRES_IN]))
|
||||
else:
|
||||
expires = time.now()
|
||||
except BaseException:
|
||||
expires = time.now()
|
||||
return Creds(time=time, id=id, access_token=access, refresh_token=refresh, secret=secret, expiration=expires, original_expiration=original_expiration)
|
||||
@@ -0,0 +1,113 @@
|
||||
from aiohttp import ClientSession, ContentTypeError, ClientConnectorError, ClientTimeout, ClientResponse
|
||||
from aiohttp.client_exceptions import ServerTimeoutError, ServerDisconnectedError, ClientOSError
|
||||
from backup.exceptions import GoogleUnexpectedError, GoogleInternalError, GoogleRateLimitError, GoogleCredentialsExpired, CredRefreshGoogleError, DriveQuotaExceeded, GoogleDrivePermissionDenied, GoogleDnsFailure, GoogleCantConnect, GoogleTimeoutError
|
||||
from backup.util import Resolver
|
||||
from backup.logger import getLogger
|
||||
from backup.config import Config, Setting
|
||||
from injector import singleton, inject
|
||||
from dns.exception import DNSException
|
||||
|
||||
RATE_LIMIT_EXCEEDED = [403]
|
||||
TOO_MANY_REQUESTS = [429]
|
||||
INTERNAL_ERROR = [500, 503]
|
||||
PERMISSION_DENIED = [401]
|
||||
REQUEST_TIMEOUT = [408]
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
|
||||
@singleton
|
||||
class DriveRequester():
|
||||
@inject
|
||||
def __init__(self, config: Config, session: ClientSession, resolver: Resolver):
|
||||
self.session = session
|
||||
self.resolver = resolver
|
||||
self.config = config
|
||||
|
||||
async def request(self, method, url, headers={}, json=None, data=None) -> ClientResponse:
|
||||
try:
|
||||
response = await self.session.request(method, url, headers=headers, json=json, timeout=self.buildTimeout(), data=data)
|
||||
if response.status < 400:
|
||||
return response
|
||||
await self.raiseForKnownErrors(response)
|
||||
if response.status in PERMISSION_DENIED:
|
||||
response.release()
|
||||
raise GoogleCredentialsExpired()
|
||||
elif response.status in INTERNAL_ERROR:
|
||||
response.release()
|
||||
raise GoogleInternalError()
|
||||
elif response.status in RATE_LIMIT_EXCEEDED or response.status in TOO_MANY_REQUESTS:
|
||||
response.release()
|
||||
raise GoogleRateLimitError()
|
||||
elif response.status in REQUEST_TIMEOUT:
|
||||
response.release()
|
||||
raise GoogleTimeoutError()
|
||||
response.raise_for_status()
|
||||
return response
|
||||
except ClientConnectorError as e:
|
||||
logger.debug(
|
||||
"Ran into trouble reaching Google Drive's servers. We'll use alternate DNS servers on the next attempt.")
|
||||
self.resolver.toggle()
|
||||
if "Cannot connect to host" in str(e) or "Connection reset by peer" in str(e):
|
||||
raise GoogleCantConnect()
|
||||
if e.os_error.errno == -2:
|
||||
# -2 means dns lookup failed.
|
||||
raise GoogleDnsFailure()
|
||||
elif str(e.os_error) == "Domain name not found":
|
||||
raise GoogleDnsFailure()
|
||||
elif e.os_error.errno in [99, 111, 10061, 104]:
|
||||
# 111 means connection refused
|
||||
# Can't connect
|
||||
raise GoogleCantConnect()
|
||||
elif "Could not contact DNS serve" in str(e.os_error):
|
||||
# Wish there was a better way to identify this exception
|
||||
raise GoogleDnsFailure()
|
||||
raise
|
||||
except ClientOSError as e:
|
||||
if e.errno == 1:
|
||||
raise GoogleUnexpectedError()
|
||||
raise
|
||||
except ServerTimeoutError:
|
||||
raise GoogleTimeoutError()
|
||||
except ServerDisconnectedError:
|
||||
raise GoogleUnexpectedError()
|
||||
except DNSException:
|
||||
logger.debug(
|
||||
"Ran into trouble resolving Google Drive's servers. We'll use normal DNS servers on the next attempt.")
|
||||
self.resolver.toggle()
|
||||
raise GoogleDnsFailure()
|
||||
|
||||
def buildTimeout(self):
|
||||
return ClientTimeout(
|
||||
sock_connect=self.config.get(
|
||||
Setting.GOOGLE_DRIVE_TIMEOUT_SECONDS),
|
||||
sock_read=self.config.get(Setting.GOOGLE_DRIVE_TIMEOUT_SECONDS))
|
||||
|
||||
async def raiseForKnownErrors(self, response):
|
||||
try:
|
||||
message = await response.json()
|
||||
except ContentTypeError:
|
||||
return
|
||||
except ValueError:
|
||||
# parsing json failed, just give up
|
||||
return
|
||||
except TypeError:
|
||||
# Same
|
||||
return
|
||||
if "error" not in message:
|
||||
return
|
||||
error_obj = message["error"]
|
||||
if isinstance(error_obj, str):
|
||||
if error_obj == "expired":
|
||||
raise GoogleCredentialsExpired()
|
||||
else:
|
||||
raise CredRefreshGoogleError(error_obj)
|
||||
if "errors" not in error_obj:
|
||||
return
|
||||
for error in error_obj["errors"]:
|
||||
if "reason" not in error:
|
||||
continue
|
||||
if error["reason"] == "storageQuotaExceeded":
|
||||
raise DriveQuotaExceeded()
|
||||
elif error["reason"] in ["forbidden", "insufficientFilePermissions"]:
|
||||
raise GoogleDrivePermissionDenied()
|
||||
@@ -0,0 +1,160 @@
|
||||
import asyncio
|
||||
from aiohttp import ClientSession, ClientConnectorError, ClientTimeout
|
||||
from .creds import Creds, KEY_CLIENT_ID, KEY_CLIENT_SECRET, KEY_ACCESS_TOKEN, KEY_REFRESH_TOKEN, KEY_EXPIRES_IN
|
||||
from ..exceptions import ensureKey, GoogleCredentialsExpired, CredRefreshGoogleError, CredRefreshMyError
|
||||
from ..config import Config, Setting, VERSION
|
||||
from yarl import URL
|
||||
from ..time import Time
|
||||
from ..logger import getLogger
|
||||
from .driverequester import DriveRequester
|
||||
from datetime import timedelta
|
||||
from injector import singleton, inject
|
||||
|
||||
|
||||
SCOPE = 'https://www.googleapis.com/auth/drive.file'
|
||||
|
||||
KEY_REDIRECT_URI = 'redirect_uri'
|
||||
KEY_SCOPE = 'scope'
|
||||
KEY_RESPONSE_TYPE = 'response_type'
|
||||
KEY_INCLUDE_GRANTED_SCOPES = 'include_granted_scopes'
|
||||
KEY_ACCESS_TYPE = 'access_type'
|
||||
KEY_STATE = 'state'
|
||||
KEY_PROMPT = 'prompt'
|
||||
KEY_CODE = 'code'
|
||||
KEY_GRANT_TYPE = 'grant_type'
|
||||
KEY_VERSION = 'version'
|
||||
KEY_CLIENT = 'client'
|
||||
|
||||
CRED_OBJECT_NAME = "credential token response"
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
|
||||
@singleton
|
||||
class Exchanger():
|
||||
@inject
|
||||
def __init__(self,
|
||||
time: Time,
|
||||
session: ClientSession,
|
||||
config: Config,
|
||||
drive: DriveRequester,
|
||||
client_id: str,
|
||||
client_secret: str,
|
||||
redirect: URL):
|
||||
self.time = time
|
||||
self.config = config
|
||||
self.session = session
|
||||
self.drive = drive
|
||||
self._client_id = client_id
|
||||
self._client_secret = client_secret
|
||||
self._redirect = redirect
|
||||
|
||||
async def getAuthorizationUrl(self, state="") -> str:
|
||||
url = URL(self.config.get(Setting.DRIVE_AUTHORIZE_URL)).with_query({
|
||||
KEY_CLIENT_ID: self._client_id,
|
||||
KEY_SCOPE: SCOPE,
|
||||
KEY_RESPONSE_TYPE: 'code',
|
||||
KEY_INCLUDE_GRANTED_SCOPES: 'true',
|
||||
KEY_ACCESS_TYPE: "offline",
|
||||
KEY_STATE: state,
|
||||
KEY_REDIRECT_URI: str(self._redirect),
|
||||
KEY_PROMPT: "consent"
|
||||
})
|
||||
return str(url)
|
||||
|
||||
async def exchange(self, code):
|
||||
data = {
|
||||
KEY_CLIENT_ID: self._client_id,
|
||||
KEY_CLIENT_SECRET: self._client_secret,
|
||||
KEY_CODE: code,
|
||||
KEY_REDIRECT_URI: str(self._redirect),
|
||||
KEY_GRANT_TYPE: 'authorization_code'
|
||||
}
|
||||
resp = None
|
||||
async with await self.drive.request("post", self.config.get(Setting.DRIVE_TOKEN_URL), data=data) as resp:
|
||||
return Creds.load(self.time, await resp.json(), id=self._client_id, secret=self._client_secret)
|
||||
|
||||
async def refresh(self, creds: Creds):
|
||||
if creds.secret is not None:
|
||||
return await self._refresh_google(creds)
|
||||
else:
|
||||
return await self._refresh_default(creds)
|
||||
|
||||
async def _refresh_google(self, creds: Creds):
|
||||
data = {
|
||||
KEY_CLIENT_ID: creds.id,
|
||||
KEY_CLIENT_SECRET: creds.secret,
|
||||
KEY_REFRESH_TOKEN: creds.refresh_token,
|
||||
KEY_GRANT_TYPE: 'refresh_token'
|
||||
}
|
||||
async with await self.drive.request("post", self.config.get(Setting.DRIVE_REFRESH_URL), data=data) as resp:
|
||||
data = await resp.json()
|
||||
return Creds(
|
||||
self.time,
|
||||
id=creds.id,
|
||||
secret=creds.secret,
|
||||
access_token=ensureKey(KEY_ACCESS_TOKEN, data, CRED_OBJECT_NAME),
|
||||
refresh_token=creds.refresh_token,
|
||||
expiration=self._get_expiration(data),
|
||||
original_expiration=creds.original_expiration)
|
||||
|
||||
async def _refresh_default(self, creds: Creds):
|
||||
data = {
|
||||
KEY_CLIENT_ID: creds.id,
|
||||
KEY_REFRESH_TOKEN: creds.refresh_token,
|
||||
}
|
||||
token_paths = self.config.getTokenServers("/drive/refresh")
|
||||
last_error = None
|
||||
for url in token_paths:
|
||||
try:
|
||||
headers = {
|
||||
'addon_version': VERSION,
|
||||
'client': self.config.clientIdentifier()
|
||||
}
|
||||
async with self.session.post(str(url), headers=headers, json=data, timeout=ClientTimeout(total=self.config.get(Setting.EXCHANGER_TIMEOUT_SECONDS))) as resp:
|
||||
if resp.status < 400:
|
||||
return Creds.load(self.time, await resp.json(), original_expiration=creds.original_expiration)
|
||||
elif resp.status == 503:
|
||||
json = {}
|
||||
try:
|
||||
json = await resp.json()
|
||||
except BaseException:
|
||||
pass
|
||||
if "error" in json:
|
||||
if "invalid_grant" in json["error"]:
|
||||
raise GoogleCredentialsExpired()
|
||||
else:
|
||||
# Record the error, but still try other hosts
|
||||
last_error = CredRefreshGoogleError(json["error"])
|
||||
else:
|
||||
last_error = CredRefreshMyError("HTTP 503 from " + url.host)
|
||||
elif resp.status == 401:
|
||||
raise GoogleCredentialsExpired()
|
||||
else:
|
||||
try:
|
||||
extra = (await resp.json())["error"]
|
||||
except BaseException:
|
||||
extra = ""
|
||||
|
||||
# this is likely due to misconfiguration
|
||||
logger.warning("Got {0}:{1} from {2}, trying alternate server(s)...".format(resp.status, extra, url.host))
|
||||
last_error = CredRefreshMyError("HTTP {} {}".format(resp.status, extra))
|
||||
except ClientConnectorError:
|
||||
logger.warning("Unable to reach " + str(url.host) + ", trying alternate server(s)...")
|
||||
last_error = "Couldn't communicate with " + url.host
|
||||
except asyncio.exceptions.TimeoutError:
|
||||
logger.warning("Timed out communicating with " + str(url.host) + ", trying alternate server(s)...")
|
||||
last_error = "Timed out communicating with " + url.host
|
||||
logger.error("Unable to refresh credentials with Google Drive")
|
||||
if isinstance(last_error, str):
|
||||
raise CredRefreshMyError(last_error)
|
||||
elif isinstance(last_error, Exception):
|
||||
raise last_error
|
||||
else:
|
||||
raise Exception("Unexpected error type: " + str(last_error))
|
||||
|
||||
def refreshCredentials(self, refresh_token):
|
||||
return Creds(self.time, id=self._client_id, expiration=None, access_token=None, refresh_token=refresh_token, secret=self._client_secret)
|
||||
|
||||
def _get_expiration(self, data):
|
||||
return self.time.now() + timedelta(seconds=int(ensureKey(KEY_EXPIRES_IN, data, CRED_OBJECT_NAME)))
|
||||
Reference in New Issue
Block a user