Updated CloudFlare
This commit is contained in:
@@ -1,9 +1,5 @@
|
||||
## What’s changed
|
||||
## ✨ New features
|
||||
|
||||
- Add Cloudflare backend connectivity check @elcajon (#432)
|
||||
|
||||
## ⬆️ Dependency updates
|
||||
|
||||
- ⬆️ Update cloudflare/cloudflared to v2023.7.0 @renovate (#434)
|
||||
- ⬆️ Update docker/setup-buildx-action action to v2.9.1 @renovate (#435)
|
||||
- ⬆️ Update Add-on base image to v14.0.3 @renovate (#439)
|
||||
- ⬆️ Update cloudflare/cloudflared to v2023.7.1 @renovate (#440)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
name: Cloudflared
|
||||
version: 4.2.0
|
||||
version: 4.2.1
|
||||
slug: cloudflared
|
||||
description: Use a Cloudflare Tunnel to remotely connect to Home Assistant without
|
||||
opening any ports
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
[run]
|
||||
omit =
|
||||
hassio-google-drive-backup/backup/logger.py
|
||||
hassio-google-drive-backup/backup/tracing_session.py
|
||||
hassio-google-drive-backup/backup/ui/debug.py
|
||||
hassio-google-drive-backup/backup/server/cloudlogger.py
|
||||
hassio-google-drive-backup/backup/debug/*
|
||||
@@ -0,0 +1,10 @@
|
||||
FROM python:3.9-buster
|
||||
|
||||
WORKDIR /usr/src/install
|
||||
RUN apt-get update
|
||||
RUN apt-get install fping
|
||||
# install gcloud api
|
||||
RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - && apt-get update -y && apt-get install google-cloud-cli -y
|
||||
|
||||
COPY requirements-dev.txt ./
|
||||
RUN pip install --no-cache-dir -r requirements-dev.txt
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"build": { "dockerfile": "Dockerfile" },
|
||||
"extensions": ["ms-python.python", "wholroyd.jinja","ms-python.vscode-pylance"],
|
||||
"forwardPorts": [3000]
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
google-api-python-client
|
||||
google-auth-httplib2
|
||||
google-auth-oauthlib
|
||||
oauth2client
|
||||
requests
|
||||
python-dateutil==2.6.1
|
||||
watchdog
|
||||
pyyaml
|
||||
dnspython
|
||||
pytest>=5.4.0
|
||||
pytest-timeout
|
||||
pytest-repeat
|
||||
pytest-asyncio
|
||||
flake8
|
||||
mypy
|
||||
aiorun
|
||||
aiohttp
|
||||
aiodns
|
||||
aiofiles
|
||||
injector
|
||||
autopep8
|
||||
colorlog
|
||||
debugpy
|
||||
google-cloud-logging
|
||||
google-cloud-firestore
|
||||
aiohttp-jinja2
|
||||
beautifulsoup4
|
||||
firebase-admin
|
||||
aiofile
|
||||
grpcio
|
||||
aioping
|
||||
pytz
|
||||
tzlocal
|
||||
pytest-cov
|
||||
@@ -0,0 +1,6 @@
|
||||
github: sabeechen
|
||||
patreon: sabeechen
|
||||
custom:
|
||||
- https://www.buymeacoffee.com/sabeechen
|
||||
- https://www.paypal.com/paypalme/stephenbeechen
|
||||
- https://github.com/sabeechen/hassio-google-drive-backup/blob/master/donate-crypto.md
|
||||
@@ -0,0 +1,7 @@
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: daily
|
||||
time: "06:00"
|
||||
@@ -0,0 +1,72 @@
|
||||
# For most projects, this workflow file will not need changing; you simply need
|
||||
# to commit it to your repository.
|
||||
#
|
||||
# You may wish to alter this file to override the set of languages analyzed,
|
||||
# or to provide custom queries or build logic.
|
||||
#
|
||||
# ******** NOTE ********
|
||||
# We have attempted to detect the languages in your repository. Please check
|
||||
# the `language` matrix defined below to confirm you have the correct set of
|
||||
# supported CodeQL languages.
|
||||
#
|
||||
name: "CodeQL"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ "master", "dev" ]
|
||||
pull_request:
|
||||
# The branches below must be a subset of the branches above
|
||||
branches: [ "master", "dev" ]
|
||||
schedule:
|
||||
- cron: '42 18 * * 5'
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
name: Analyze
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
security-events: write
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
language: [ 'javascript', 'python' ]
|
||||
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
|
||||
# Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v3
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v2
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
# If you wish to specify custom queries, you can do so here or in a config file.
|
||||
# By default, queries listed here will override any specified in a config file.
|
||||
# Prefix the list here with "+" to use these queries and those in the config file.
|
||||
|
||||
# Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
|
||||
# queries: security-extended,security-and-quality
|
||||
|
||||
|
||||
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
|
||||
# If this step fails, then you should remove it and run the build manually (see below)
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@v2
|
||||
|
||||
# ℹ️ Command-line programs to run using the OS shell.
|
||||
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
|
||||
|
||||
# If the Autobuild fails above, remove it and uncomment the following three lines.
|
||||
# modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
|
||||
|
||||
# - run: |
|
||||
# echo "Run, Build Application using script"
|
||||
# ./location_of_script_within_repo/buildscript.sh
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v2
|
||||
@@ -0,0 +1,13 @@
|
||||
name: Lint
|
||||
on: [push, pull_request]
|
||||
jobs:
|
||||
build:
|
||||
name: Lint add-on configuration
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: ⤵️ Check out code from GitHub
|
||||
uses: actions/checkout@v3
|
||||
- name: 🚀 Run Home Assistant Add-on Linter
|
||||
uses: frenck/action-addon-linter@v2.13
|
||||
with:
|
||||
path: "./hassio-google-drive-backup"
|
||||
@@ -0,0 +1,31 @@
|
||||
name: push_to_prod
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
deploy_to_staging:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: '3.8'
|
||||
|
||||
- name: Check out dev repo
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
path: dev
|
||||
persist-credentials: false
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@v2.2.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }}
|
||||
|
||||
- name: Publish Addon Image
|
||||
uses: home-assistant/builder@master
|
||||
with:
|
||||
args: |
|
||||
--all \
|
||||
--target dev/hassio-google-drive-backup
|
||||
@@ -0,0 +1,54 @@
|
||||
name: run-pytest
|
||||
on:
|
||||
push:
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
jobs:
|
||||
run-pytest:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: '3.8'
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r .devcontainer/requirements-dev.txt
|
||||
- name: Install fping
|
||||
run: sudo apt-get install fping
|
||||
- name: Lint with flake8
|
||||
run: |
|
||||
# stop the build if there are Python syntax errors or undefined names
|
||||
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
|
||||
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
|
||||
flake8 . --count --exit-zero --max-line-length=300 --statistics
|
||||
- name: Test with pytest
|
||||
run: |
|
||||
pip install pytest
|
||||
pip install pytest-cov
|
||||
pytest hassio-google-drive-backup/tests --junitxml=junit/test-results.xml --cov=hassio-google-drive-backup/backup --cov-report=xml --cov-report=html
|
||||
- name: Upload pytest test results
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: pytest-results
|
||||
path: junit/test-results.xml
|
||||
if: ${{ always() }}
|
||||
- name: Publish Unit Test Results
|
||||
uses: EnricoMi/publish-unit-test-result-action@v2
|
||||
if: always()
|
||||
with:
|
||||
files: junit/**/*.xml
|
||||
- name: Upload coverage to Codecov
|
||||
uses: codecov/codecov-action@v3.1.4
|
||||
with:
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
files: ./coverage.xml
|
||||
directory: ./coverage/reports/
|
||||
flags: unittests
|
||||
env_vars: OS,PYTHON
|
||||
name: codecov-umbrella
|
||||
fail_ci_if_error: true
|
||||
path_to_write_report: ./coverage/codecov_report.txt
|
||||
verbose: true
|
||||
@@ -0,0 +1,43 @@
|
||||
name: Server Image Push
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
IMAGE_NAME: addon-server
|
||||
|
||||
jobs:
|
||||
# Push image to GitHub Packages.
|
||||
# See also https://docs.docker.com/docker-hub/builds/
|
||||
push:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
packages: write
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Build image
|
||||
run: docker build -f hassio-google-drive-backup/Dockerfile-server --tag $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}" hassio-google-drive-backup/.
|
||||
|
||||
- name: Log in to registry
|
||||
# This is where you will update the PAT to GITHUB_TOKEN
|
||||
run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
|
||||
|
||||
- name: Push image
|
||||
run: |
|
||||
IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
|
||||
|
||||
# Change all uppercase to lowercase
|
||||
IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
|
||||
# Strip git ref prefix from version
|
||||
VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
|
||||
# Strip "v" prefix from tag name
|
||||
[[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
|
||||
# Use Docker `latest` tag convention
|
||||
[ "$VERSION" == "master" ] && VERSION=latest
|
||||
echo IMAGE_ID=$IMAGE_ID
|
||||
echo VERSION=$VERSION
|
||||
docker tag $IMAGE_NAME $IMAGE_ID:run-$GITHUB_RUN_NUMBER
|
||||
docker push $IMAGE_ID:run-$GITHUB_RUN_NUMBER
|
||||
@@ -0,0 +1,36 @@
|
||||
name: push_to_staging
|
||||
concurrency:
|
||||
# Only run this workflow one at a time, and cancel any in progress
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
branches:
|
||||
- dev
|
||||
inputs:
|
||||
name:
|
||||
# Friendly description to be shown in the UI instead of 'name'
|
||||
description: 'Staging Verison Override'
|
||||
# Default value if no value is explicitly provided
|
||||
default: 'increment'
|
||||
|
||||
jobs:
|
||||
run-pytest:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: '3.8'
|
||||
# Check out the current branch
|
||||
- uses: actions/checkout@v3
|
||||
with:
|
||||
path: dev
|
||||
# Check out the staging barnch
|
||||
- uses: actions/checkout@v3
|
||||
with:
|
||||
path: staging
|
||||
repository: sabeechen/hgdb-dev-staging
|
||||
- run: |
|
||||
python3 staging/update.py dev staging
|
||||
@@ -0,0 +1,55 @@
|
||||
name: push_to_staging
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- dev
|
||||
|
||||
jobs:
|
||||
deploy_to_staging:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: '3.8'
|
||||
|
||||
- name: Check out dev repo
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
path: dev
|
||||
persist-credentials: false
|
||||
|
||||
- name: Checkout Staging Repo
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
path: staging
|
||||
repository: sabeechen/hgdb-dev-staging
|
||||
persist-credentials: true
|
||||
token: ${{ secrets.STAGING_REPO_TOKEN }}
|
||||
|
||||
- name: Update addon verison number
|
||||
run: |
|
||||
python3 staging/update.py dev staging
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@v2.2.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }}
|
||||
|
||||
- name: Publish Staging Addon image
|
||||
uses: home-assistant/builder@master
|
||||
with:
|
||||
args: |
|
||||
--all \
|
||||
--target dev/hassio-google-drive-backup
|
||||
|
||||
- name: Publish Staging Addon Version
|
||||
run: |
|
||||
cd staging
|
||||
git config user.name github-actions
|
||||
git config user.email github-actions@github.com
|
||||
git add .
|
||||
git commit -m "Updating staging addon config"
|
||||
git push
|
||||
@@ -0,0 +1,22 @@
|
||||
*.pyc
|
||||
*.dat
|
||||
*.id
|
||||
*pycache*
|
||||
.pytest_cache/*
|
||||
htmlcov/*
|
||||
.mypy_cache/*
|
||||
*drive_creds*
|
||||
dev/backup/*.tar
|
||||
hassio-google-drive-backup/backup/__pycache__/server.cpython-37.pyc.16291344
|
||||
.vscode/tags
|
||||
dev/error.py
|
||||
dev/data/retained.json
|
||||
*.egg-info
|
||||
hassio-google-drive-backup/dev/data/id.json
|
||||
hassio-google-drive-backup/dev/data/stop_addon_state.json
|
||||
hassio-google-drive-backup/dev/data/retained.json
|
||||
junit/test-results.xml
|
||||
.coverage
|
||||
coverage.xml
|
||||
hassio-google-drive-backup/dev/data/data_cache.json
|
||||
hassio-google-drive-backup/dev/data/*.backup
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"printWidth": 120,
|
||||
"trailingComma": "all",
|
||||
"quoteProps": "preserve",
|
||||
"singleQuote": false,
|
||||
"overrides": [
|
||||
{
|
||||
"files": "*.jinja2",
|
||||
"options": {
|
||||
"printWidth": 180
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"recommendations": ["wholroyd.jinja"]
|
||||
}
|
||||
+152
@@ -0,0 +1,152 @@
|
||||
{
|
||||
// Use IntelliSense to learn about possible attributes.
|
||||
// Hover to view descriptions of existing attributes.
|
||||
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
|
||||
"version": "0.2.0",
|
||||
"configurations": [
|
||||
{
|
||||
"name": "Run Addon (Dev Backends)",
|
||||
"type": "python",
|
||||
"request": "launch",
|
||||
"python": "${command:python.interpreterPath}",
|
||||
"module": "hassio-google-drive-backup.backup",
|
||||
"args": ["--config", "hassio-google-drive-backup/dev/data/dev_options.json"],
|
||||
"env": {
|
||||
"PYTHONPATH": "${workspaceFolder}/hassio-google-drive-backup"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Run Addon (Dev Drive)",
|
||||
"type": "python",
|
||||
"request": "launch",
|
||||
"python": "${command:python.interpreterPath}",
|
||||
"module": "hassio-google-drive-backup.backup",
|
||||
"args": ["--config", "hassio-google-drive-backup/dev/data/drive_dev_options.json"],
|
||||
"env": {
|
||||
"PYTHONPATH": "${workspaceFolder}/hassio-google-drive-backup"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Run Addon (Real Google Drive)",
|
||||
"type": "python",
|
||||
"request": "launch",
|
||||
"python": "${command:python.interpreterPath}",
|
||||
"module": "hassio-google-drive-backup.backup",
|
||||
"args": ["--config", "hassio-google-drive-backup/dev/data/drive_options.json"],
|
||||
"env": {
|
||||
"PYTHONPATH": "${workspaceFolder}/hassio-google-drive-backup"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Run Mock Backend Server",
|
||||
"type": "python",
|
||||
"request": "launch",
|
||||
"python": "${command:python.interpreterPath}",
|
||||
"module": "hassio-google-drive-backup.dev.simulationserver",
|
||||
"env": {
|
||||
"PYTHONPATH": "${workspaceFolder}/hassio-google-drive-backup"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Run Error Analyzer",
|
||||
"type": "python",
|
||||
"request": "launch",
|
||||
"python": "${command:python.interpreterPath}",
|
||||
"module": "hassio-google-drive-backup.backup.util.error_analyzer",
|
||||
"env": {
|
||||
"PYTHONPATH": "${workspaceFolder}/hassio-google-drive-backup"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Python: Current File (External Terminal)",
|
||||
"type": "python",
|
||||
"request": "launch",
|
||||
"program": "${file}",
|
||||
"console": "externalTerminal",
|
||||
"env": {
|
||||
"PYTHONPATH": "${workspaceFolder}/hassio-google-drive-backup"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Python Attach (Remote Debug docker addon)",
|
||||
"type": "python",
|
||||
"request": "attach",
|
||||
"pathMappings": [
|
||||
{
|
||||
"localRoot": "${workspaceFolder}/hassio-google-drive-backup", // You may also manually specify the directory containing your source code.
|
||||
"remoteRoot": "/app" // Linux example; adjust as necessary for your OS and situation.
|
||||
}
|
||||
],
|
||||
"port": 1627, // Set to the remote port.
|
||||
"host": "hassio", // Set to your remote host's public IP address.
|
||||
"env": {
|
||||
"PYTHONPATH": "${workspaceFolder}/hassio-google-drive-backup"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Run Auth Server",
|
||||
"type": "python",
|
||||
"request": "launch",
|
||||
"python": "${command:python.interpreterPath}",
|
||||
"module": "hassio-google-drive-backup.server",
|
||||
"env": {
|
||||
"PYTHONPATH": "${workspaceFolder}/hassio-google-drive-backup",
|
||||
"PORT": "12345",
|
||||
"CLIENT_SECRET": "client_secret",
|
||||
"CLIENT_ID": "client_id"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "(DEV) Build/Upload Addon Container",
|
||||
"type": "python",
|
||||
"request": "launch",
|
||||
"python": "${command:python.interpreterPath}",
|
||||
"module": "hassio-google-drive-backup.dev.deploy_dev_addon",
|
||||
"env": {
|
||||
"PYTHONPATH": "${workspaceFolder}/hassio-google-drive-backup",
|
||||
"PORT": "12345",
|
||||
"CLIENT_SECRET": "client_secret",
|
||||
"CLIENT_ID": "client_id"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "(PROD) Build/Upload Server Container",
|
||||
"type": "python",
|
||||
"request": "launch",
|
||||
"python": "${command:python.interpreterPath}",
|
||||
"module": "hassio-google-drive-backup.dev.deploy_server",
|
||||
"env": {
|
||||
"PYTHONPATH": "${workspaceFolder}/hassio-google-drive-backup",
|
||||
"PORT": "12345",
|
||||
"CLIENT_SECRET": "client_secret",
|
||||
"CLIENT_ID": "client_id"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "(DEV) Build/Upload Server Container",
|
||||
"type": "python",
|
||||
"request": "launch",
|
||||
"python": "${command:python.interpreterPath}",
|
||||
"module": "hassio-google-drive-backup.dev.deploy_dev_server",
|
||||
"env": {
|
||||
"PYTHONPATH": "${workspaceFolder}/hassio-google-drive-backup",
|
||||
"PORT": "12345",
|
||||
"CLIENT_SECRET": "client_secret",
|
||||
"CLIENT_ID": "client_id"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "(PROD) Build/Upload Addon Containers",
|
||||
"type": "python",
|
||||
"request": "launch",
|
||||
"python": "${command:python.interpreterPath}",
|
||||
"module": "hassio-google-drive-backup.dev.deploy_addon",
|
||||
"env": {
|
||||
"PYTHONPATH": "${workspaceFolder}/hassio-google-drive-backup",
|
||||
"PORT": "12345",
|
||||
"CLIENT_SECRET": "client_secret",
|
||||
"CLIENT_ID": "client_id"
|
||||
}
|
||||
},
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"python.testing.unittestEnabled": false,
|
||||
"python.testing.nosetestsEnabled": false,
|
||||
"python.testing.pytestEnabled": true,
|
||||
"python.linting.enabled": true,
|
||||
"python.linting.flake8Enabled": true,
|
||||
"python.linting.pylintEnabled": false,
|
||||
"python.linting.mypyEnabled": false,
|
||||
"python.linting.mypyArgs": ["--python-version", "3.8"],
|
||||
"python.linting.flake8Args": ["--ignore=E501,E731", "--verbose"],
|
||||
"python.autoComplete.addBrackets": true,
|
||||
"files.associations": {
|
||||
"*.jinja2": "html"
|
||||
},
|
||||
"[html]": {
|
||||
"editor.defaultFormatter": "vscode.html-language-features",
|
||||
"editor.tabSize": 2,
|
||||
},
|
||||
"python.analysis.completeFunctionParens": true,
|
||||
"python.analysis.typeCheckingMode": "basic"
|
||||
}
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
{
|
||||
// See https://go.microsoft.com/fwlink/?LinkId=733558 for help with task definitions
|
||||
"version": "2.0.0",
|
||||
"tasks": [
|
||||
{
|
||||
"label": "PUBLISH: New version to Docker Hub (requires working docker)",
|
||||
"type": "shell",
|
||||
"command": "./deploy.sh",
|
||||
"problemMatcher": []
|
||||
},
|
||||
{
|
||||
"label": "Install addon libraries",
|
||||
"type": "process",
|
||||
"command": "${config:python.pythonPath}",
|
||||
"args": ["-m", "pip", "install", "-e", "hassio-google-drive-backup/"],
|
||||
"problemMatcher": [],
|
||||
"group": {
|
||||
"kind": "build",
|
||||
"isDefault": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"label": "Install development dependencies",
|
||||
"type": "process",
|
||||
"command": "${config:python.pythonPath}",
|
||||
"args": ["-m", "pip", "install", "-r", ".devcontainer/requirements-dev.txt"],
|
||||
"problemMatcher": [],
|
||||
"group": {
|
||||
"kind": "build",
|
||||
"isDefault": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"label": "[Re]create and run local addon",
|
||||
"type": "process",
|
||||
"command": "${config:python.pythonPath}",
|
||||
"args": ["-m", "hassio-google-drive-backup.dev.deploy_local_addon"],
|
||||
"problemMatcher": [],
|
||||
"group": {
|
||||
"kind": "build",
|
||||
"isDefault": true
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
# Contributing
|
||||
|
||||
## About the project
|
||||
|
||||
The project is mostly maintained by Stephen Beechen (stephen@beechens.com) whom you can reach out to for guidance. Before digging in to this, you might be helpful to familiarize yourself with some of the technologies used in the project.
|
||||
|
||||
- [Developing Addons for Home Assistant](https://developers.home-assistant.io/docs/add-ons) - Useful to understand how addons work.
|
||||
- [Python](https://www.python.org/) - The addon is written in Python 3.8 and makes heavy use of the asyncio framework.
|
||||
- [AIOHTTP](https://docs.aiohttp.org/en/stable/) - The addon serves its web interface through an AIOHTTP server, and uses the AIOHTTP client library for all web requests.
|
||||
- [pytest](https://docs.pytest.org/en/latest/) - The addon uses pytest for all of its test.
|
||||
- [Visual Studio Code](https://code.visualstudio.com/) - The addon codebase is designed to work with Visual Studio code, but in practice you could use any editor (it would be harder). These instructions assume you're using VSCode, it’s a free cross-platform download.
|
||||
- [Docker](https://www.docker.com/) - All Home Assistant addons run in their own Docker container, and while you could certainly contribute without knowing much about it, knowledge of the basic commands will help.
|
||||
|
||||
## Approval Process
|
||||
- Please only make PR's against the [dev branch](https://github.com/sabeechen/hassio-google-drive-backup/tree/dev). Making a PR against master/main will result in an embarrassing song-and-dance where I ignore your PR for a little while, then ask you to remake it against dev, then ignore it again for a little while out of spite. Neither of us wants this, and you can avoid it by making it against dev in the first place.
|
||||
- If you're making a small change that fixes a bug I'm going to approve your PR quickly and heap you with praise. If you make a huge change without talking to me first I'm going to review your PR slowly and move through it with suspicion. A spectrum exists between those two extremes. Please try to understand that I'm the one ultimately on the line for the addon's reputation.
|
||||
- Breaking up a large change into smaller manageable pieces make things easier.
|
||||
- You can reach out to me in any of these ways to talk about a change you're considering:
|
||||
- Preferred: [File an issue on github](https://github.com/sabeechen/hassio-google-drive-backup/issues) proposing your changes.
|
||||
- Next best: Email: stephen@beechens.com
|
||||
- Acceptable but worst: Home Assistant Forums: [@sabeechen](https://community.home-assistant.io/u/sabeechen/summary)
|
||||
- Any submissions to the dev branch get automatically built and pushed to a staging version of the addon that you can install using [this repository](https://github.com/sabeechen/hgdb-dev-staging). Its identical to the "Production" addon but talks to [https://dev.habackup.io](https://dev.habackup.io) instead of [https://habackup.io](https://habackup.io).
|
||||
- Releases of the addon are made as-needed for bug fixes and new features. If you've made a signifigant change to the addon, you can expect me to communicate to you when you can expect to see it released. Important fixes will often demand an out-of-schedule rushed release.
|
||||
## Setting up a Development Environment
|
||||
|
||||
### Easy: Using the Dev Container
|
||||
If the you open the repository folder in Visual Studio code with docker installed, it will notice that it provides a devcontainer configuration and ask you to open it. This is the easiest method to use, sets up all the necessary plugins and dependencies, and its how I (the primary maintainer of the project) develop the addon.
|
||||
|
||||
### Harder but also works: Manual Setup
|
||||
1. Install [Visual Studio Code](https://code.visualstudio.com/)
|
||||
2. Install [Python 3.8](https://www.python.org/downloads/) for your platform.
|
||||
3. Install a git client. I like [GitHub Desktop](https://desktop.github.com/)
|
||||
4. Clone the project repository
|
||||
```
|
||||
https://github.com/sabeechen/hassio-google-drive-backup.git
|
||||
```
|
||||
5. Open Visual studio Code, go to the extension menu, and install the Desktop] (Python extension from Microsoft. It may prompt you to choose a Python interpreter (you want Python 3.8) and select a test framework (you want pytest).
|
||||
6. <kbd>File</kbd> > <kbd>Open Folder</kbd> to open the cloned repository folder.
|
||||
7. Open the terminal (`Ctrl` + `Shift` + <code>`</code>) and install the Python packages required for development:
|
||||
```
|
||||
> python3.8 -m pip install -r .devcontainer/requirements-dev.txt
|
||||
```
|
||||
That should set you up!
|
||||
|
||||
## Helpful Pointers
|
||||
|
||||
Here are some pointers about how things work that might get you to where you want to get faster:
|
||||
|
||||
- Constructor dependencies are handled through dependency injection. You can look at the attributes defined on most any class or constructor to see how they should be defined.
|
||||
- The project has almost **100% test coverage** and the expectation for all submissions (including my own) is that they will not lower that number. If you change something, your PR **must** include tests that cover it. The only exception is all the javascript, which has no unit tests.
|
||||
- The web server for the addon is in `uiserver.py`.
|
||||
- You'll want to make your changes to the `dev` branch, since the `master` branch is where new releases are made.
|
||||
|
||||
## Trying Out Changes
|
||||
|
||||
To try out changes locally during development, I've written a server that simulates Home Assistant, Supervisor, habackup.io, and Google Drive HTTP endpoints that the addon expects in [simulationserver.py](https://github.com/sabeechen/hassio-google-drive-backup/blob/master/hassio-google-drive-backup/dev/simulationserver.py). It’s a beast of a class and does a lot. It simulates the services for development and is also used to make unit tests work.
|
||||
|
||||
To give it a shot, open up Visual Studio's "Run" Dialog and start up `Run Mock Backend Server`. Then also run one of these options:
|
||||
|
||||
- `Run Addons (Dev Backends)` - This starts up the addon web server and connects it to the simulated Home Assistant, Supervisor, and Google Drive. All of the functionality of the addon is supported (creating/deleting backups, authenticating with Google drive, etc.).
|
||||
- `Run Addons (Dev Drive)` - This should be unused by contributors, as its only used for testing prior to a release by @sabeechen.
|
||||
- `Run Addons (Real Drive)` - This uses a simulated Home Assistant and Supervisor, but connects to the real Google Drive. You'll have to use a real Google account to work with this configuration.
|
||||
|
||||
## The Staging Addon
|
||||
Any submissions made to the dev branch (including PR's) get automatically built and deployed to a staging version of the addon. You can install this by adding the repository [https://github.com/sabeechen/hgdb-dev-staging](https://github.com/sabeechen/hgdb-dev-staging) to your home assistant machine. This addon is identical to what will be released with the next version of the addon but:
|
||||
- It is a separate "App" in Google's perspective, so it can't see any backups created by the "Production" addon.
|
||||
- Its not reocmmended to run it along side the "Production" addon on the same machine (it see's the same backups).
|
||||
- It talks to [https://dev.habackup.io](https://dev.habackup.io) instead of [https://habackup.io](https://habackup.io) to authenticate with Google Drive.
|
||||
- If you submit code to the dev branch, you should see an update to the addon show up in Home Assistant ~25 minutes later.
|
||||
- It is the "bleeding edge" of changes, so it might have bugs. Be warned!
|
||||
|
||||
For some changes, just testing locally might not be enough, you may want to run it as a real addon. You can do this roughly following the instruction for [Add-on Testing](https://developers.home-assistant.io/docs/add-ons/testing#local-build). Here are the two methods I've found work best:
|
||||
|
||||
- ### Building a Local Addon Container in Home Assistant
|
||||
Copy the folder `hassio-google-drive-backup` (the one with `config.json` inside it) into the local addon folder (you'll need the samba addon or something similar to do so). Modify the uploaded `config.json` to remove the `"image"` line near the bottom. Then in Home Assistant Web-UI go to <kbd>Supervisor</kbd> -> <kbd>Addon-Store</kbd>, <kbd>Reload</kbd>, and the addon should show up under "Local Addons". It should include buttons for building the container, starting/stopping etc.
|
||||
- ### Building a container
|
||||
You could also build the container as a docker container locally, upload it to Docker Hub, and then have Home Assistant download the image. First install docker desktop, then:
|
||||
```bash
|
||||
> cd hassio-google-drive-backup
|
||||
> docker login
|
||||
> docker build -f Dockerfile-addon -t YOUR_DOCKER_USERNAME/hassio-google-drive-backup-amd64:dev_testing --build-arg BUILD_FROM=homeassistant/amd64-base .
|
||||
> docker push YOUR_DOCKER_USERNAME/hassio-google-drive-backup-amd64:dev-testing
|
||||
```
|
||||
Then make a folder in the local addon directory like before, but only copy in config.json. change these two keys in config.json to match what you uploaded:
|
||||
```json
|
||||
{
|
||||
"image": "YOUR_DOCKER_USERNAME/hassio-google-drive-backup-{arch}",
|
||||
"version": "dev-testing"
|
||||
}
|
||||
```
|
||||
From there you should be able to see the addon in local addons, and installing will download the container from Docker Hub. To make it see changes, you'll need to rebuild and reupload the container, then uninstall and reinstall the addon in Home Assistant. I've found this to be faster than rebuilding the image from scratch within Home Assistant.
|
||||
> Note: Make sure you stop any other versions of the installed addon in Home Assistant before starting it as a local addon.
|
||||
|
||||
I haven't tried using the Supervisor's new devcontainers for development yet (the addon predates this), let me know if you can get that working well.
|
||||
|
||||
## Running Tests
|
||||
|
||||
You should be able to run tests from within the Visual Studio tests tab. Make sure all the tests pass before you to make a PR. You can also run them from the command line with:
|
||||
|
||||
```bash
|
||||
> python3.8 -m pytest hassio-google-drive-backup
|
||||
```
|
||||
|
||||
## Writing Tests
|
||||
|
||||
Test dependencies get injected by `pytest`, which are defined in the [conftest.py](https://github.com/sabeechen/hassio-google-drive-backup/blob/master/hassio-google-drive-backup/tests/conftest.py) file. This is responsible for starting the simulation server, mocking necessary classes, etc.
|
||||
Most classes have their own test file in the [tests](https://github.com/sabeechen/hassio-google-drive-backup/tree/master/hassio-google-drive-backup/tests) directory. If you change anything in the code, you must also submit tests with your PR that verify that change. The only exception is that all the addon's JavaScript, I've never found a good way to do JavaScript tests.
|
||||
@@ -0,0 +1,22 @@
|
||||
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2022 Stephen Beechen
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,55 @@
|
||||
# Using Custom/Personal Google Credentials
|
||||
You've arrived here because you'd like to use your own client ID and client secret to authenticate the add-on with Google Drive. I'll caution that this is a very detailed and complicated process geared more toward developers than end users, so if you'd like to do it the easy way, go back to your add-on (typically http://homeassistant.local:8123/hassio/ingress/hassio_google_drive_backup) and click the "Authenticate with Google Drive" button. These instructions will have you create a project on Google's Developer Cloud console, generate your own credentials, and use them to authenticate with Google Drive. You can expect this to take about 15 minutes. Typically this is what would be done by a developer when releasing a project that several users would use, but in this case you will be the only user. This workflow is for you if:
|
||||
* You'd like to avoid having your account's credentials go through a server maintained by the developer of this addon. The typical authentication workflow never sees your Google account password, but it does receive a token from Google that, if I were malicious, I could use to see the backups you've uploaded to Google Drive. I don't store this token anywhere and instead just pass it back to you, but because of how Google OAuth tokens are generated there is no way you could verify that. I tip my tinfoil hat to yours and respect your desire to protect your personal information :)
|
||||
* The typical authentication flow didn't work. This may be because of a bug, or because the server I set up to handle it is down or broken. Its just me back here providing this as a free service to the community, so apologies if things fall into disrepair.
|
||||
|
||||
These instructions are current as of June 2022. If you do this and notice they're out of date, Please file an issue on this project's issue page so I can be made aware of it and make the changes necessary. Thanks!
|
||||
|
||||
## Step 0 - Check addon version
|
||||
You must be running version 0.106.1 or greater of the add-on for this to work. In Feb 2022 Google changed how some of their authentication APIs work which broke the way the addon did it before that version
|
||||
## Step 1 - Create a Google Cloud Project
|
||||
>**NOTE:** If you already followed these steps in the past, you do not need to create a new project. Just use the one you used before.
|
||||
* Go to http://console.developers.google.com and log in with your Google account.
|
||||
* Click "Select Project" on the top left.
|
||||
* Click "New Project" to create a project.
|
||||
* Give the project any name you like, and click "Create Project". Don't worry about billing or location information, you won't be charged for anything we're doing here.
|
||||
|
||||
|
||||
## Step 2 - Enable the Drive API
|
||||
|
||||
>**NOTE:** If you already followed these steps in the past, the API shoudl already be enabled
|
||||
|
||||
With your project now created:
|
||||
* Go to https://console.developers.google.com/apis/library
|
||||
* Search for "Google Drive API", and click "Enable". This is necessary because the "Project" you're creating will use the [Google Drive API](https://developers.google.com/drive/api/v3/reference).
|
||||
|
||||
## Step 3 - Create a Consent Screen
|
||||
>**NOTE:** If you already followed these steps in the past, you probably already have a consent screen. Reuse your existing one and just make sure it matches the information described below. In particular ensure the consent screen in "Publishing status" is "In production"
|
||||
|
||||
Before creating credentials, you'll need to create a consent screen. Normally this is what people would see when they request to allow your new application to access their Google Drive, but because you're creating it just for yourself this is basically just a necessary formality.
|
||||
* Go back to http://console.developers.google.com and ensure the project name you created earlier is displayed in the upper left.
|
||||
* In the menu on the upper left, click **Enabled APIs & Services** then *OAuth Consent Screen*.
|
||||
* Select *External* for the user type and then click "Create". Even though you're probably making these credentials with the same account you'll be using to authenticate the addon, you'll still be considered an *External* user.
|
||||
* On the next screen "App Information", fill in all the required fields, *App Name*, *Support Email*, and *Developer Email*. Then click "Save & Continue". What you enter here doesn't really matter, but a good App Name is something that will make you laugh if you ever have to see this again, like "Buy the name-brand SD Card this time, maybe?"
|
||||
* On the next screen, click **Add OR Remove Scopes**. In the dialog that pops up check the box for "../auth/drive.file" and then click "Update". You might have to search for "drive.file" to make it show up. This part is very important since it gives the credentials we're about to create permission to see files in Google Drive. If you don't see this in the dialog that comes up, make sure you did step 2.
|
||||
* You can leave the rest of this form blank, just click **Save** or **Continue** for any other screens.
|
||||
* Once its created, either click **Go Back to Dashboard** or click **OAuth Consent Screen** on the left. Under **Publishing status** click **Publish App** and then **Confirm**. This dialog will warn that the app will be available to all users, but in our case it will still only be you if you keep the credentials you create later just to yourself. This step is necessary because "Testing" credentials would require you to manually re-authorize the addon ever 7 days, which is a pain.
|
||||
|
||||
## Step 4 - Create Credentials
|
||||
>**NOTE:** If you already followed these steps in the past, you probably already have credentials. If they don't whats described below, you'll need to create a new ones. In particular ensure the credential type is "TVs and Limited Input Devices". So long as you use the same consent screen, you can create any number of credentials you want.
|
||||
|
||||
Now you've set up everything necessary to actually create credentials.
|
||||
* From http://console.developers.google.com, click **Enabled APIs & Services** then **Credentials** on the left.
|
||||
* Click **+ Create Credentials** at the top of the page.
|
||||
* Select "OAuth client ID" form the drop down.
|
||||
This should have opened a dialog titled "Create OAuth client ID".
|
||||
* Select **TVs and Limited Input Devices** for **Application Type**. Home Assistant might not seem like a "Limited Input Device" but is is necessary because its the only OAuth authentication method Google provides that doesn't require you to maintain a public SSL encrypted web service.
|
||||
* Give the credentials a **Name**, anything will do and it doesn't matter.
|
||||
|
||||
* Click "Create"
|
||||
|
||||
|
||||
## Step 5 - Copy your credentials
|
||||
This should have opened a new dialog with your generated client ID and client secret. Take these back to the Add-on, and paste them into the appropriate fields of the add-on web-UI, and follow the instructions from there.
|
||||
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
## Contributing with Cryptocurrency
|
||||
Below are the addresses and relevant QR Codes for contributiong to the project with Monreo, Ethereum and Bitcoin. Thank you for considering! If you'd like to donate using a currency not listed, make an [issue](https://github.com/sabeechen/hassio-google-drive-backup/issues) so I can consider it. Cryptocurrency might be the purest way to contribute, because I have no way of knowing who you are. Your anonymous support is appreciated <3
|
||||
<br><br>
|
||||
### <img src="images/monero-button.svg" width="200" /><br>
|
||||
<img src="images/monero-qr.svg" width="200" />
|
||||
|
||||
Monero/XMR Address:
|
||||
```
|
||||
8BtRhV9vUNkTDST7WEWuH7JumpvrFApVag5D2fVTLgfdJH8sjiy5LQo68WK4GaLBijTYb7XW6D6bChQGkDeDMmfTTTZGNTr
|
||||
```
|
||||
|
||||
<br><br>
|
||||
### <img src="images/ethereum-button.svg" width="200" /><br>
|
||||
<img src="images/ethereum-qr.svg" width="200" />
|
||||
|
||||
Ethereum/ETH Address:
|
||||
```
|
||||
0xfa455Bdd245F67Dd1D549C2Ac67EA6BBf91402a7
|
||||
```
|
||||
|
||||
|
||||
<br><br>
|
||||
### <img src="images/bitcoin-button.svg" width="200" /><br>
|
||||
<img src="images/bitcoin-qr.svg" width="200" />
|
||||
|
||||
Bitcoin/BTC Address:
|
||||
```
|
||||
bc1qystpwsqqwusam38mhwafh8n4yln798zqprsl93
|
||||
```
|
||||
@@ -0,0 +1,44 @@
|
||||
# Authentication with Google Drive
|
||||
This document describes how the addon (Home Assistant Google Drive Backup) authenticates with Google Drive and stores your credentials. It's geared toward those who wish to know more detail and is not necessary to take advantage of the full features of the addon. The document is provided in the interest of providing full transparency into how the add-on works. I've tried to describe this as plainly as possible, but it is technical and therefore may not be understandable to everyone. Feedback on its clarity is appreciated.
|
||||
|
||||
> This document describes how authentication works if you use the big blue "AUTHENTICATE WITH GOOGLE DRIVE" button in the addon. If you're using [your own Google Drive credentials](https://github.com/sabeechen/hassio-google-drive-backup/blob/master/LOCAL_AUTH.md), then none of this applies.
|
||||
|
||||
## Your Credentials and the Needed Permission
|
||||
To have access to any information in Google Drive, Google's authentication servers must be told that the add-on has the permission. The add-on uses [Google Drive's Rest API (v3)](https://developers.google.com/drive/api/v3/about-sdk) for communication and requests the [drive.file](https://developers.google.com/drive/api/v3/about-auth) permission *scope*. This *scope* means the add-on has access to files and folders that the add-on created, but nothing else. It can't see files you've added to Google Drive through their web interface or anywhere else. Google Drive's Rest API allows the addon to periodically check what backups are uploaded and upload new ones if necessary by making requests over the internet.
|
||||
|
||||
## Authentication with Google Services
|
||||
For reference, Google's documentation for how to authenticate users with the Google Drive REST API is [here](https://developers.google.com/drive/api/v3/about-auth). Authentication is handled through [OAuth 2.0](https://developers.google.com/identity/protocols/OAuth2), which means that the add-on never actually sees your Google username and password, only an opaque [security token](https://en.wikipedia.org/wiki/Access_token) used to verify that the addon has been given permission. More detail is provided about what that token is and where it is stored later in this document.
|
||||
|
||||
The way a web-based application would normally authenticate with a Google service (eg Google Drive) looks something like this:
|
||||
1. User navigates to the app's webpage, eg http://examplegoogleapp.com
|
||||
2. The app generates a URL to Google's servers (https://accounts.google.com) used to grant the app permission.
|
||||
3. User navigates there, enters their Google username and password, and confirms the intention to give the app some permission (eg one or more *scopes*).
|
||||
4. Google redirects the user back to the app's webpage with an access token appended to the URL (eg http://examplegoogleapp.com/authenticate?token=0x12345678)
|
||||
5. The app stores the access token (0x12345678 in this example), and then passes it back to Google whenever it wishes to make access the API on behalf of the user who logged in.
|
||||
|
||||
This access token allows the app to act as if it is the user who created it. In the case of this add-on, the permission granted by the drive.file scope allows it to create folders, upload backups, and retrieve the previously created folders. Because the add-on only ever sees the access token (not the username/password), and the access token only grants limited permissions, the add-on doesn't have a way to elevate its permission further to access other information in Google Drive or your Google account.
|
||||
|
||||
## Authentication for the Add-on
|
||||
|
||||
Google puts some limitations on how the access token must be generated that will be important for understanding how the add-on authenticates in reality:
|
||||
* When the user is redirected to https://accounts.google.com (step 2), the redirect must be from a known public website associated with the app.
|
||||
* When the user is redirected back to the app after authorization (step 4), the redirect must be a statically addressed and publicly accessible website.
|
||||
|
||||
These limitations make a technical problem for the addon because most people's Home Assistant instances aren't publicly accessible and the address is different for each one. Performing the authentication workflow exactly as described above won't work. To get around this, I (the developer of this addon) set up a website, https://habackup.io, which serves as the known public and statically addressable website that Google redirects from/to. The source code for this server is available within the add-on's GitHub repository.
|
||||
|
||||
So when you authenticate the add-on, the workflow looks like this:
|
||||
1. You start at the add-on's web interface, something like https://homeassistant.local:8123/ingress/hassio_google_drive_backup
|
||||
2. You click the "Authenticate With Google Drive" button, which takes note of the address of your Home Assistant installation (https://homeassistant.local:8123 in this case) and sends you to https://habackup.io/drive/authorize
|
||||
3. https://habackup.io immediately generates the Google login URL for you and redirects you to https://accounts.google.com
|
||||
4. You log in with your Google credentials on Google's domain, and confirm you want to give the add-on permission to see files and folders it creates (the drive.file scope)
|
||||
5. Google redirects you back to https://habackup.io, along with the access token that will be used for future authentication.
|
||||
6. https://habackup.io redirects you back to your add-on web-UI (which is kept track of in step 2) along with the access token.
|
||||
7. The addon (on your local Home Assistant installation) persists the access token and uses it in the future any time it needs to talk to Google Drive.
|
||||
|
||||
Notably, your access token isn't persisted at https://habackup.io, it is only passed through back to your local add-on installation. I do this because:
|
||||
- It ensures your information is only ever stored on your machine, which is reassuring from the user's perspective (eg you).
|
||||
- If my server (https://habackup.io) ever gets compromised, there isn't any valuable information stored there that compromises you as well.
|
||||
- This is practicing a form of [defense-in-depth](https://en.wikipedia.org/wiki/Defense_in_depth_%28computing%29) security, where-in [personal data](https://en.wikipedia.org/wiki/Personal_data) is only stored in the places where it is strictly critical.
|
||||
- It makes the server more simple since it is a stateless machine that doesn't require a database (eg to store your token).
|
||||
|
||||
After your token is generated and stored on your machine, it needs to be *refreshed* periodically with Google Drive. To do this, the addon will again ask https://habackup.io who will relay the request with Google Drive.
|
||||
@@ -0,0 +1,123 @@
|
||||
# 'Snapshot' vs 'Backup'
|
||||
In August 2021 [the Home Assistant team announced](https://www.home-assistant.io/blog/2021/08/24/supervisor-update/) that 'snapshots' will be called 'backups' moving forward. This addon exposes a binary sensor to indicate if snapshots are stale and a another sensor that publishes details about backups. Both of the sensors used 'snapshot' in their names and values, so they had to be changed to match the new language. To prevent breaking any existing automations you might have, the addon will only start using the new names and values when you upgrade if you tell it to.
|
||||
|
||||
This can be controlled by using the configuration option ```call_backup_snapshot```, which will use the old names and values for sensors when it is true. If you updated the addon from a version that used to use 'snapshot' in it names, this option will be automatically added when you update to make sure it doesn't break any existing automations.
|
||||
|
||||
Here is a breakdown of what the new and old sensor values mean:
|
||||
|
||||
## Old sensor name/values
|
||||
These will be the sensor values used when ```call_backup_snapshot: True``` or if the addon is below version 0.105.1. The addon sets ```call_backup_snapshot: True``` automatically if you upgrade the addon from an older version.
|
||||
### Backup Stale Binary Sensor
|
||||
#### Entity Id:
|
||||
```yaml
|
||||
binary_sensor.snapshots_stale
|
||||
```
|
||||
#### Possible states:
|
||||
```yaml
|
||||
on
|
||||
off
|
||||
```
|
||||
#### Example Attributes:
|
||||
```yaml
|
||||
friendly_name: Snapshots Stale
|
||||
device_class: problem
|
||||
```
|
||||
### Backup State Sensor
|
||||
#### Entity Id:
|
||||
```yaml
|
||||
sensor.snapshot_backup
|
||||
```
|
||||
#### Possible States:
|
||||
```yaml
|
||||
error
|
||||
waiting
|
||||
backed_up
|
||||
```
|
||||
#### Example Attributes:
|
||||
```yaml
|
||||
friendly_name: Snapshots State
|
||||
last_snapshot: 2021-09-01T20:26:49.100376+00:00
|
||||
snapshots_in_google_drive: 2
|
||||
snapshots_in_hassio: 2
|
||||
snapshots_in_home_assistant: 2
|
||||
size_in_google_drive: 2.5 GB
|
||||
size_in_home_assistant: 2.5 GB
|
||||
snapshots:
|
||||
- name: Full Snapshot 2021-02-06 11:37:00
|
||||
date: '2021-02-06T18:37:00.916510+00:00'
|
||||
state: Backed Up
|
||||
slug: DFG123
|
||||
- name: Full Snapshot 2021-02-07 11:00:00
|
||||
date: '2021-02-07T18:00:00.916510+00:00'
|
||||
state: Backed Up
|
||||
slug: DFG124
|
||||
```
|
||||
|
||||
## New Sensor Names/Values
|
||||
These will be the sensor values used when ```call_backup_snapshot: False``` or if the configuration option is un-set. New installations of the addon will default to this.
|
||||
### Backup Stale Binary Sensor
|
||||
#### Entity Id
|
||||
```yaml
|
||||
binary_sensor.backups_stale
|
||||
```
|
||||
#### Possible States
|
||||
```yaml
|
||||
on
|
||||
off
|
||||
```
|
||||
#### Example Attributes:
|
||||
```yaml
|
||||
friendly_name: Backups Stale
|
||||
device_class: problem
|
||||
```
|
||||
### Backup State Sensor
|
||||
#### Entity Id
|
||||
```yaml
|
||||
sensor.backup_state
|
||||
```
|
||||
#### Possible States
|
||||
```yaml
|
||||
error
|
||||
waiting
|
||||
backed_up
|
||||
```
|
||||
#### Example Attributes:
|
||||
```yaml
|
||||
friendly_name: Backup State
|
||||
last_backup: 2021-09-01T20:26:49.100376+00:00
|
||||
last_upload: 2021-09-01T20:26:49.100376+00:00
|
||||
backups_in_google_drive: 2
|
||||
backups_in_home_assistant: 2
|
||||
size_in_google_drive: 2.5 GB
|
||||
size_in_home_assistant: 2.5 GB
|
||||
backups:
|
||||
- name: Full Snapshot 2021-02-06 11:37:00
|
||||
date: '2021-02-06T18:37:00.916510+00:00
|
||||
state: Backed Up
|
||||
slug: DFG123
|
||||
- name: Full Snapshot 2021-02-07 11:00:00
|
||||
date: '2021-02-07T18:00:00.916510+00:00'
|
||||
state: Backed Up
|
||||
slug: DFG124
|
||||
```
|
||||
|
||||
### What do the values mean?
|
||||
```binary_sensor.backups_stale``` is "on" when backups are stale and "off"" otherwise. Backups are stale when the addon is 6 hours past a scheduled backup and no new backup has been made. This delay is in place to avoid triggerring on transient errors (eg internet connectivity problems or one-off problems in Home Assistant).
|
||||
|
||||
```sensor.backup_state``` is:
|
||||
- ```waiting``` when the addon is first booted up or hasn't been connected to Google Drive yet.
|
||||
- ```error``` immediately after any error is encountered, even transient ones.
|
||||
- ```backed_up``` when everything is running fine without errors.
|
||||
|
||||
It's attributes are:
|
||||
- ```last_backup``` The UTC ISO-8601 date of the most recent backup in Home Assistant or Google Drive.
|
||||
- ```last_upload``` The UTC ISO-8601 date of the most recent backup uploaded to Google Drive.
|
||||
- ```backups_in_google_drive``` The number of backups in Google Drive.
|
||||
- ```backups_in_home_assistant``` The number of backups in Home Assistant.
|
||||
- ```size_in_google_drive``` A string representation of the space used by backups in Google Drive.
|
||||
- ```size_in_home_assistant``` A string representation of the space used by backups in Home Assistant.
|
||||
- ```backups``` The list of each snapshot in decending order of date. Each snapshot includes its ```name```, ```date```, ```slug```, and ```state```. ```state``` can be one of:
|
||||
- ```Backed Up``` if its in Home Assistant and Google Drive.
|
||||
- ```HA Only``` if its only in Home Assistant.
|
||||
- ```Drive Only``` if its only in Google Drive.
|
||||
- ```Pending``` if the snapshot was requested but not yet complete.
|
||||
@@ -0,0 +1,26 @@
|
||||
## v0.111.1 [2023-06-19]
|
||||
- Support for the new network storage features in Home Assistant. The addon will now create backups in what Home Assistant has configured as its default backup location. This can be overridden in the addon's settings.
|
||||
- Raised the addon's required permissions to "Admin" in order to access the supervisor's mount API.
|
||||
- Fixed a CSS error causing toast messages to render partially off screen on small displays.
|
||||
- Fixed misreporting of some error codes from Google Drive when a partial upload can't be resumed.
|
||||
|
||||
## v0.110.4 [2023-04-28]
|
||||
- Fix a whitespace error causing authorization to fail.
|
||||
|
||||
## v0.110.3 [2023-03-24]
|
||||
- Fix an error causing "Days Between Backups" to be ignored when "Time of Day" for a backup is set.
|
||||
- Fix a bug causing some timezones to make the addon to fail to start.
|
||||
|
||||
## v0.110.2 [2023-03-24]
|
||||
- Fix a potential cause of SSL errors when communicating with Google Drive
|
||||
- Fix a bug causing backups to be requested indefinitely if scheduled during DST transitions.
|
||||
|
||||
## v0.110.1 [2023-01-09]
|
||||
- Adds some additional options for donating
|
||||
- Mitgigates SD card corruption by redundantly storing config files needed for addon startup.
|
||||
- Avoid global throttling of Google Drive API calls by:
|
||||
- Making sync intervals more spread out and a little random.
|
||||
- Syncing more selectively when there are modifications to the /backup directory.
|
||||
- Caching data from Google Drive for short periods during periodic syncing.
|
||||
- Backing off for a longer time (2 hours) when the addon hits permanent errors.
|
||||
- Fixes CSS issues that made the logs page hard to use.
|
||||
@@ -0,0 +1,205 @@
|
||||
# Home Assistant Add-on: Google Assistant SDK
|
||||
|
||||
## Installation
|
||||
|
||||
To install the add-on, first follow the installation steps from the [README on GitHub](https://github.com/sabeechen/hassio-google-drive-backup#installation).
|
||||
|
||||
## Configuration
|
||||
|
||||
_Note_: The configuration can be changed easily by starting the add-on and clicking `Settings` in the web UI.
|
||||
The UI explains what each setting is and you don't need to modify anything before clicking `Start`.
|
||||
If you would still prefer to modify the settings in yaml, the options are detailed below.
|
||||
|
||||
### Add-on configuration example
|
||||
Don't use this directly, the addon has a lot of configuration options that most users don't need or want:
|
||||
|
||||
```yaml
|
||||
# Keep 10 backups in Home Assistant
|
||||
max_backups_in_ha: 10
|
||||
|
||||
# Keep 10 backups in Google Drive
|
||||
max_backups_in_google_drive: 10
|
||||
|
||||
# Create backups in Home Assistant on network storage
|
||||
backup_location: my_nfs_share
|
||||
|
||||
# Ignore backups the add-on hasn't created
|
||||
ignore_other_backups: True
|
||||
|
||||
# Ignore backups that look like they were created by Home Assistant automatic backup option during upgrades
|
||||
ignore_upgrade_backups: True
|
||||
|
||||
# Automatically delete "ignored" snapshots after this many days
|
||||
delete_ignored_after_days: 7
|
||||
|
||||
# Take a backup every 3 days
|
||||
days_between_backups: 3
|
||||
|
||||
# Create backups at 1:30pm exactly
|
||||
backup_time_of_day: "13:30"
|
||||
|
||||
# Delete backups from Home Assistant immediately after uploading them to Google Drive
|
||||
delete_after_upload: True
|
||||
|
||||
# Manually specify the backup folder used in Google Drive
|
||||
specify_backup_folder: true
|
||||
|
||||
# Use a dark and red theme
|
||||
background_color: "#242424"
|
||||
accent_color: "#7D0034"
|
||||
|
||||
# Use a password for backup archives. Use "!secret secret_name" to use a password form your secrets file
|
||||
backup_password: "super_secret"
|
||||
|
||||
# Create backup names like 'Full Backup HA 0.92.0'
|
||||
backup_name: "{type} Backup HA {version_ha}"
|
||||
|
||||
# Keep a backup once every day for 3 days and once a week for 4 weeks
|
||||
generational_days: 3
|
||||
generational_weeks: 4
|
||||
|
||||
# Create partial backups with no folders and no configurator add-on
|
||||
exclude_folders: "homeassistant,ssl,share,addons/local,media"
|
||||
exclude_addons: "core_configurator"
|
||||
|
||||
# Turn off notifications and staleness sensor
|
||||
enable_backup_stale_sensor: false
|
||||
notify_for_stale_backups: false
|
||||
|
||||
# Enable server directly on port 1627
|
||||
expose_extra_server: true
|
||||
|
||||
# Allow sending error reports
|
||||
send_error_reports: true
|
||||
|
||||
# Delete backups after they're uploaded to Google Drive
|
||||
delete_after_upload: true
|
||||
```
|
||||
|
||||
### Option: `max_backups_in_ha` (default: 4)
|
||||
|
||||
The number of backups the add-on will allow Home Assistant to store locally before old ones are deleted.
|
||||
|
||||
### Option: `max_backups_in_google_drive` (default: 4)
|
||||
|
||||
The number of backups the add-on will keep in Google Drive before old ones are deleted. Google Drive gives you 15GB of free storage (at the time of writing) so plan accordingly if you know how big your backups are.
|
||||
|
||||
### Option: `backup_location` (default: None)
|
||||
The place where backups are created in Home Assistant before uploading to Google Drive. Can be "local-disk" or the name of any backup network storage you've configured in Home Assistant. Leave unspecified (the default) to have backups created in whatever Home Assistant uses as the default backup location.
|
||||
|
||||
### Option: `ignore_other_backups` (default: False)
|
||||
Make the addon ignore any backups it didn't directly create. Any backup already uploaded to Google Drive will not be ignored until you delete it from Google Drive.
|
||||
|
||||
### Option: `ignore_upgrade_backups` (default: False)
|
||||
Ignores backups that look like they were automatically created from updating an add-on or Home Assistant itself. This will make the add-on ignore any partial backup that has only one add-on or folder in it.
|
||||
|
||||
### Option: `days_between_backups` (default: 3)
|
||||
|
||||
How often a new backup should be scheduled, eg `1` for daily and `7` for weekly.
|
||||
|
||||
### Option: `backup_time_of_day`
|
||||
|
||||
The time of day (local time) that new backups should be created in 24-hour ("HH:MM") format. When not specified backups are created at (roughly) the same time of day as the most recent backup.
|
||||
|
||||
|
||||
### Options: `delete_after_upload` (default: False)
|
||||
|
||||
Deletes backups from Home Assistant immediately after uploading them to Google Drive. This is useful if you have very limited space inside Home Assistant since you only need to have available space for a single backup locally.
|
||||
|
||||
### Option: `specify_backup_folder` (default: False)
|
||||
|
||||
When true, you must select the folder in Google Drive where backups are stored. Once you turn this on, restart the add-on and visit the Web-UI to be prompted to select the backup folder.
|
||||
|
||||
### Option: `background_color` and `accent_color`
|
||||
|
||||
The background and accent colors for the web UI. You can use this to make the UI fit in with whatever color scheme you use in Home Assistant. When unset, the interface matches Home Assistant's default blue/white style.
|
||||
|
||||
### Option: `backup_password`
|
||||
|
||||
When set, backups are created with a password. You can use a value from your secrets.yaml by prefixing the password with "!secret". You'll need to remember this password when restoring a backup.
|
||||
|
||||
> Example: Use a password for backup archives
|
||||
>
|
||||
> ```yaml
|
||||
> backup_password: "super_secret"
|
||||
> ```
|
||||
>
|
||||
> Example: Use a password from secrets.yaml
|
||||
>
|
||||
> ```yaml
|
||||
> backup_password: "!secret backup_password"
|
||||
> ```
|
||||
|
||||
### Option: `backup_name` (default: "{type} Backup {year}-{month}-{day} {hr24}:{min}:{sec}")
|
||||
|
||||
Sets the name for new backups. Variable parameters of the form `{variable_name}` can be used to modify the name to your liking. A list of available variables is available [here](https://github.com/sabeechen/hassio-google-drive-backup#can-i-give-backups-a-different-name).
|
||||
|
||||
### Option: `generational_*`
|
||||
|
||||
When set, older backups will be kept longer using a [generational backup scheme](https://en.wikipedia.org/wiki/Backup_rotation_scheme). See the [question here](https://github.com/sabeechen/hassio-google-drive-backup#can-i-keep-older-backups-for-longer) for configuration options.
|
||||
|
||||
### Option: `exclude_folders`
|
||||
|
||||
When set, excludes the comma-separated list of folders by creating a partial backup.
|
||||
|
||||
### Option: `exclude_addons`
|
||||
|
||||
When set, excludes the comma-separated list of addons by creating a partial backup.
|
||||
|
||||
_Note_: Folders and add-ons must be identified by their "slug" name. It is recommended to use the `Settings` dialog within the add-on web UI to configure partial backups since these names are esoteric and hard to find.
|
||||
|
||||
### Option: `enable_backup_stale_sensor` (default: True)
|
||||
|
||||
When false, the add-on will not publish the [binary_sensor.backups_stale](https://github.com/sabeechen/hassio-google-drive-backup#how-will-i-know-this-will-be-there-when-i-need-it) stale sensor.
|
||||
|
||||
### Option: `enable_backup_state_sensor` (default: True)
|
||||
|
||||
When false, the add-on will not publish the [sensor.backup_state](https://github.com/sabeechen/hassio-google-drive-backup#how-will-i-know-this-will-be-there-when-i-need-it) sensor.
|
||||
|
||||
### Option: `notify_for_stale_backups` (default: True)
|
||||
|
||||
When false, the add-on will send a [persistent notification](https://github.com/sabeechen/hassio-google-drive-backup#how-will-i-know-this-will-be-there-when-i-need-it) in Home Assistant when backups are stale.
|
||||
|
||||
---
|
||||
|
||||
### UI Server Options
|
||||
|
||||
The UI is available through Home Assistant [ingress](https://www.home-assistant.io/blog/2019/04/15/hassio-ingress/).
|
||||
|
||||
It can also be exposed through a web server on port `1627`, which you can map to an externally visible port from the add-on `Network` panel. You can configure a few more options to add SSL or require your Home Assistant username/password.
|
||||
|
||||
#### Option: `expose_extra_server` (default: False)
|
||||
|
||||
Expose the webserver on port `1627`. This is optional, as the add-on is already available with Home Assistant ingress.
|
||||
|
||||
#### Option: `require_login` (default: False)
|
||||
|
||||
When true, requires your home assistant username and password to access the Web UI.
|
||||
|
||||
#### Option: `use_ssl` (default: False)
|
||||
|
||||
When true, requires your home assistant username and password to access the Web UI.
|
||||
|
||||
#### Option: `certfile` (default: `/ssl/certfile.pem`)
|
||||
|
||||
Required when `use_ssl: True`. The path to your SSL key file
|
||||
|
||||
#### Option: `keyfile` (default: `/ssl/keyfile.pem`)
|
||||
|
||||
Required when `use_ssl: True`. The path to your SSL cert file.
|
||||
|
||||
#### Option: `verbose` (default: False)
|
||||
|
||||
If true, enable additional debug logging. Useful if you start seeing errors and need to file a bug with me.
|
||||
|
||||
#### Option: `send_error_reports` (default: False)
|
||||
|
||||
When true, the text of unexpected errors will be sent to a database maintained by the developer. This helps identify problems with new releases and provide better context messages when errors come up.
|
||||
|
||||
#### Option: `delete_after_upload` (default: False)
|
||||
|
||||
When true, backups are always deleted after they've been uploaded to Google Drive. 'max_backups_in_ha' is ignored when this option is True, since a backup is always deleted from Home Assistant after it gets uploaded to Google Drive. Some find this useful if they only have enough space on their Home Assistant machine for one backup.
|
||||
|
||||
## FAQ
|
||||
|
||||
Read the [FAQ on GitHub](https://github.com/sabeechen/hassio-google-drive-backup#faq).
|
||||
@@ -0,0 +1,12 @@
|
||||
ARG BUILD_FROM
|
||||
FROM $BUILD_FROM
|
||||
WORKDIR /app
|
||||
COPY . /app
|
||||
RUN chmod +x addon_deps.sh
|
||||
RUN ./addon_deps.sh
|
||||
RUN pip3 install .
|
||||
COPY config.json /usr/local/lib/python3.8/site-packages/config.json
|
||||
|
||||
EXPOSE 1627
|
||||
EXPOSE 8099
|
||||
ENTRYPOINT ["python3", "-m", "backup"]
|
||||
@@ -0,0 +1,16 @@
|
||||
# Use the official lightweight Python image.
|
||||
# https://hub.docker.com/_/python
|
||||
FROM python:3.9-buster
|
||||
|
||||
# Copy local code to the container image.
|
||||
ENV APP_HOME /server
|
||||
WORKDIR $APP_HOME
|
||||
COPY . ./
|
||||
COPY config.json /usr/local/lib/python3.9/site-packages/config.json
|
||||
|
||||
# Install server python requirements
|
||||
RUN pip3 install --trusted-host pypi.python.org -r requirements-server.txt
|
||||
RUN pip3 install .
|
||||
|
||||
WORKDIR /
|
||||
ENTRYPOINT ["python3", "-m", "backup.server"]
|
||||
@@ -0,0 +1,41 @@
|
||||
# Generational Backup
|
||||
Generational backup lets you keep a longer history of backups on daily, weekly, monthly, and yearly cycles. This is in contrast to the "regular" scheme for keeping history backups, which will always just delete the oldest backup when needed. This has the effect of keeping older backups around for a longer time, which is particularly useful if you've made a bad configuration change but didn't notice until several days later.
|
||||
|
||||
## Configuration
|
||||
The generational backup will be used when any one of `generational_days`, `generational_weeks`, `generational_months`, or `generational_years` is greater than zero. All of the available configuration options are given below, but utes much easier to configure from the Settings dialog accessible from the "Settings" menu at the top of the web UI.
|
||||
* `generational_days` (int): The number of days to keep
|
||||
* `generational_weeks` (int): The number of weeks to keep
|
||||
* `generational_months` (int): The number of months to keep
|
||||
* `generational_years` (int): The number of years to keep
|
||||
* `generational_day_of_week` (str): The day of the week when weekly backups will be kept. It can be one of 'mon', 'tue', 'wed', 'thu', 'fri', 'sat' or 'sun'. The default is 'mon'.
|
||||
* `generational_day_of_month` (int): The day of the month when monthly backups will be kept, from 1 to 31. If a month has less than the configured number of days, the latest day of that month is used.
|
||||
* `generational_day_of_year` (int): The day of the year that yearly backups are kept, from 1 to 365.
|
||||
|
||||
## Some Details to Consider
|
||||
* Generational backup assumes that a backup is available for every day to work properly, so it's recommended that you set `days_between_backups`=1 if you're using the feature. Otherwise, a backup may not be available to be saved for a given day.
|
||||
* The backups maintained by generational backup will still never exceed the number you permit to be maintained in Google Drive or Home Assistant. For example, if `max_backups_in_google_drive`=3 and `generational_weeks`=4, then only 3 weeks of backups will be kept in Google Drive.
|
||||
* Generational backup will only delete older backups when it has to. For example, if you've configured it to keep 5 weekly backups on Monday, you've been running it for a week (so you have 7 backups), and `max_backups_in_google_drive`=7, then your backups on Tuesday, Wednesday, etc won't get deleted yet. They won't get deleted until doing so is necessary to keep older backups around without violating the maximum allowed in Google Drive.
|
||||
>Note: You can configure the addon to delete backups more aggressively by setting `generational_delete_early`=true. With this, the addon will delete old backups that don't match a daily, weekly, monthly, or yearly configured cycle even if you aren't yet at risk of exceeding `max_backups_in_ha` or `max_backups_in_google_drive`. Careful though! You can accidentally delete all your backups this way if you don't have all your settings configured just the way you want them.
|
||||
* If more than one backup is created for a day (for example if you create one manually) then only the latest backup from that day will be kept.
|
||||
|
||||
## Schedule
|
||||
Figuring out date math in your head is hard, so it's useful to see a concrete example. Consider you have the following configuration. Two backups for each day, week, month, and year along with a limit in Google drive large enough to accommodate them all:
|
||||
```json
|
||||
"days_between_backups": 1,
|
||||
"generational_days": 2,
|
||||
"generational_weeks": 2,
|
||||
"generational_months": 2
|
||||
"generational_years": 2
|
||||
"max_backups_in_google_drive": 8
|
||||
```
|
||||
Imagine you've been running the add-on for 2 years now, diligently making a backup every day with no interruptions. On 19 May 2021, you could expect your list of backups in Google Drive to look like this:
|
||||
- May 19, 2021 <-- 1st Daily backup
|
||||
- May 18, 2021 <-- 2nd Daily backup
|
||||
- May 13, 2021 <-- 1st Weekly backup
|
||||
- May 06, 2021 <-- 2nd Weekly backup
|
||||
- May 01, 2021 <-- 1st Monthly backup
|
||||
- April 01, 2021 <-- 2nd Monthly backup
|
||||
- January 01, 2021 <-- 1st Yearly backup
|
||||
- January 01, 2020 <-- 2nd Yearly backup
|
||||
|
||||
Note that sometimes a day might overlap more than one schedule. For example, a backup on January 1st could satisfy the constraints for both a yearly and monthly backup. In this case, the add-on will only delete older backups when it *must* to keep from exceeding `max_backups_in_ha` or `max_backups_in_google_drive`. Thus, the most recent backup that would otherwise be deleted will be kept until space is needed somewhere else in the schedule.
|
||||
@@ -0,0 +1,34 @@
|
||||
# Home Assistant Add-on: Google Drive Backup
|
||||
|
||||
A complete and easy way to upload your Home Assistant backups to Google Drive.
|
||||
|
||||
## About
|
||||
|
||||
Quickly set up a backup strategy without much fuss. It doesn't require much familiarity with Home Assistant, its architecture, or Google Drive. Detailed install instructions are provided below but you can just add the repo, click install and open the Web UI. It will tell you what to do and only takes a few simple clicks.
|
||||
|
||||
>This project requires financial support to make the Google Drive integration work, but it is free for you to use. You can join those helping to keep the lights on at:
|
||||
>
|
||||
>[<img src="https://raw.githubusercontent.com/sabeechen/hassio-google-drive-backup/master/images/bmc-button.svg" width=150 height=40 style="margin: 5px"/>](https://www.buymeacoffee.com/sabeechen)
|
||||
>[<img src="https://raw.githubusercontent.com/sabeechen/hassio-google-drive-backup/master/images/paypal-button.svg" width=150 height=40 style="margin: 5px"/>](https://www.paypal.com/paypalme/stephenbeechen)
|
||||
>[<img src="https://raw.githubusercontent.com/sabeechen/hassio-google-drive-backup/master/images/patreon-button.svg" width=150 height=40 style="margin: 5px"/>](https://www.patreon.com/bePatron?u=4064183)
|
||||
>[<img src="https://raw.githubusercontent.com/sabeechen/hassio-google-drive-backup/master/images/github-sponsors-button.svg" width=150 height=40 style="margin: 5px"/>](https://github.com/sponsors/sabeechen)
|
||||
>[<img src="https://raw.githubusercontent.com/sabeechen/hassio-google-drive-backup/master/images/monero-button.svg" width=150 height=40 style="margin: 5px"/>](https://github.com/sabeechen/hassio-google-drive-backup/blob/master/donate-crypto.md)
|
||||
>[<img src="https://raw.githubusercontent.com/sabeechen/hassio-google-drive-backup/master/images/bitcoin-button.svg" width=150 height=40 style="margin: 5px"/>](https://github.com/sabeechen/hassio-google-drive-backup/blob/master/donate-crypto.md)
|
||||
>[<img src="https://raw.githubusercontent.com/sabeechen/hassio-google-drive-backup/master/images/ethereum-button.svg" width=150 height=40 style="margin: 5px"/>](https://github.com/sabeechen/hassio-google-drive-backup/blob/master/donate-crypto.md)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
- Creates backups on a configurable schedule.
|
||||
- Uploads backups to Drive, even the ones it didn't create.
|
||||
- Clean up old backups in Home Assistant and Google Drive, so you don't run out of space.
|
||||
- Restore from a fresh install or recover quickly from disaster by uploading your backups directly from Google Drive.
|
||||
- Integrates with Home Assistant Notifications, and provides sensors you can trigger off of.
|
||||
- Notifies you when something goes wrong with your backups.
|
||||
- Super easy installation and configuration.
|
||||
- Privacy-centric design philosophy.
|
||||
- Comprehensive documentation.
|
||||
- _Most certainly_ doesn't mine bitcoin on your home automation server. Definitely no.
|
||||
|
||||
See the [README on GitHub](https://github.com/sabeechen/hassio-google-drive-backup) for all the details, or just install the add-on and open the Web UI.
|
||||
The Web-UI explains everything you have to do.
|
||||
@@ -0,0 +1,7 @@
|
||||
#!/bin/bash
|
||||
|
||||
apk add python3 fping linux-headers libc-dev libffi-dev python3-dev gcc py3-pip
|
||||
pip3 install --upgrade pip wheel setuptools
|
||||
pip3 install --trusted-host pypi.python.org -r requirements-addon.txt
|
||||
# Remove packages we only needed for installation
|
||||
apk del linux-headers libc-dev libffi-dev python3-dev gcc
|
||||
@@ -0,0 +1,22 @@
|
||||
import platform
|
||||
import asyncio
|
||||
from aiorun import run
|
||||
from injector import Injector
|
||||
|
||||
from backup.module import MainModule, BaseModule
|
||||
from backup.starter import Starter
|
||||
|
||||
|
||||
async def main():
|
||||
await Injector([BaseModule(), MainModule()]).get(Starter).start()
|
||||
while True:
|
||||
await asyncio.sleep(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if platform.system() == "Windows":
|
||||
# Needed for dev on windows machines
|
||||
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
|
||||
asyncio.run(main())
|
||||
else:
|
||||
run(main())
|
||||
@@ -0,0 +1,11 @@
|
||||
# flake8: noqa
|
||||
from .config import Config, GenConfig, UPGRADE_OPTIONS
|
||||
from .settings import Setting, _DEFAULTS, _VALIDATORS, _LOOKUP, VERSION, PRIVATE, isStaging, addon_config, _CONFIG
|
||||
from .createoptions import CreateOptions
|
||||
from .boolvalidator import BoolValidator
|
||||
from .startable import Startable
|
||||
from .listvalidator import ListValidator
|
||||
from .durationasstringvalidator import DurationAsStringValidator
|
||||
from .bytesizeasstringvalidator import BytesizeAsStringValidator
|
||||
from .version import Version
|
||||
from .durationparser import DurationParser
|
||||
@@ -0,0 +1,18 @@
|
||||
from .validator import Validator
|
||||
from ..logger import getLogger
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
|
||||
class BoolValidator(Validator):
|
||||
def __init__(self, name):
|
||||
super().__init__(name)
|
||||
|
||||
def validate(self, value):
|
||||
if value is None or (type(value) == str and len(value) == 0):
|
||||
return None
|
||||
return BoolValidator.strToBool(value)
|
||||
|
||||
@classmethod
|
||||
def strToBool(cls, value) -> bool:
|
||||
return str(value).lower() in ['true', 't', 'on', 'yes', 'y', '1', 'hai', 'si', 'omgyesplease']
|
||||
@@ -0,0 +1,56 @@
|
||||
import re
|
||||
from injector import inject, singleton
|
||||
|
||||
SECOND_IDENTIFIERS = ["s", "sec", "secs", "second", "seconds"]
|
||||
MINUTE_IDENTIFIERS = ["m", "min", "mins", "minute", "minutes"]
|
||||
HOUR_IDENTIFIERS = ["h", "hr", "hour", "hours"]
|
||||
DAY_IDENTIFIERS = ["d", "day", "days"]
|
||||
NUMBER_REGEX = "^([0-9]*[.])?[0-9]+"
|
||||
VALID_REGEX = "^[ ]*([0-9,]*\\.?[0-9]*)[ ]*(b|B|k|K|m|M|g|G|t|T|p|P|e|E|z|Z|y|Y)[a-zA-Z ]*[ ]*$"
|
||||
BYTES_BASE = 1024
|
||||
PREFIX_VALUES = {
|
||||
"b": 1,
|
||||
"k": BYTES_BASE,
|
||||
"m": pow(BYTES_BASE, 2),
|
||||
"g": pow(BYTES_BASE, 3),
|
||||
"t": pow(BYTES_BASE, 4),
|
||||
"p": pow(BYTES_BASE, 5),
|
||||
"e": pow(BYTES_BASE, 6),
|
||||
"z": pow(BYTES_BASE, 7),
|
||||
"y": pow(BYTES_BASE, 8)
|
||||
}
|
||||
|
||||
PREFIX_CANONICAL = ["", "K", "M", "G", "T", "P", "E", "Z", "Y"]
|
||||
|
||||
|
||||
@singleton
|
||||
class ByteFormatter():
|
||||
@inject
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def parse(self, source: str):
|
||||
source = source.lower()
|
||||
match = re.match(VALID_REGEX, source.lower())
|
||||
if not match:
|
||||
raise ValueError()
|
||||
number, prefix = match.group(1, 2)
|
||||
if prefix not in PREFIX_VALUES:
|
||||
raise ValueError()
|
||||
|
||||
return float(number) * PREFIX_VALUES[prefix]
|
||||
|
||||
def format(self, bytes):
|
||||
for prefix in PREFIX_CANONICAL:
|
||||
if bytes < BYTES_BASE:
|
||||
if int(bytes) == bytes:
|
||||
return f"{int(bytes)} {prefix}B"
|
||||
else:
|
||||
return f"{bytes} {prefix}B"
|
||||
bytes /= BYTES_BASE
|
||||
|
||||
bytes *= BYTES_BASE
|
||||
if int(bytes) == bytes:
|
||||
return f"{int(bytes)} YB"
|
||||
else:
|
||||
return f"{bytes} YB"
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
from .byteformatter import ByteFormatter
|
||||
from .validator import Validator
|
||||
|
||||
|
||||
class BytesizeAsStringValidator(Validator):
|
||||
def __init__(self, name, minimum=None, maximum=None):
|
||||
super().__init__(name)
|
||||
self.min = minimum
|
||||
self.max = maximum
|
||||
|
||||
def validate(self, value):
|
||||
if type(value) is str:
|
||||
value = value.strip()
|
||||
if value is None or (type(value) == str and len(value) == 0):
|
||||
return None
|
||||
try:
|
||||
if type(value) == str:
|
||||
value = ByteFormatter().parse(value)
|
||||
value = float(value)
|
||||
except ValueError:
|
||||
self.raiseForValue(value)
|
||||
|
||||
if self.max is not None and value > self.max:
|
||||
self.raiseForValue(value)
|
||||
if self.min is not None and value < self.min:
|
||||
self.raiseForValue(value)
|
||||
return value
|
||||
|
||||
def formatForUi(self, value):
|
||||
return ByteFormatter().format(value)
|
||||
@@ -0,0 +1,307 @@
|
||||
import json
|
||||
import os
|
||||
import os.path
|
||||
import uuid
|
||||
from typing import Any, Dict, List, Optional
|
||||
from yarl import URL
|
||||
|
||||
from .settings import _LOOKUP, Setting, _VALIDATORS
|
||||
from ..logger import getLogger
|
||||
from backup.file import JsonFileSaver
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
ALWAYS_KEEP = {
|
||||
Setting.DAYS_BETWEEN_BACKUPS,
|
||||
Setting.MAX_BACKUPS_IN_HA,
|
||||
Setting.MAX_BACKUPS_IN_GOOGLE_DRIVE,
|
||||
}
|
||||
|
||||
KEEP_DEFAULT = {
|
||||
Setting.SEND_ERROR_REPORTS,
|
||||
Setting.IGNORE_UPGRADE_BACKUPS
|
||||
}
|
||||
|
||||
# these are the options that should trigger a restart of the server
|
||||
SERVER_OPTIONS = {
|
||||
Setting.USE_SSL,
|
||||
Setting.REQUIRE_LOGIN,
|
||||
Setting.CERTFILE,
|
||||
Setting.KEYFILE,
|
||||
Setting.EXPOSE_EXTRA_SERVER
|
||||
}
|
||||
|
||||
NON_UI_SETTING = {
|
||||
Setting.SUPERVISOR_URL,
|
||||
Setting.TOKEN_SERVER_HOSTS,
|
||||
Setting.DRIVE_AUTHORIZE_URL,
|
||||
Setting.DRIVE_DEVICE_CODE_URL,
|
||||
Setting.DEFAULT_DRIVE_CLIENT_ID,
|
||||
Setting.NEW_BACKUP_TIMEOUT_SECONDS,
|
||||
Setting.LOG_LEVEL,
|
||||
Setting.CONSOLE_LOG_LEVEL,
|
||||
Setting.DEFAULT_SYNC_INTERVAL_VARIATION,
|
||||
Setting.CACHE_WARMUP_MAX_SECONDS,
|
||||
Setting.CACHE_WARMUP_ERROR_TIMEOUT_SECONDS,
|
||||
Setting.WATCH_BACKUP_DIRECTORY,
|
||||
Setting.TRACE_REQUESTS,
|
||||
Setting.MAX_BACKOFF_SECONDS
|
||||
}
|
||||
|
||||
UPGRADE_OPTIONS = {
|
||||
Setting.DEPRECTAED_MAX_BACKUPS_IN_HA: Setting.MAX_BACKUPS_IN_HA,
|
||||
Setting.DEPRECTAED_MAX_BACKUPS_IN_GOOGLE_DRIVE: Setting.MAX_BACKUPS_IN_GOOGLE_DRIVE,
|
||||
Setting.DEPRECATED_DAYS_BETWEEN_BACKUPS: Setting.DAYS_BETWEEN_BACKUPS,
|
||||
Setting.DEPRECTAED_IGNORE_OTHER_BACKUPS: Setting.IGNORE_OTHER_BACKUPS,
|
||||
Setting.DEPRECTAED_IGNORE_UPGRADE_BACKUPS: Setting.IGNORE_UPGRADE_BACKUPS,
|
||||
Setting.DEPRECTAED_DELETE_BEFORE_NEW_BACKUP: Setting.DELETE_BEFORE_NEW_BACKUP,
|
||||
Setting.DEPRECTAED_BACKUP_NAME: Setting.BACKUP_NAME,
|
||||
Setting.DEPRECTAED_BACKUP_TIME_OF_DAY: Setting.BACKUP_TIME_OF_DAY,
|
||||
Setting.DEPRECTAED_SPECIFY_BACKUP_FOLDER: Setting.SPECIFY_BACKUP_FOLDER,
|
||||
Setting.DEPRECTAED_NOTIFY_FOR_STALE_BACKUPS: Setting.NOTIFY_FOR_STALE_BACKUPS,
|
||||
Setting.DEPRECTAED_ENABLE_BACKUP_STALE_SENSOR: Setting.ENABLE_BACKUP_STALE_SENSOR,
|
||||
Setting.DEPRECTAED_ENABLE_BACKUP_STATE_SENSOR: Setting.ENABLE_BACKUP_STATE_SENSOR,
|
||||
Setting.DEPRECATED_BACKUP_PASSWORD: Setting.BACKUP_PASSWORD
|
||||
}
|
||||
|
||||
EMPTY_IS_DEFAULT = {
|
||||
Setting.ACCENT_COLOR,
|
||||
Setting.BACKGROUND_COLOR,
|
||||
}
|
||||
|
||||
|
||||
class GenConfig():
|
||||
def __init__(self, days=0, weeks=0, months=0, years=0, day_of_week='mon', day_of_month=1, day_of_year=1, aggressive=False):
|
||||
self.days = days
|
||||
self.weeks = weeks
|
||||
self.months = months
|
||||
self.years = years
|
||||
self.day_of_week = day_of_week
|
||||
self.day_of_month = day_of_month
|
||||
self.day_of_year = day_of_year
|
||||
self.aggressive = aggressive
|
||||
self._config_was_upgraded = False
|
||||
|
||||
def __eq__(self, other):
|
||||
"""Overrides the default implementation"""
|
||||
if isinstance(other, GenConfig):
|
||||
return self.__dict__ == other.__dict__
|
||||
return NotImplemented
|
||||
|
||||
def __hash__(self):
|
||||
"""Overrides the default implementation"""
|
||||
return hash(tuple(sorted(self.__dict__.items())))
|
||||
|
||||
|
||||
class Config():
|
||||
@classmethod
|
||||
def fromFile(cls, config_path):
|
||||
return Config(JsonFileSaver.read(config_path))
|
||||
|
||||
@classmethod
|
||||
def withOverrides(cls, overrides):
|
||||
config = Config()
|
||||
for key in overrides.keys():
|
||||
config.override(key, overrides[key])
|
||||
return config
|
||||
|
||||
@classmethod
|
||||
def withFileOverrides(cls, override_path):
|
||||
data = JsonFileSaver.read(override_path)
|
||||
overrides = {}
|
||||
for key in data.keys():
|
||||
overrides[_LOOKUP[key]] = data[key]
|
||||
return Config.withOverrides(overrides)
|
||||
|
||||
@classmethod
|
||||
def fromEnvironment(cls):
|
||||
config = {}
|
||||
for key in os.environ:
|
||||
if key in _LOOKUP:
|
||||
config[_LOOKUP[key]] = _VALIDATORS[_LOOKUP[key]].validate(os.environ[key])
|
||||
elif str.lower(key) in _LOOKUP:
|
||||
config[_LOOKUP[str.lower(key)]] = _VALIDATORS[_LOOKUP[str.lower(key)]].validate(os.environ[key])
|
||||
return Config(config)
|
||||
|
||||
def __init__(self, data=None):
|
||||
self.overrides = {}
|
||||
if data is None:
|
||||
self.config = {}
|
||||
else:
|
||||
self.config = data
|
||||
self._legacy_ignored_behavior = False
|
||||
self._subscriptions = []
|
||||
self._clientIdentifier = None
|
||||
self.retained = self._loadRetained()
|
||||
self._gen_config_cache = self.getGenerationalConfig()
|
||||
|
||||
# Tracks when hosts have been seen to be offline to retry on different hosts.
|
||||
self._commFailure = {}
|
||||
|
||||
def getConfigFor(self, options):
|
||||
new_config = Config()
|
||||
new_config.overrides = self.overrides.copy()
|
||||
new_config.update(options)
|
||||
return new_config
|
||||
|
||||
def validateUpdate(self, additions):
|
||||
new_config = self.config.copy()
|
||||
new_config.update(additions)
|
||||
validated, upgraded = self.validate(new_config)
|
||||
return validated
|
||||
|
||||
def validate(self, new_config) -> Dict[str, Any]:
|
||||
final_config = {}
|
||||
|
||||
upgraded = False
|
||||
# validate each item
|
||||
for key in new_config:
|
||||
if type(key) == str:
|
||||
if key not in _LOOKUP:
|
||||
# its not in the schema, just ignore it
|
||||
continue
|
||||
setting = _LOOKUP[key]
|
||||
else:
|
||||
setting = key
|
||||
|
||||
value = setting.validator().validate(new_config[key])
|
||||
if setting in UPGRADE_OPTIONS:
|
||||
upgraded = True
|
||||
if isinstance(value, str) and len(value) == 0 and setting in EMPTY_IS_DEFAULT:
|
||||
value = setting.default()
|
||||
if value is not None and (setting in KEEP_DEFAULT or value != setting.default()):
|
||||
if setting in UPGRADE_OPTIONS and (UPGRADE_OPTIONS[setting] not in new_config or new_config[UPGRADE_OPTIONS[setting]] == UPGRADE_OPTIONS[setting].default()):
|
||||
upgraded = True
|
||||
final_config[UPGRADE_OPTIONS[setting]] = value
|
||||
elif setting not in UPGRADE_OPTIONS:
|
||||
final_config[setting] = value
|
||||
|
||||
if upgraded:
|
||||
final_config[Setting.CALL_BACKUP_SNAPSHOT] = True
|
||||
|
||||
# add in non-ui settings
|
||||
for setting in NON_UI_SETTING:
|
||||
if self.get(setting) != setting.default() and not (setting in new_config or setting.key in new_config) and setting not in self.overrides:
|
||||
final_config[setting] = self.get(setting)
|
||||
|
||||
# add defaults
|
||||
for key in ALWAYS_KEEP:
|
||||
if key not in final_config:
|
||||
final_config[key] = key.default()
|
||||
|
||||
if not final_config.get(Setting.USE_SSL, False):
|
||||
for key in [Setting.CERTFILE, Setting.KEYFILE]:
|
||||
if key in final_config:
|
||||
del final_config[key]
|
||||
|
||||
return final_config, upgraded
|
||||
|
||||
def update(self, new_config):
|
||||
validated, upgraded = self.validate(new_config)
|
||||
self._config_was_upgraded = upgraded
|
||||
self.config = validated
|
||||
self._gen_config_cache = self.getGenerationalConfig()
|
||||
for sub in self._subscriptions:
|
||||
sub()
|
||||
|
||||
def getServerOptions(self):
|
||||
ret = {}
|
||||
for setting in SERVER_OPTIONS:
|
||||
ret[setting] = self.get(setting)
|
||||
return ret
|
||||
|
||||
def subscribe(self, func):
|
||||
self._subscriptions.append(func)
|
||||
|
||||
def clientIdentifier(self) -> str:
|
||||
if self._clientIdentifier is None:
|
||||
try:
|
||||
if JsonFileSaver.exists(self.get(Setting.ID_FILE_PATH)):
|
||||
self._clientIdentifier = JsonFileSaver.read(self.get(Setting.ID_FILE_PATH))['id']
|
||||
else:
|
||||
self._clientIdentifier = str(uuid.uuid4())
|
||||
JsonFileSaver.write(self.get(Setting.ID_FILE_PATH), {'id': self._clientIdentifier})
|
||||
except Exception:
|
||||
self._clientIdentifier = str(uuid.uuid4())
|
||||
return self._clientIdentifier
|
||||
|
||||
def getGenerationalConfig(self) -> Optional[Dict[str, Any]]:
|
||||
days = self.get(Setting.GENERATIONAL_DAYS)
|
||||
weeks = self.get(Setting.GENERATIONAL_WEEKS)
|
||||
months = self.get(Setting.GENERATIONAL_MONTHS)
|
||||
years = self.get(Setting.GENERATIONAL_YEARS)
|
||||
if days + weeks + months + years == 0:
|
||||
return None
|
||||
base = GenConfig(
|
||||
days=days,
|
||||
weeks=weeks,
|
||||
months=months,
|
||||
years=years,
|
||||
day_of_week=self.get(Setting.GENERATIONAL_DAY_OF_WEEK),
|
||||
day_of_month=self.get(Setting.GENERATIONAL_DAY_OF_MONTH),
|
||||
day_of_year=self.get(Setting.GENERATIONAL_DAY_OF_YEAR),
|
||||
aggressive=self.get(Setting.GENERATIONAL_DELETE_EARLY)
|
||||
)
|
||||
if base.days <= 1:
|
||||
# must always be >= 1, otherwise we'll just create and delete backups constantly.
|
||||
base.days = 1
|
||||
return base
|
||||
|
||||
def _loadRetained(self) -> List[str]:
|
||||
if JsonFileSaver.exists(self.get(Setting.RETAINED_FILE_PATH)):
|
||||
try:
|
||||
return JsonFileSaver.read(self.get(Setting.RETAINED_FILE_PATH))['retained']
|
||||
except json.decoder.JSONDecodeError:
|
||||
logger.error("Unable to parse retained backup settings")
|
||||
return []
|
||||
return []
|
||||
|
||||
def isRetained(self, slug):
|
||||
return slug in self.retained
|
||||
|
||||
def setRetained(self, slug, retain):
|
||||
if retain and slug not in self.retained:
|
||||
self.retained.append(slug)
|
||||
JsonFileSaver.write(self.get(Setting.RETAINED_FILE_PATH), {'retained': self.retained})
|
||||
elif not retain and slug in self.retained:
|
||||
self.retained.remove(slug)
|
||||
JsonFileSaver.write(self.get(Setting.RETAINED_FILE_PATH), {'retained': self.retained})
|
||||
|
||||
def isExplicit(self, setting):
|
||||
return setting in self.config or setting.value in self.config
|
||||
|
||||
def override(self, setting: Setting, value):
|
||||
self.overrides[setting] = value
|
||||
return self
|
||||
|
||||
def get(self, setting: Setting) -> Any:
|
||||
if setting in self.overrides:
|
||||
return self.overrides[setting]
|
||||
if setting in self.config:
|
||||
return self.config[setting]
|
||||
if setting.key() in self.config:
|
||||
return self.config[setting.key()]
|
||||
else:
|
||||
if setting == Setting.IGNORE_UPGRADE_BACKUPS and self._legacy_ignored_behavior:
|
||||
# Use the old behavior, rather than the new one
|
||||
return False
|
||||
return setting.default()
|
||||
|
||||
def getForUi(self, setting: Setting):
|
||||
return _VALIDATORS[setting].formatForUi(self.get(setting))
|
||||
|
||||
def getTokenServers(self, path: str = "") -> List[URL]:
|
||||
return list(map(lambda s: URL(s).with_path(path), self.get(Setting.TOKEN_SERVER_HOSTS).split(",")))
|
||||
|
||||
def mustSaveUpgradeChanges(self):
|
||||
return self._config_was_upgraded
|
||||
|
||||
def getAllConfig(self) -> Dict[Setting, Any]:
|
||||
return self.config.copy()
|
||||
|
||||
def persistedChanges(self):
|
||||
self._config_was_upgraded = False
|
||||
|
||||
def useLegacyIgnoredBehavior(self, value: bool):
|
||||
"""If the user upgrades from an old version and hasn't explicitely said they want to include upgrade backups, then this reverts them to the old behavior where they aren't ignored"""
|
||||
self._legacy_ignored_behavior = value
|
||||
@@ -0,0 +1,13 @@
|
||||
from datetime import datetime
|
||||
from typing import Dict
|
||||
from ..logger import getLogger
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
|
||||
class CreateOptions(object):
|
||||
def __init__(self, when: datetime, name_template: str, retain_sources: Dict[str, bool] = {}, note: str = None):
|
||||
self.when: datetime = when
|
||||
self.name_template: str = name_template
|
||||
self.retain_sources: Dict[str, bool] = retain_sources
|
||||
self.note = note
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
from datetime import timedelta
|
||||
from .durationparser import DurationParser
|
||||
from .validator import Validator
|
||||
|
||||
|
||||
class DurationAsStringValidator(Validator):
|
||||
def __init__(self, name, minimum=None, maximum=None, base_seconds=1, default_as_empty=None):
|
||||
super().__init__(name)
|
||||
self.min = minimum
|
||||
self.max = maximum
|
||||
self.base_seconds = base_seconds
|
||||
self.default_as_empty = default_as_empty
|
||||
|
||||
def validate(self, value):
|
||||
if value is None or (type(value) == str and len(value) == 0):
|
||||
return None
|
||||
try:
|
||||
if type(value) == str:
|
||||
if self.default_as_empty is not None and value == "":
|
||||
value = self.default_as_empty
|
||||
else:
|
||||
value = DurationParser().parse(value).total_seconds() / self.base_seconds
|
||||
value = float(value)
|
||||
except ValueError:
|
||||
self.raiseForValue(value)
|
||||
|
||||
if self.max is not None and value > self.max:
|
||||
self.raiseForValue(value)
|
||||
if self.min is not None and value < self.min:
|
||||
self.raiseForValue(value)
|
||||
return value
|
||||
|
||||
def formatForUi(self, value):
|
||||
if self.default_as_empty is not None and value == self.default_as_empty:
|
||||
return ""
|
||||
else:
|
||||
return DurationParser().format(timedelta(seconds=value * self.base_seconds))
|
||||
@@ -0,0 +1,80 @@
|
||||
import re
|
||||
from datetime import timedelta
|
||||
from injector import inject, singleton
|
||||
|
||||
SECOND_IDENTIFIERS = ["s", "sec", "secs", "second", "seconds"]
|
||||
MINUTE_IDENTIFIERS = ["m", "min", "mins", "minute", "minutes"]
|
||||
HOUR_IDENTIFIERS = ["h", "hr", "hour", "hours"]
|
||||
DAY_IDENTIFIERS = ["d", "day", "days"]
|
||||
NUMBER_REGEX = "^([0-9]*[.])?[0-9]+"
|
||||
VALID_REGEX = "^([ ]*([0-9]*[.])?[0-9]+[ ]*(seconds|second|secs|sec|s|minutes|minute|mins|min|m|hours|hour|hr|h|days|day|d)?[ ,]*)*"
|
||||
|
||||
|
||||
@singleton
|
||||
class DurationParser():
|
||||
@inject
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def parse(self, source: str):
|
||||
source = source.lower()
|
||||
total_match = re.match(VALID_REGEX, source)
|
||||
if not total_match or total_match.group(0) != source:
|
||||
raise ValueError()
|
||||
parts = source.split()
|
||||
i = 0
|
||||
total = timedelta(seconds=0)
|
||||
while (i < len(parts)):
|
||||
part = parts[i].strip().strip(',')
|
||||
match = re.match(NUMBER_REGEX, part)
|
||||
i += 1
|
||||
if not match:
|
||||
raise ValueError()
|
||||
length = float(match.group(0))
|
||||
if match.group(0) == part:
|
||||
|
||||
if i < len(parts):
|
||||
next_part = parts[i].strip().strip(',')
|
||||
if next_part in SECOND_IDENTIFIERS or next_part in MINUTE_IDENTIFIERS or next_part in HOUR_IDENTIFIERS or next_part in DAY_IDENTIFIERS:
|
||||
identifier = next_part
|
||||
i += 1
|
||||
else:
|
||||
identifier = SECOND_IDENTIFIERS[0]
|
||||
else:
|
||||
identifier = "s"
|
||||
else:
|
||||
identifier = part[len(match.group(0)):]
|
||||
if identifier in SECOND_IDENTIFIERS:
|
||||
total += timedelta(seconds=length)
|
||||
elif identifier in MINUTE_IDENTIFIERS:
|
||||
total += timedelta(minutes=length)
|
||||
elif identifier in HOUR_IDENTIFIERS:
|
||||
total += timedelta(hours=length)
|
||||
elif identifier in DAY_IDENTIFIERS:
|
||||
total += timedelta(days=length)
|
||||
else:
|
||||
raise ValueError()
|
||||
return total
|
||||
|
||||
def format(self, duration: timedelta):
|
||||
parts = []
|
||||
if duration >= timedelta(days=1):
|
||||
days = int(duration.days)
|
||||
parts.append("{} days".format(days))
|
||||
duration = duration - timedelta(days=days)
|
||||
if duration >= timedelta(hours=1):
|
||||
hours = int(duration.seconds / (60 * 60))
|
||||
parts.append("{} hours".format(hours))
|
||||
duration = duration - timedelta(hours=hours)
|
||||
if duration >= timedelta(minutes=1):
|
||||
minutes = int(duration.seconds / 60)
|
||||
parts.append("{} minutes".format(minutes))
|
||||
duration = duration - timedelta(minutes=minutes)
|
||||
if duration >= timedelta(seconds=1):
|
||||
seconds = int(duration.seconds)
|
||||
parts.append("{} seconds".format(seconds))
|
||||
duration = duration - timedelta(seconds=seconds)
|
||||
if len(parts) > 0:
|
||||
return ", ".join(parts)
|
||||
else:
|
||||
return "0 seconds"
|
||||
@@ -0,0 +1,25 @@
|
||||
from .validator import Validator
|
||||
from ..logger import getLogger
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
|
||||
class FloatValidator(Validator):
|
||||
def __init__(self, name, minimum=None, maximum=None):
|
||||
super().__init__(name)
|
||||
self.min = minimum
|
||||
self.max = maximum
|
||||
|
||||
def validate(self, value):
|
||||
if value is None or (type(value) == str and len(value) == 0):
|
||||
return None
|
||||
try:
|
||||
value = float(value)
|
||||
except ValueError:
|
||||
self.raiseForValue(value)
|
||||
|
||||
if self.max is not None and value > self.max:
|
||||
self.raiseForValue(value)
|
||||
if self.min is not None and value < self.min:
|
||||
self.raiseForValue(value)
|
||||
return value
|
||||
@@ -0,0 +1,25 @@
|
||||
from .validator import Validator
|
||||
from ..logger import getLogger
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
|
||||
class IntValidator(Validator):
|
||||
def __init__(self, name, minimum=None, maximum=None):
|
||||
super().__init__(name)
|
||||
self.min = minimum
|
||||
self.max = maximum
|
||||
|
||||
def validate(self, value):
|
||||
if value is None or (type(value) == str and len(value) == 0):
|
||||
return None
|
||||
try:
|
||||
value = int(value)
|
||||
except ValueError:
|
||||
self.raiseForValue(value)
|
||||
|
||||
if self.max is not None and value > self.max:
|
||||
self.raiseForValue(value)
|
||||
if self.min is not None and value < self.min:
|
||||
self.raiseForValue(value)
|
||||
return value
|
||||
@@ -0,0 +1,15 @@
|
||||
from .validator import Validator
|
||||
from ..logger import getLogger
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
|
||||
class ListValidator(Validator):
|
||||
def __init__(self, name, values):
|
||||
super().__init__(name)
|
||||
self.values = values
|
||||
|
||||
def validate(self, value):
|
||||
if value not in self.values:
|
||||
self.raiseForValue(value)
|
||||
return value
|
||||
@@ -0,0 +1,19 @@
|
||||
from .validator import Validator
|
||||
import re
|
||||
from ..logger import getLogger
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
|
||||
class RegexValidator(Validator):
|
||||
def __init__(self, name, regex):
|
||||
super().__init__(name)
|
||||
self.re = re.compile(regex)
|
||||
|
||||
def validate(self, value):
|
||||
if value is None or (type(value) == str and len(value) == 0):
|
||||
return ""
|
||||
value = str(value)
|
||||
if not self.re.match(value):
|
||||
self.raiseForValue(value)
|
||||
return value
|
||||
@@ -0,0 +1,517 @@
|
||||
import json
|
||||
from enum import Enum, unique
|
||||
from os.path import abspath, join
|
||||
|
||||
from .boolvalidator import BoolValidator
|
||||
from .floatvalidator import FloatValidator
|
||||
from .intvalidator import IntValidator
|
||||
from .regexvalidator import RegexValidator
|
||||
from .stringvalidator import StringValidator
|
||||
from .listvalidator import ListValidator
|
||||
from .durationasstringvalidator import DurationAsStringValidator
|
||||
from .bytesizeasstringvalidator import BytesizeAsStringValidator
|
||||
from ..logger import getLogger
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
|
||||
@unique
|
||||
class Setting(Enum):
|
||||
MAX_BACKUPS_IN_HA = "max_backups_in_ha"
|
||||
MAX_BACKUPS_IN_GOOGLE_DRIVE = "max_backups_in_google_drive"
|
||||
DAYS_BETWEEN_BACKUPS = "days_between_backups"
|
||||
IGNORE_OTHER_BACKUPS = "ignore_other_backups"
|
||||
IGNORE_UPGRADE_BACKUPS = "ignore_upgrade_backups"
|
||||
DELETE_IGNORED_AFTER_DAYS = "delete_ignored_after_days"
|
||||
DELETE_BEFORE_NEW_BACKUP = "delete_before_new_backup"
|
||||
BACKUP_NAME = "backup_name"
|
||||
BACKUP_TIME_OF_DAY = "backup_time_of_day"
|
||||
SPECIFY_BACKUP_FOLDER = "specify_backup_folder"
|
||||
NOTIFY_FOR_STALE_BACKUPS = "notify_for_stale_backups"
|
||||
ENABLE_BACKUP_STALE_SENSOR = "enable_backup_stale_sensor"
|
||||
ENABLE_BACKUP_STATE_SENSOR = "enable_backup_state_sensor"
|
||||
BACKUP_PASSWORD = "backup_password"
|
||||
BACKUP_STORAGE = "backup_storage"
|
||||
CALL_BACKUP_SNAPSHOT = "call_backup_snapshot"
|
||||
|
||||
# Basic backup settings
|
||||
WARN_FOR_LOW_SPACE = "warn_for_low_space"
|
||||
LOW_SPACE_THRESHOLD = "low_space_threshold"
|
||||
DELETE_AFTER_UPLOAD = "delete_after_upload"
|
||||
|
||||
# generational settings
|
||||
GENERATIONAL_DAYS = "generational_days"
|
||||
GENERATIONAL_WEEKS = "generational_weeks"
|
||||
GENERATIONAL_MONTHS = "generational_months"
|
||||
GENERATIONAL_YEARS = "generational_years"
|
||||
GENERATIONAL_DAY_OF_WEEK = "generational_day_of_week"
|
||||
GENERATIONAL_DAY_OF_MONTH = "generational_day_of_month"
|
||||
GENERATIONAL_DAY_OF_YEAR = "generational_day_of_year"
|
||||
GENERATIONAL_DELETE_EARLY = "generational_delete_early"
|
||||
|
||||
# Partial backups
|
||||
EXCLUDE_FOLDERS = "exclude_folders"
|
||||
EXCLUDE_ADDONS = "exclude_addons"
|
||||
|
||||
STOP_ADDONS = "stop_addons"
|
||||
DISABLE_WATCHDOG_WHEN_STOPPING = "disable_watchdog_when_stopping"
|
||||
|
||||
# UI Server Options
|
||||
USE_SSL = "use_ssl"
|
||||
CERTFILE = "certfile"
|
||||
KEYFILE = "keyfile"
|
||||
INGRESS_PORT = "ingress_port"
|
||||
PORT = "port"
|
||||
REQUIRE_LOGIN = "require_login"
|
||||
EXPOSE_EXTRA_SERVER = "expose_extra_server"
|
||||
|
||||
# Add-on options
|
||||
VERBOSE = "verbose"
|
||||
SEND_ERROR_REPORTS = "send_error_reports"
|
||||
CONFIRM_MULTIPLE_DELETES = "confirm_multiple_deletes"
|
||||
ENABLE_DRIVE_UPLOAD = "enable_drive_upload"
|
||||
WATCH_BACKUP_DIRECTORY = "watch_backup_directory"
|
||||
TRACE_REQUESTS = "trace_requests"
|
||||
|
||||
# Theme Settings
|
||||
BACKGROUND_COLOR = "background_color"
|
||||
ACCENT_COLOR = "accent_color"
|
||||
|
||||
# Network and dns stuff
|
||||
DRIVE_EXPERIMENTAL = "drive_experimental"
|
||||
DRIVE_IPV4 = "drive_ipv4"
|
||||
IGNORE_IPV6_ADDRESSES = "ignore_ipv6_addresses"
|
||||
GOOGLE_DRIVE_TIMEOUT_SECONDS = "google_drive_timeout_seconds"
|
||||
GOOGLE_DRIVE_PAGE_SIZE = "google_drive_page_size"
|
||||
ALTERNATE_DNS_SERVERS = "alternate_dns_servers"
|
||||
DEFAULT_DRIVE_CLIENT_ID = "default_drive_client_id"
|
||||
DEFAULT_DRIVE_CLIENT_SECRET = "default_drive_client_secret"
|
||||
DRIVE_PICKER_API_KEY = "drive_picker_api_key"
|
||||
MAXIMUM_UPLOAD_CHUNK_BYTES = "maximum_upload_chunk_bytes"
|
||||
|
||||
# Files and folders
|
||||
FOLDER_FILE_PATH = "folder_file_path"
|
||||
CREDENTIALS_FILE_PATH = "credentials_file_path"
|
||||
RETAINED_FILE_PATH = "retained_file_path"
|
||||
SECRETS_FILE_PATH = "secrets_file_path"
|
||||
BACKUP_DIRECTORY_PATH = "backup_directory_path"
|
||||
INGRESS_TOKEN_FILE_PATH = "ingress_token_file_path"
|
||||
CONFIG_FILE_PATH = "config_file_path"
|
||||
ID_FILE_PATH = "id_file_path"
|
||||
DATA_CACHE_FILE_PATH = "data_cache_file_path"
|
||||
|
||||
# endpoints
|
||||
AUTHORIZATION_HOST = "authorization_host"
|
||||
TOKEN_SERVER_HOSTS = "token_server_hosts"
|
||||
SUPERVISOR_URL = "supervisor_url"
|
||||
DRIVE_URL = "drive_url"
|
||||
SUPERVISOR_TOKEN = "hassio_header"
|
||||
DRIVE_HOST_NAME = "drive_host_name"
|
||||
DRIVE_REFRESH_URL = "drive_refresh_url"
|
||||
DRIVE_AUTHORIZE_URL = "drive_authorize_url"
|
||||
DRIVE_DEVICE_CODE_URL = "drive_device_code_url"
|
||||
DRIVE_TOKEN_URL = "drive_token_url"
|
||||
SAVE_DRIVE_CREDS_PATH = "save_drive_creds_path"
|
||||
STOP_ADDON_STATE_PATH = "stop_addon_state_path"
|
||||
|
||||
# Timing and timeouts
|
||||
MAX_SYNC_INTERVAL_SECONDS = "max_sync_interval_seconds"
|
||||
DEFAULT_SYNC_INTERVAL_VARIATION = "default_sync_interval_variation"
|
||||
BACKUP_STALE_SECONDS = "backup_stale_seconds"
|
||||
PENDING_BACKUP_TIMEOUT_SECONDS = "pending_backup_timeout_seconds"
|
||||
FAILED_BACKUP_TIMEOUT_SECONDS = "failed_backup_timeout_seconds"
|
||||
NEW_BACKUP_TIMEOUT_SECONDS = "new_backup_timeout_seconds"
|
||||
DOWNLOAD_TIMEOUT_SECONDS = "download_timeout_seconds"
|
||||
DEFAULT_CHUNK_SIZE = "default_chunk_size"
|
||||
DEBUGGER_PORT = "debugger_port"
|
||||
SERVER_PROJECT_ID = "server_project_id"
|
||||
LOG_LEVEL = "log_level"
|
||||
CONSOLE_LOG_LEVEL = "console_log_level"
|
||||
BACKUP_STARTUP_DELAY_MINUTES = "backup_startup_delay_minutes"
|
||||
EXCHANGER_TIMEOUT_SECONDS = "exchanger_timeout_seconds"
|
||||
HA_REPORTING_INTERVAL_SECONDS = "ha_reporting_interval_seconds"
|
||||
LONG_TERM_STALE_BACKUP_SECONDS = "long_term_stale_backup_seconds"
|
||||
PING_TIMEOUT = "ping_timeout"
|
||||
CACHE_WARMUP_MAX_SECONDS = "cache_warmup_max_seconds"
|
||||
CACHE_WARMUP_ERROR_TIMEOUT_SECONDS = "cache_warmup_error_timeout"
|
||||
MAX_BACKOFF_SECONDS = "max_backoff_seconds"
|
||||
|
||||
# Old, deprecated settings
|
||||
DEPRECTAED_MAX_BACKUPS_IN_HA = "max_snapshots_in_hassio"
|
||||
DEPRECTAED_MAX_BACKUPS_IN_GOOGLE_DRIVE = "max_snapshots_in_google_drive"
|
||||
DEPRECATED_DAYS_BETWEEN_BACKUPS = "days_between_snapshots"
|
||||
DEPRECTAED_IGNORE_OTHER_BACKUPS = "ignore_other_snapshots"
|
||||
DEPRECTAED_IGNORE_UPGRADE_BACKUPS = "ignore_upgrade_snapshots"
|
||||
DEPRECTAED_BACKUP_NAME = "snapshot_name"
|
||||
DEPRECTAED_BACKUP_TIME_OF_DAY = "snapshot_time_of_day"
|
||||
DEPRECATED_BACKUP_PASSWORD = "snapshot_password"
|
||||
DEPRECTAED_SPECIFY_BACKUP_FOLDER = "specify_snapshot_folder"
|
||||
DEPRECTAED_DELETE_BEFORE_NEW_BACKUP = "delete_before_new_snapshot"
|
||||
DEPRECTAED_NOTIFY_FOR_STALE_BACKUPS = "notify_for_stale_snapshots"
|
||||
DEPRECTAED_ENABLE_BACKUP_STALE_SENSOR = "enable_snapshot_stale_sensor"
|
||||
DEPRECTAED_ENABLE_BACKUP_STATE_SENSOR = "enable_snapshot_state_sensor"
|
||||
|
||||
def default(self):
|
||||
if "staging" in VERSION and self in _STAGING_DEFAULTS:
|
||||
return _STAGING_DEFAULTS[self]
|
||||
return _DEFAULTS[self]
|
||||
|
||||
def validator(self):
|
||||
return _VALIDATORS[self]
|
||||
|
||||
def key(self):
|
||||
return self.value
|
||||
|
||||
|
||||
_DEFAULTS = {
|
||||
Setting.MAX_BACKUPS_IN_HA: 4,
|
||||
Setting.MAX_BACKUPS_IN_GOOGLE_DRIVE: 4,
|
||||
Setting.DAYS_BETWEEN_BACKUPS: 3,
|
||||
Setting.IGNORE_OTHER_BACKUPS: False,
|
||||
Setting.IGNORE_UPGRADE_BACKUPS: True,
|
||||
Setting.DELETE_IGNORED_AFTER_DAYS: 0,
|
||||
Setting.DELETE_BEFORE_NEW_BACKUP: False,
|
||||
Setting.BACKUP_NAME: "{type} Backup {year}-{month}-{day} {hr24}:{min}:{sec}",
|
||||
Setting.BACKUP_TIME_OF_DAY: "",
|
||||
Setting.SPECIFY_BACKUP_FOLDER: False,
|
||||
Setting.NOTIFY_FOR_STALE_BACKUPS: True,
|
||||
Setting.ENABLE_BACKUP_STALE_SENSOR: True,
|
||||
Setting.ENABLE_BACKUP_STATE_SENSOR: True,
|
||||
Setting.BACKUP_PASSWORD: "",
|
||||
Setting.BACKUP_STORAGE: "",
|
||||
Setting.WATCH_BACKUP_DIRECTORY: True,
|
||||
Setting.TRACE_REQUESTS: False,
|
||||
|
||||
# Basic backup settings
|
||||
Setting.DEPRECTAED_MAX_BACKUPS_IN_HA: 4,
|
||||
Setting.DEPRECTAED_MAX_BACKUPS_IN_GOOGLE_DRIVE: 4,
|
||||
Setting.DEPRECATED_DAYS_BETWEEN_BACKUPS: 3,
|
||||
Setting.DEPRECTAED_IGNORE_OTHER_BACKUPS: False,
|
||||
Setting.DEPRECTAED_IGNORE_UPGRADE_BACKUPS: False,
|
||||
Setting.DEPRECTAED_BACKUP_TIME_OF_DAY: "",
|
||||
Setting.DEPRECTAED_BACKUP_NAME: "{type} Snapshot {year}-{month}-{day} {hr24}:{min}:{sec}",
|
||||
Setting.DEPRECATED_BACKUP_PASSWORD: "",
|
||||
Setting.DEPRECTAED_SPECIFY_BACKUP_FOLDER: False,
|
||||
Setting.WARN_FOR_LOW_SPACE: True,
|
||||
Setting.LOW_SPACE_THRESHOLD: 1024 * 1024 * 1024,
|
||||
Setting.DELETE_AFTER_UPLOAD: False,
|
||||
Setting.DEPRECTAED_DELETE_BEFORE_NEW_BACKUP: False,
|
||||
Setting.CALL_BACKUP_SNAPSHOT: False,
|
||||
|
||||
# Generational backup settings
|
||||
Setting.GENERATIONAL_DAYS: 0,
|
||||
Setting.GENERATIONAL_WEEKS: 0,
|
||||
Setting.GENERATIONAL_MONTHS: 0,
|
||||
Setting.GENERATIONAL_YEARS: 0,
|
||||
Setting.GENERATIONAL_DAY_OF_WEEK: "mon",
|
||||
Setting.GENERATIONAL_DAY_OF_MONTH: 1,
|
||||
Setting.GENERATIONAL_DAY_OF_YEAR: 1,
|
||||
Setting.GENERATIONAL_DELETE_EARLY: False,
|
||||
|
||||
# Partial backup settings
|
||||
Setting.EXCLUDE_FOLDERS: "",
|
||||
Setting.EXCLUDE_ADDONS: "",
|
||||
|
||||
Setting.STOP_ADDONS: "",
|
||||
Setting.DISABLE_WATCHDOG_WHEN_STOPPING: False,
|
||||
|
||||
# UI Server settings
|
||||
Setting.USE_SSL: False,
|
||||
Setting.REQUIRE_LOGIN: False,
|
||||
Setting.EXPOSE_EXTRA_SERVER: False,
|
||||
Setting.CERTFILE: "/ssl/fullchain.pem",
|
||||
Setting.KEYFILE: "/ssl/privkey.pem",
|
||||
Setting.INGRESS_PORT: 8099,
|
||||
Setting.PORT: 1627,
|
||||
|
||||
# Add-on options
|
||||
Setting.DEPRECTAED_NOTIFY_FOR_STALE_BACKUPS: True,
|
||||
Setting.DEPRECTAED_ENABLE_BACKUP_STALE_SENSOR: True,
|
||||
Setting.DEPRECTAED_ENABLE_BACKUP_STATE_SENSOR: True,
|
||||
Setting.SEND_ERROR_REPORTS: False,
|
||||
Setting.VERBOSE: False,
|
||||
Setting.CONFIRM_MULTIPLE_DELETES: True,
|
||||
Setting.ENABLE_DRIVE_UPLOAD: True,
|
||||
|
||||
# Theme Settings
|
||||
Setting.BACKGROUND_COLOR: "",
|
||||
Setting.ACCENT_COLOR: "",
|
||||
|
||||
# Network and DNS settings
|
||||
Setting.ALTERNATE_DNS_SERVERS: "8.8.8.8,8.8.4.4",
|
||||
Setting.DRIVE_EXPERIMENTAL: False,
|
||||
Setting.DRIVE_IPV4: "",
|
||||
Setting.IGNORE_IPV6_ADDRESSES: False,
|
||||
Setting.GOOGLE_DRIVE_TIMEOUT_SECONDS: 180,
|
||||
Setting.GOOGLE_DRIVE_PAGE_SIZE: 100,
|
||||
Setting.MAXIMUM_UPLOAD_CHUNK_BYTES: 10 * 1024 * 1024,
|
||||
|
||||
# Remote endpoints
|
||||
Setting.AUTHORIZATION_HOST: "https://habackup.io",
|
||||
Setting.TOKEN_SERVER_HOSTS: "https://token1.habackup.io,https://habackup.io",
|
||||
Setting.SUPERVISOR_URL: "",
|
||||
Setting.SUPERVISOR_TOKEN: "",
|
||||
Setting.DRIVE_URL: "https://www.googleapis.com",
|
||||
Setting.DRIVE_REFRESH_URL: "https://www.googleapis.com/oauth2/v4/token",
|
||||
Setting.DRIVE_AUTHORIZE_URL: "https://accounts.google.com/o/oauth2/v2/auth",
|
||||
Setting.DRIVE_DEVICE_CODE_URL: "https://oauth2.googleapis.com/device/code",
|
||||
Setting.DRIVE_TOKEN_URL: "https://oauth2.googleapis.com/token",
|
||||
Setting.DRIVE_HOST_NAME: "www.googleapis.com",
|
||||
Setting.SAVE_DRIVE_CREDS_PATH: "token",
|
||||
|
||||
# File locations used to store things
|
||||
Setting.FOLDER_FILE_PATH: "/data/folder.dat",
|
||||
Setting.CREDENTIALS_FILE_PATH: "/data/credentials.dat",
|
||||
Setting.BACKUP_DIRECTORY_PATH: "/backup",
|
||||
Setting.RETAINED_FILE_PATH: "/data/retained.json",
|
||||
Setting.SECRETS_FILE_PATH: "/config/secrets.yaml",
|
||||
Setting.INGRESS_TOKEN_FILE_PATH: "/data/ingress.dat",
|
||||
Setting.CONFIG_FILE_PATH: "/data/options.json",
|
||||
Setting.ID_FILE_PATH: "/data/id.json",
|
||||
Setting.STOP_ADDON_STATE_PATH: '/data/stop_addon_state.json',
|
||||
Setting.DATA_CACHE_FILE_PATH: '/data/data_cache.json',
|
||||
|
||||
# Various timeouts and intervals
|
||||
Setting.BACKUP_STALE_SECONDS: 60 * 60 * 3,
|
||||
Setting.PENDING_BACKUP_TIMEOUT_SECONDS: 60 * 60 * 5,
|
||||
Setting.FAILED_BACKUP_TIMEOUT_SECONDS: 60 * 15,
|
||||
Setting.NEW_BACKUP_TIMEOUT_SECONDS: 5,
|
||||
Setting.MAX_SYNC_INTERVAL_SECONDS: 60 * 60 * 3, # 3 hours
|
||||
Setting.DEFAULT_SYNC_INTERVAL_VARIATION: 0.5, # intermittent checkup syncs happen between 1.5 and 3 hours since the last one, randomly
|
||||
Setting.DEFAULT_DRIVE_CLIENT_ID: "933944288016-n35gnn2juc76ub7u5326ls0iaq9dgjgu.apps.googleusercontent.com",
|
||||
Setting.DEFAULT_DRIVE_CLIENT_SECRET: "",
|
||||
Setting.DRIVE_PICKER_API_KEY: "",
|
||||
Setting.DEFAULT_CHUNK_SIZE: 1024 * 1024 * 5,
|
||||
Setting.DOWNLOAD_TIMEOUT_SECONDS: 60,
|
||||
Setting.DEBUGGER_PORT: None,
|
||||
Setting.SERVER_PROJECT_ID: "",
|
||||
Setting.LOG_LEVEL: 'DEBUG',
|
||||
Setting.CONSOLE_LOG_LEVEL: 'INFO',
|
||||
Setting.BACKUP_STARTUP_DELAY_MINUTES: 10,
|
||||
Setting.EXCHANGER_TIMEOUT_SECONDS: 10,
|
||||
Setting.HA_REPORTING_INTERVAL_SECONDS: 10,
|
||||
Setting.LONG_TERM_STALE_BACKUP_SECONDS: 60 * 60 * 24,
|
||||
Setting.PING_TIMEOUT: 5,
|
||||
Setting.CACHE_WARMUP_MAX_SECONDS: 15 * 60, # 30 minutes
|
||||
Setting.CACHE_WARMUP_ERROR_TIMEOUT_SECONDS: 24 * 60 * 60, # 1 day
|
||||
Setting.MAX_BACKOFF_SECONDS: 60 * 60 * 2, # 2 hours
|
||||
}
|
||||
|
||||
_STAGING_DEFAULTS = {
|
||||
Setting.AUTHORIZATION_HOST: "https://dev.habackup.io",
|
||||
Setting.TOKEN_SERVER_HOSTS: "https://token1.dev.habackup.io,https://dev.habackup.io",
|
||||
Setting.DEFAULT_DRIVE_CLIENT_ID: "795575624694-jcdhoh1jr1ngccfsbi2f44arr4jupl79.apps.googleusercontent.com",
|
||||
}
|
||||
|
||||
_CONFIG = {
|
||||
Setting.MAX_BACKUPS_IN_HA: "int(0,)?",
|
||||
Setting.MAX_BACKUPS_IN_GOOGLE_DRIVE: "int(0,)?",
|
||||
Setting.DAYS_BETWEEN_BACKUPS: "float(0,)?",
|
||||
Setting.IGNORE_OTHER_BACKUPS: "bool?",
|
||||
Setting.IGNORE_UPGRADE_BACKUPS: "bool?",
|
||||
Setting.DELETE_IGNORED_AFTER_DAYS: "float(0,)?",
|
||||
Setting.DELETE_BEFORE_NEW_BACKUP: "bool?",
|
||||
Setting.BACKUP_NAME: "str?",
|
||||
Setting.BACKUP_TIME_OF_DAY: "match(^[0-2]\\d:[0-5]\\d$)?",
|
||||
Setting.SPECIFY_BACKUP_FOLDER: "bool?",
|
||||
Setting.NOTIFY_FOR_STALE_BACKUPS: "bool?",
|
||||
Setting.ENABLE_BACKUP_STALE_SENSOR: "bool?",
|
||||
Setting.ENABLE_BACKUP_STATE_SENSOR: "bool?",
|
||||
Setting.BACKUP_PASSWORD: "str?",
|
||||
Setting.BACKUP_STORAGE: "str?",
|
||||
Setting.WATCH_BACKUP_DIRECTORY: "bool?",
|
||||
Setting.TRACE_REQUESTS: "bool?",
|
||||
|
||||
# Basic backup settings
|
||||
Setting.DEPRECTAED_MAX_BACKUPS_IN_HA: "int(0,)?",
|
||||
Setting.DEPRECTAED_MAX_BACKUPS_IN_GOOGLE_DRIVE: "int(0,)?",
|
||||
Setting.DEPRECATED_DAYS_BETWEEN_BACKUPS: "float(0,)?",
|
||||
Setting.DEPRECTAED_IGNORE_OTHER_BACKUPS: "bool?",
|
||||
Setting.DEPRECTAED_IGNORE_UPGRADE_BACKUPS: "bool?",
|
||||
Setting.DEPRECTAED_BACKUP_TIME_OF_DAY: "match(^[0-2]\\d:[0-5]\\d$)?",
|
||||
Setting.DEPRECTAED_BACKUP_NAME: "str?",
|
||||
Setting.DEPRECATED_BACKUP_PASSWORD: "str?",
|
||||
Setting.DEPRECTAED_SPECIFY_BACKUP_FOLDER: "bool?",
|
||||
Setting.WARN_FOR_LOW_SPACE: "bool?",
|
||||
Setting.LOW_SPACE_THRESHOLD: "int(0,)?",
|
||||
Setting.DELETE_AFTER_UPLOAD: "bool?",
|
||||
Setting.DEPRECTAED_DELETE_BEFORE_NEW_BACKUP: "bool?",
|
||||
Setting.CALL_BACKUP_SNAPSHOT: "bool?",
|
||||
|
||||
# Generational backup settings
|
||||
Setting.GENERATIONAL_DAYS: "int(0,)?",
|
||||
Setting.GENERATIONAL_WEEKS: "int(0,)?",
|
||||
Setting.GENERATIONAL_MONTHS: "int(0,)?",
|
||||
Setting.GENERATIONAL_YEARS: "int(0,)?",
|
||||
Setting.GENERATIONAL_DAY_OF_WEEK: "match(^(mon|tue|wed|thu|fri|sat|sun)$)?",
|
||||
Setting.GENERATIONAL_DAY_OF_MONTH: "int(1,31)?",
|
||||
Setting.GENERATIONAL_DAY_OF_YEAR: "int(1,365)?",
|
||||
Setting.GENERATIONAL_DELETE_EARLY: "bool?",
|
||||
|
||||
# Partial backup settings
|
||||
Setting.EXCLUDE_FOLDERS: "str?",
|
||||
Setting.EXCLUDE_ADDONS: "str?",
|
||||
|
||||
Setting.STOP_ADDONS: "str?",
|
||||
Setting.DISABLE_WATCHDOG_WHEN_STOPPING: "bool?",
|
||||
|
||||
# UI Server settings
|
||||
Setting.USE_SSL: "bool?",
|
||||
Setting.REQUIRE_LOGIN: "bool?",
|
||||
Setting.EXPOSE_EXTRA_SERVER: "bool?",
|
||||
Setting.CERTFILE: "str?",
|
||||
Setting.KEYFILE: "str?",
|
||||
Setting.INGRESS_PORT: "int(0,)?",
|
||||
Setting.PORT: "int(0,)?",
|
||||
|
||||
# Add-on options
|
||||
Setting.DEPRECTAED_NOTIFY_FOR_STALE_BACKUPS: "bool?",
|
||||
Setting.DEPRECTAED_ENABLE_BACKUP_STALE_SENSOR: "bool?",
|
||||
Setting.DEPRECTAED_ENABLE_BACKUP_STATE_SENSOR: "bool?",
|
||||
Setting.SEND_ERROR_REPORTS: "bool?",
|
||||
Setting.VERBOSE: "bool?",
|
||||
Setting.CONFIRM_MULTIPLE_DELETES: "bool?",
|
||||
Setting.ENABLE_DRIVE_UPLOAD: "bool?",
|
||||
|
||||
# Theme Settings
|
||||
Setting.BACKGROUND_COLOR: "match(^(#[0-9ABCDEFabcdef]{6}|)$)?",
|
||||
Setting.ACCENT_COLOR: "match(^(#[0-9ABCDEFabcdef]{6}|)$)?",
|
||||
|
||||
# Network and DNS settings
|
||||
Setting.ALTERNATE_DNS_SERVERS: "match(^([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3})(,[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3})*$)?",
|
||||
Setting.DRIVE_EXPERIMENTAL: "bool?",
|
||||
Setting.DRIVE_IPV4: "match(^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}$)?",
|
||||
Setting.IGNORE_IPV6_ADDRESSES: "bool?",
|
||||
Setting.GOOGLE_DRIVE_TIMEOUT_SECONDS: "float(1,)?",
|
||||
Setting.GOOGLE_DRIVE_PAGE_SIZE: "int(1,)?",
|
||||
Setting.MAXIMUM_UPLOAD_CHUNK_BYTES: f"float({1024 * 256},)?",
|
||||
|
||||
# Remote endpoints
|
||||
Setting.AUTHORIZATION_HOST: "url?",
|
||||
Setting.TOKEN_SERVER_HOSTS: "str?",
|
||||
Setting.SUPERVISOR_URL: "url?",
|
||||
Setting.SUPERVISOR_TOKEN: "str?",
|
||||
Setting.DRIVE_URL: "url?",
|
||||
Setting.DRIVE_REFRESH_URL: "url?",
|
||||
Setting.DRIVE_AUTHORIZE_URL: "url?",
|
||||
Setting.DRIVE_DEVICE_CODE_URL: "url?",
|
||||
Setting.DRIVE_TOKEN_URL: "url?",
|
||||
Setting.DRIVE_HOST_NAME: "str?",
|
||||
Setting.SAVE_DRIVE_CREDS_PATH: "str?",
|
||||
|
||||
# File locations used to store things
|
||||
Setting.FOLDER_FILE_PATH: "str?",
|
||||
Setting.CREDENTIALS_FILE_PATH: "str?",
|
||||
Setting.BACKUP_DIRECTORY_PATH: "str?",
|
||||
Setting.RETAINED_FILE_PATH: "str?",
|
||||
Setting.SECRETS_FILE_PATH: "str?",
|
||||
Setting.INGRESS_TOKEN_FILE_PATH: "str?",
|
||||
Setting.CONFIG_FILE_PATH: "str?",
|
||||
Setting.ID_FILE_PATH: "str?",
|
||||
Setting.STOP_ADDON_STATE_PATH: "str?",
|
||||
Setting.DATA_CACHE_FILE_PATH: "str?",
|
||||
|
||||
# Various timeouts and intervals
|
||||
Setting.BACKUP_STALE_SECONDS: "float(0,)?",
|
||||
Setting.PENDING_BACKUP_TIMEOUT_SECONDS: "float(0,)?",
|
||||
Setting.FAILED_BACKUP_TIMEOUT_SECONDS: "float(0,)?",
|
||||
Setting.NEW_BACKUP_TIMEOUT_SECONDS: "float(0,)?",
|
||||
Setting.MAX_SYNC_INTERVAL_SECONDS: "float(300,)?",
|
||||
Setting.DEFAULT_SYNC_INTERVAL_VARIATION: "float(0,1)?",
|
||||
Setting.DEFAULT_DRIVE_CLIENT_ID: "str?",
|
||||
Setting.DEFAULT_DRIVE_CLIENT_SECRET: "str?",
|
||||
Setting.DRIVE_PICKER_API_KEY: "str?",
|
||||
Setting.DEFAULT_CHUNK_SIZE: "int(1,)?",
|
||||
Setting.DOWNLOAD_TIMEOUT_SECONDS: "float(0,)?",
|
||||
Setting.DEBUGGER_PORT: "int(100,)?",
|
||||
Setting.SERVER_PROJECT_ID: "str?",
|
||||
Setting.LOG_LEVEL: "list(DEBUG|TRACE|INFO|WARN|CRITICAL|WARNING)?",
|
||||
Setting.CONSOLE_LOG_LEVEL: "list(DEBUG|TRACE|INFO|WARN|CRITICAL|WARNING)?",
|
||||
Setting.BACKUP_STARTUP_DELAY_MINUTES: "float(0,)?",
|
||||
Setting.EXCHANGER_TIMEOUT_SECONDS: "float(0,)?",
|
||||
Setting.HA_REPORTING_INTERVAL_SECONDS: "int(1,)?",
|
||||
Setting.LONG_TERM_STALE_BACKUP_SECONDS: "int(1,)?",
|
||||
Setting.PING_TIMEOUT: "float(0,)?",
|
||||
Setting.CACHE_WARMUP_MAX_SECONDS: "float(0,)",
|
||||
Setting.CACHE_WARMUP_ERROR_TIMEOUT_SECONDS: "float(0,)",
|
||||
Setting.MAX_BACKOFF_SECONDS: "int(3600,)?",
|
||||
}
|
||||
|
||||
PRIVATE = [
|
||||
Setting.DEPRECATED_BACKUP_PASSWORD,
|
||||
Setting.DEPRECTAED_BACKUP_NAME,
|
||||
Setting.BACKUP_PASSWORD,
|
||||
Setting.BACKUP_NAME
|
||||
]
|
||||
|
||||
_LOOKUP = {}
|
||||
_VALIDATORS = {}
|
||||
|
||||
|
||||
def getValidator(name, schema):
|
||||
if schema.endswith("?"):
|
||||
schema = schema[:-1]
|
||||
if schema.startswith("int("):
|
||||
# its a int
|
||||
parts = schema[4:-1]
|
||||
minimum = None
|
||||
maximum = None
|
||||
if parts.endswith(","):
|
||||
minimum = int(parts[0:-1])
|
||||
elif parts.startswith(","):
|
||||
maximum = int(parts[1:])
|
||||
else:
|
||||
digits = parts.split(",")
|
||||
minimum = int(digits[0])
|
||||
maximum = int(digits[1])
|
||||
return IntValidator(name, minimum, maximum)
|
||||
elif schema.startswith("float("):
|
||||
# its a float
|
||||
parts = schema[6:-1]
|
||||
minimum = None
|
||||
maximum = None
|
||||
if parts.endswith(","):
|
||||
minimum = float(parts[0:-1])
|
||||
elif parts.startswith(","):
|
||||
maximum = float(parts[1:])
|
||||
else:
|
||||
digits = parts.split(",")
|
||||
minimum = float(digits[0])
|
||||
maximum = float(digits[1])
|
||||
return FloatValidator(name, minimum, maximum)
|
||||
elif schema.startswith("bool"):
|
||||
# its a bool
|
||||
return BoolValidator(name)
|
||||
elif schema.startswith("str") or schema.startswith("url"):
|
||||
# its a url (treat it just like any string)
|
||||
return StringValidator(name)
|
||||
elif schema.startswith("match("):
|
||||
return RegexValidator(name, schema[6:-1])
|
||||
elif schema.startswith("list("):
|
||||
return ListValidator(name, schema[5:-1].split("|"))
|
||||
else:
|
||||
raise Exception("Invalid schema: " + schema)
|
||||
|
||||
|
||||
# initalize validators
|
||||
for setting in Setting:
|
||||
_LOOKUP[setting.value] = setting
|
||||
|
||||
with open(abspath(join(__file__, "..", "..", "..", "config.json"))) as f:
|
||||
# Thsi is a static file included in the container, so don't worry about using JsonFileLoader
|
||||
addon_config = json.load(f)
|
||||
|
||||
for setting in Setting:
|
||||
_VALIDATORS[setting] = getValidator(setting.value, _CONFIG[setting])
|
||||
for key in addon_config["schema"]:
|
||||
_VALIDATORS[_LOOKUP[key]] = getValidator(key, addon_config["schema"][key])
|
||||
|
||||
_VALIDATORS[Setting.MAX_SYNC_INTERVAL_SECONDS] = DurationAsStringValidator("max_sync_interval_seconds", minimum=1, maximum=None)
|
||||
_VALIDATORS[Setting.HA_REPORTING_INTERVAL_SECONDS] = DurationAsStringValidator("ha_reporting_interval_seconds", minimum=1, maximum=None)
|
||||
_VALIDATORS[Setting.DELETE_IGNORED_AFTER_DAYS] = DurationAsStringValidator("delete_ignored_after_days", minimum=0, maximum=None, base_seconds=60 * 60 * 24, default_as_empty=0)
|
||||
_VALIDATORS[Setting.MAXIMUM_UPLOAD_CHUNK_BYTES] = BytesizeAsStringValidator("maximum_upload_chunk_bytes", minimum=256 * 1024)
|
||||
VERSION = addon_config["version"]
|
||||
|
||||
|
||||
def isStaging():
|
||||
return "staging" in VERSION
|
||||
@@ -0,0 +1,11 @@
|
||||
from ..logger import getLogger
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
|
||||
class Startable():
|
||||
async def start(self):
|
||||
pass
|
||||
|
||||
async def stop(self):
|
||||
pass
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
from .validator import Validator
|
||||
from ..logger import getLogger
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
|
||||
class StringValidator(Validator):
|
||||
def __init__(self, name):
|
||||
super().__init__(name)
|
||||
|
||||
def validate(self, value):
|
||||
if value is None or (type(value) == str and len(value) == 0):
|
||||
return ""
|
||||
return str(value)
|
||||
@@ -0,0 +1,21 @@
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
from ..exceptions import InvalidConfigurationValue
|
||||
from ..logger import getLogger
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
|
||||
class Validator(ABC):
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
|
||||
@abstractmethod
|
||||
def validate(self, value):
|
||||
return True
|
||||
|
||||
def raiseForValue(self, value):
|
||||
raise InvalidConfigurationValue(self.name, str(value))
|
||||
|
||||
def formatForUi(self, value):
|
||||
return value
|
||||
@@ -0,0 +1,87 @@
|
||||
STAGING_KEY = ".staging."
|
||||
EXPECTED_VERISON_CHARS = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '.']
|
||||
|
||||
class Version:
|
||||
def __init__(self, *args):
|
||||
self._identifiers = args
|
||||
self.staging = False
|
||||
|
||||
@classmethod
|
||||
def default(cls):
|
||||
return Version(0)
|
||||
|
||||
@classmethod
|
||||
def max(cls):
|
||||
return Version(99999999)
|
||||
|
||||
@classmethod
|
||||
def parse(cls, version: str):
|
||||
staging_version = None
|
||||
if STAGING_KEY in version:
|
||||
index = version.find(STAGING_KEY)
|
||||
staging_version = int(version[index + len(STAGING_KEY):])
|
||||
version = version[0:index]
|
||||
version = Version._removeUnexpected(version)
|
||||
parts = []
|
||||
for part in version.split("."):
|
||||
if len(part) > 0:
|
||||
parts.append(int(part))
|
||||
if staging_version is not None:
|
||||
parts.append(staging_version)
|
||||
if len(parts) == 0:
|
||||
parts.append(0)
|
||||
ret = Version(*parts)
|
||||
if staging_version is not None:
|
||||
ret.staging = True
|
||||
return ret
|
||||
|
||||
@classmethod
|
||||
def _removeUnexpected(cls, version: str):
|
||||
ret = ""
|
||||
for c in version:
|
||||
if c in EXPECTED_VERISON_CHARS:
|
||||
ret += c
|
||||
while ".." in ret:
|
||||
ret = ret.replace("..", ".")
|
||||
return ret
|
||||
|
||||
def __getitem__(self, key):
|
||||
return self._identifiers[key]
|
||||
|
||||
def length(self):
|
||||
return len(self._identifiers)
|
||||
|
||||
def _compare(self, other):
|
||||
i = 0
|
||||
while(i < min(self.length(), other.length())):
|
||||
if self[i] < other[i]:
|
||||
return -1
|
||||
if self[i] > other[i]:
|
||||
return 1
|
||||
i += 1
|
||||
if self.length() < other.length():
|
||||
return -1
|
||||
if self.length() > other.length():
|
||||
return 1
|
||||
return 0
|
||||
|
||||
def __lt__(self, other):
|
||||
return self._compare(other) < 0
|
||||
|
||||
def __le__(self, other):
|
||||
return self._compare(other) <= 0
|
||||
|
||||
def __eq__(self, other):
|
||||
return self._compare(other) == 0
|
||||
|
||||
def __ne__(self, other):
|
||||
return self._compare(other) != 0
|
||||
|
||||
def __gt__(self, other):
|
||||
return self._compare(other) > 0
|
||||
|
||||
def __ge__(self, other):
|
||||
return self._compare(other) >= 0
|
||||
|
||||
def __str__(self):
|
||||
return ".".join(str(i) for i in self._identifiers)
|
||||
@@ -0,0 +1,132 @@
|
||||
|
||||
SOURCE_GOOGLE_DRIVE = "GoogleDrive"
|
||||
SOURCE_HA = "HomeAssistant"
|
||||
|
||||
ERROR_PLEASE_WAIT = "please_wait"
|
||||
ERROR_NOT_UPLOADABLE = "not_uploadable"
|
||||
ERROR_NO_BACKUP = "invalid_slug"
|
||||
ERROR_CREDS_EXPIRED = "creds_bad"
|
||||
ERROR_UPLOAD_FAILED = "upload_failed"
|
||||
ERROR_BAD_PASSWORD_KEY = "password_key_invalid"
|
||||
ERROR_BACKUP_IN_PROGRESS = "backup_in_progress"
|
||||
ERROR_PROTOCOL = "protocol_error"
|
||||
ERROR_LOGIC = "logic_error"
|
||||
ERROR_INVALID_CONFIG = "illegal_config"
|
||||
ERROR_DRIVE_FULL = "drive_full"
|
||||
ERROR_GOOGLE_DNS = "google_dns"
|
||||
ERROR_GOOGLE_CONNECT = "google_cant_connect"
|
||||
ERROR_GOOGLE_INTERNAL = "google_server_error"
|
||||
ERROR_GOOGLE_SESSION = "google_session_expired"
|
||||
ERROR_GOOGLE_TIMEOUT = "google_timeout"
|
||||
ERROR_GOOGLE_UNEXPECTED = "google_unexpected"
|
||||
ERROR_HA_DELETE_ERROR = "delete_error"
|
||||
ERROR_MULTIPLE_DELETES = "multiple_deletes"
|
||||
ERROR_SUPERVISOR_UNEXPECTED = "supervisor_unexpected"
|
||||
ERROR_SUPERVISOR_TIMEOUT = "supervisor_timeout"
|
||||
ERROR_SUPERVISOR_FILE_SYSTEM = "supervisor_fs_error"
|
||||
ERROR_GOOGLE_CRED_PROCESS = "unable_to_make_creds"
|
||||
|
||||
ERROR_EXISTING_FOLDER = "existing_backup_folder"
|
||||
ERROR_BACKUP_FOLDER_MISSING = "backup_folder_missing"
|
||||
CHOOSE_BACKUP_FOLDER = "choose_backup_folder"
|
||||
ERROR_BACKUP_FOLDER_INACCESSIBLE = "backup_folder_inaccessible"
|
||||
ERROR_LOW_SPACE = "low_space"
|
||||
LOG_IN_TO_DRIVE = "log_in_to_drive"
|
||||
SUPERVISOR_PERMISSION = "supervisor_permission"
|
||||
|
||||
# Network storage errors
|
||||
UNKONWN_NETWORK_STORAGE = "unknown_network_storage"
|
||||
INACTIVE_NETWORK_STORAGE = "inactive_network_storage"
|
||||
|
||||
# these keys are necessary because they use the name "snapshot" in non-user-visible
|
||||
# places persisted outside the codebase. They can't be changed without an upgrade path.
|
||||
NECESSARY_OLD_BACKUP_NAME = "snapshot"
|
||||
NECESSARY_OLD_BACKUP_PLURAL_NAME = "snapshots"
|
||||
NECESSARY_OLD_SUPERVISOR_URL = "http://hassio"
|
||||
NECESSARY_PROP_KEY_SLUG = "snapshot_slug"
|
||||
NECESSARY_PROP_KEY_DATE = "snapshot_date"
|
||||
NECESSARY_PROP_KEY_NAME = "snapshot_name"
|
||||
PROP_NOTE = "note"
|
||||
|
||||
DRIVE_FOLDER_URL_FORMAT = "https://drive.google.com/drive/u/0/folders/{0}"
|
||||
GITHUB_ISSUE_URL = "https://github.com/sabeechen/hassio-google-drive-backup/issues/new?labels[]=People%20Management&labels[]=[Type]%20Bug&title={title}&assignee=sabeechen&body={body}"
|
||||
GITHUB_BUG_TEMPLATE = """
|
||||
###### Description:
|
||||
```
|
||||
If you have anything else that could help explain what happened, click "Markdown" above and write it here.
|
||||
```
|
||||
|
||||
Addon version: `{version}`
|
||||
Home Assistant Version: `{ha_version}`
|
||||
Supervisor Version: `{super_version}`
|
||||
Supervisor Channel: `{supervisor_channel}`
|
||||
Hassos Version: `{hassos_version}`
|
||||
Docker Version: `{docker_version}`
|
||||
Architecture: `{arch}`
|
||||
Machine: `{machine}`
|
||||
Date: `{time}`
|
||||
Timezone: `{timezone}`
|
||||
Failure Time: `{failure_time}`
|
||||
Last Good Sync: `{sync_last_start}`
|
||||
Next Sync: `{next_sync}`
|
||||
Next Backup: `{next_backup}`
|
||||
Next Cache Warm: `{next_cache_warm}`
|
||||
Time Offset: `{time_offset}`
|
||||
###### Exception:
|
||||
```
|
||||
{error}
|
||||
```
|
||||
Backups:
|
||||
```
|
||||
{backups}
|
||||
```
|
||||
###### Config:
|
||||
```
|
||||
{config}
|
||||
```
|
||||
###### Addon Logs:
|
||||
```
|
||||
{addon_logs}
|
||||
```
|
||||
###### Supervisor Logs:
|
||||
```
|
||||
{super_logs}
|
||||
```
|
||||
###### Home Assistant Core Logs:
|
||||
```
|
||||
{core_logs}
|
||||
```
|
||||
"""
|
||||
|
||||
FOLDERS = [
|
||||
{
|
||||
'slug': "homeassistant",
|
||||
'id': "folder_homeassistant",
|
||||
'name': "Home Assistant Configuration",
|
||||
'description': 'Backup the files and folders from your Home Assistant config directory, eg configuration.yaml'
|
||||
},
|
||||
{
|
||||
'slug': "media",
|
||||
'id': "folder_media",
|
||||
'name': "Media",
|
||||
'description': 'Backup your "/media" directory.'
|
||||
},
|
||||
{
|
||||
'slug': "ssl",
|
||||
'id': "folder_ssl",
|
||||
'name': "SSL",
|
||||
'description': 'Backup your "/ssl" directory, where your certfile and keyfile are typically stored.'
|
||||
},
|
||||
{
|
||||
'slug': "share",
|
||||
'id': "folder_share",
|
||||
'name': "Share",
|
||||
'description': 'Backup your "/share" directory.'
|
||||
},
|
||||
{
|
||||
'slug': "addons/local",
|
||||
'id': "folder_addons",
|
||||
'name': "Local Addons",
|
||||
'description': 'Backup your local addons directory. This directory will be empty unless you use it for add-on development.'
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,5 @@
|
||||
# flake8: noqa
|
||||
from .exchanger import Exchanger
|
||||
from .creds import Creds, KEY_TOKEN_EXPIRY, KEY_ACCESS_TOKEN, KEY_CLIENT_ID, KEY_CLIENT_SECRET
|
||||
from .driverequester import DriveRequester
|
||||
MANUAL_CODE_REDIRECT_URI: str = "urn:ietf:wg:oauth:2.0:oob"
|
||||
@@ -0,0 +1,90 @@
|
||||
from ..exceptions import ensureKey
|
||||
from ..time import Time
|
||||
from typing import Optional
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
KEY_REFRESH_TOKEN = 'refresh_token'
|
||||
KEY_CLIENT_ID = 'client_id'
|
||||
KEY_CLIENT_SECRET = 'client_secret'
|
||||
KEY_EXPIRES_IN = 'expires_in'
|
||||
KEY_TOKEN_EXPIRY = 'token_expiry'
|
||||
KEY_ACCESS_TOKEN = 'access_token'
|
||||
|
||||
|
||||
class Creds():
|
||||
def __init__(self, time: Time, id: str, expiration: datetime,
|
||||
access_token: str, refresh_token: str,
|
||||
secret: Optional[str] = None, original_expiration: datetime = None):
|
||||
self._id = id
|
||||
self.time: Time = time
|
||||
self._secret = secret
|
||||
self._access_token = access_token
|
||||
self._refresh_token = refresh_token
|
||||
self._expiration = expiration
|
||||
self._original_expiration = original_expiration
|
||||
|
||||
@property
|
||||
def id(self):
|
||||
return self._id
|
||||
|
||||
@property
|
||||
def secret(self):
|
||||
return self._secret
|
||||
|
||||
@property
|
||||
def refresh_token(self):
|
||||
return self._refresh_token
|
||||
|
||||
@property
|
||||
def access_token(self):
|
||||
return self._access_token
|
||||
|
||||
@property
|
||||
def expiration(self):
|
||||
if self._expiration is None:
|
||||
return self.time.now()
|
||||
return self._expiration
|
||||
|
||||
@property
|
||||
def original_expiration(self) -> datetime:
|
||||
return self._original_expiration
|
||||
|
||||
@property
|
||||
def is_expired(self):
|
||||
return self.time.now() >= self.expiration
|
||||
|
||||
def serialize(self, include_secret=True):
|
||||
ret = {
|
||||
"client_id": self.id
|
||||
}
|
||||
if self.secret is not None and include_secret:
|
||||
ret[KEY_CLIENT_SECRET] = self.secret
|
||||
if self.refresh_token is not None:
|
||||
ret[KEY_REFRESH_TOKEN] = self.refresh_token
|
||||
if self.access_token is not None:
|
||||
ret[KEY_ACCESS_TOKEN] = self.access_token
|
||||
if self.expiration is not None:
|
||||
ret[KEY_TOKEN_EXPIRY] = self.time.asRfc3339String(self.expiration)
|
||||
return ret
|
||||
|
||||
@classmethod
|
||||
def load(cls, time: Time, data, id=None, secret=None, original_expiration=None):
|
||||
if id is None:
|
||||
id = ensureKey(KEY_CLIENT_ID, data, "credentials")
|
||||
if secret is None and KEY_CLIENT_SECRET in data:
|
||||
secret = data[KEY_CLIENT_SECRET]
|
||||
refresh = ensureKey(KEY_REFRESH_TOKEN, data, "credentials")
|
||||
access = ensureKey(KEY_ACCESS_TOKEN, data, "credentials")
|
||||
expires = None
|
||||
try:
|
||||
if KEY_TOKEN_EXPIRY in data:
|
||||
expires = time.parse(data[KEY_TOKEN_EXPIRY])
|
||||
if original_expiration is None:
|
||||
original_expiration = expires
|
||||
elif KEY_EXPIRES_IN in data:
|
||||
expires = time.now() + timedelta(seconds=int(data[KEY_EXPIRES_IN]))
|
||||
else:
|
||||
expires = time.now()
|
||||
except BaseException:
|
||||
expires = time.now()
|
||||
return Creds(time=time, id=id, access_token=access, refresh_token=refresh, secret=secret, expiration=expires, original_expiration=original_expiration)
|
||||
@@ -0,0 +1,113 @@
|
||||
from aiohttp import ClientSession, ContentTypeError, ClientConnectorError, ClientTimeout, ClientResponse
|
||||
from aiohttp.client_exceptions import ServerTimeoutError, ServerDisconnectedError, ClientOSError
|
||||
from backup.exceptions import GoogleUnexpectedError, GoogleInternalError, GoogleRateLimitError, GoogleCredentialsExpired, CredRefreshGoogleError, DriveQuotaExceeded, GoogleDrivePermissionDenied, GoogleDnsFailure, GoogleCantConnect, GoogleTimeoutError
|
||||
from backup.util import Resolver
|
||||
from backup.logger import getLogger
|
||||
from backup.config import Config, Setting
|
||||
from injector import singleton, inject
|
||||
from dns.exception import DNSException
|
||||
|
||||
RATE_LIMIT_EXCEEDED = [403]
|
||||
TOO_MANY_REQUESTS = [429]
|
||||
INTERNAL_ERROR = [500, 503]
|
||||
PERMISSION_DENIED = [401]
|
||||
REQUEST_TIMEOUT = [408]
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
|
||||
@singleton
|
||||
class DriveRequester():
|
||||
@inject
|
||||
def __init__(self, config: Config, session: ClientSession, resolver: Resolver):
|
||||
self.session = session
|
||||
self.resolver = resolver
|
||||
self.config = config
|
||||
|
||||
async def request(self, method, url, headers={}, json=None, data=None) -> ClientResponse:
|
||||
try:
|
||||
response = await self.session.request(method, url, headers=headers, json=json, timeout=self.buildTimeout(), data=data)
|
||||
if response.status < 400:
|
||||
return response
|
||||
await self.raiseForKnownErrors(response)
|
||||
if response.status in PERMISSION_DENIED:
|
||||
response.release()
|
||||
raise GoogleCredentialsExpired()
|
||||
elif response.status in INTERNAL_ERROR:
|
||||
response.release()
|
||||
raise GoogleInternalError()
|
||||
elif response.status in RATE_LIMIT_EXCEEDED or response.status in TOO_MANY_REQUESTS:
|
||||
response.release()
|
||||
raise GoogleRateLimitError()
|
||||
elif response.status in REQUEST_TIMEOUT:
|
||||
response.release()
|
||||
raise GoogleTimeoutError()
|
||||
response.raise_for_status()
|
||||
return response
|
||||
except ClientConnectorError as e:
|
||||
logger.debug(
|
||||
"Ran into trouble reaching Google Drive's servers. We'll use alternate DNS servers on the next attempt.")
|
||||
self.resolver.toggle()
|
||||
if "Cannot connect to host" in str(e) or "Connection reset by peer" in str(e):
|
||||
raise GoogleCantConnect()
|
||||
if e.os_error.errno == -2:
|
||||
# -2 means dns lookup failed.
|
||||
raise GoogleDnsFailure()
|
||||
elif str(e.os_error) == "Domain name not found":
|
||||
raise GoogleDnsFailure()
|
||||
elif e.os_error.errno in [99, 111, 10061, 104]:
|
||||
# 111 means connection refused
|
||||
# Can't connect
|
||||
raise GoogleCantConnect()
|
||||
elif "Could not contact DNS serve" in str(e.os_error):
|
||||
# Wish there was a better way to identify this exception
|
||||
raise GoogleDnsFailure()
|
||||
raise
|
||||
except ClientOSError as e:
|
||||
if e.errno == 1:
|
||||
raise GoogleUnexpectedError()
|
||||
raise
|
||||
except ServerTimeoutError:
|
||||
raise GoogleTimeoutError()
|
||||
except ServerDisconnectedError:
|
||||
raise GoogleUnexpectedError()
|
||||
except DNSException:
|
||||
logger.debug(
|
||||
"Ran into trouble resolving Google Drive's servers. We'll use normal DNS servers on the next attempt.")
|
||||
self.resolver.toggle()
|
||||
raise GoogleDnsFailure()
|
||||
|
||||
def buildTimeout(self):
|
||||
return ClientTimeout(
|
||||
sock_connect=self.config.get(
|
||||
Setting.GOOGLE_DRIVE_TIMEOUT_SECONDS),
|
||||
sock_read=self.config.get(Setting.GOOGLE_DRIVE_TIMEOUT_SECONDS))
|
||||
|
||||
async def raiseForKnownErrors(self, response):
|
||||
try:
|
||||
message = await response.json()
|
||||
except ContentTypeError:
|
||||
return
|
||||
except ValueError:
|
||||
# parsing json failed, just give up
|
||||
return
|
||||
except TypeError:
|
||||
# Same
|
||||
return
|
||||
if "error" not in message:
|
||||
return
|
||||
error_obj = message["error"]
|
||||
if isinstance(error_obj, str):
|
||||
if error_obj == "expired":
|
||||
raise GoogleCredentialsExpired()
|
||||
else:
|
||||
raise CredRefreshGoogleError(error_obj)
|
||||
if "errors" not in error_obj:
|
||||
return
|
||||
for error in error_obj["errors"]:
|
||||
if "reason" not in error:
|
||||
continue
|
||||
if error["reason"] == "storageQuotaExceeded":
|
||||
raise DriveQuotaExceeded()
|
||||
elif error["reason"] in ["forbidden", "insufficientFilePermissions"]:
|
||||
raise GoogleDrivePermissionDenied()
|
||||
@@ -0,0 +1,160 @@
|
||||
import asyncio
|
||||
from aiohttp import ClientSession, ClientConnectorError, ClientTimeout
|
||||
from .creds import Creds, KEY_CLIENT_ID, KEY_CLIENT_SECRET, KEY_ACCESS_TOKEN, KEY_REFRESH_TOKEN, KEY_EXPIRES_IN
|
||||
from ..exceptions import ensureKey, GoogleCredentialsExpired, CredRefreshGoogleError, CredRefreshMyError
|
||||
from ..config import Config, Setting, VERSION
|
||||
from yarl import URL
|
||||
from ..time import Time
|
||||
from ..logger import getLogger
|
||||
from .driverequester import DriveRequester
|
||||
from datetime import timedelta
|
||||
from injector import singleton, inject
|
||||
|
||||
|
||||
SCOPE = 'https://www.googleapis.com/auth/drive.file'
|
||||
|
||||
KEY_REDIRECT_URI = 'redirect_uri'
|
||||
KEY_SCOPE = 'scope'
|
||||
KEY_RESPONSE_TYPE = 'response_type'
|
||||
KEY_INCLUDE_GRANTED_SCOPES = 'include_granted_scopes'
|
||||
KEY_ACCESS_TYPE = 'access_type'
|
||||
KEY_STATE = 'state'
|
||||
KEY_PROMPT = 'prompt'
|
||||
KEY_CODE = 'code'
|
||||
KEY_GRANT_TYPE = 'grant_type'
|
||||
KEY_VERSION = 'version'
|
||||
KEY_CLIENT = 'client'
|
||||
|
||||
CRED_OBJECT_NAME = "credential token response"
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
|
||||
@singleton
|
||||
class Exchanger():
|
||||
@inject
|
||||
def __init__(self,
|
||||
time: Time,
|
||||
session: ClientSession,
|
||||
config: Config,
|
||||
drive: DriveRequester,
|
||||
client_id: str,
|
||||
client_secret: str,
|
||||
redirect: URL):
|
||||
self.time = time
|
||||
self.config = config
|
||||
self.session = session
|
||||
self.drive = drive
|
||||
self._client_id = client_id
|
||||
self._client_secret = client_secret
|
||||
self._redirect = redirect
|
||||
|
||||
async def getAuthorizationUrl(self, state="") -> str:
|
||||
url = URL(self.config.get(Setting.DRIVE_AUTHORIZE_URL)).with_query({
|
||||
KEY_CLIENT_ID: self._client_id,
|
||||
KEY_SCOPE: SCOPE,
|
||||
KEY_RESPONSE_TYPE: 'code',
|
||||
KEY_INCLUDE_GRANTED_SCOPES: 'true',
|
||||
KEY_ACCESS_TYPE: "offline",
|
||||
KEY_STATE: state,
|
||||
KEY_REDIRECT_URI: str(self._redirect),
|
||||
KEY_PROMPT: "consent"
|
||||
})
|
||||
return str(url)
|
||||
|
||||
async def exchange(self, code):
|
||||
data = {
|
||||
KEY_CLIENT_ID: self._client_id,
|
||||
KEY_CLIENT_SECRET: self._client_secret,
|
||||
KEY_CODE: code,
|
||||
KEY_REDIRECT_URI: str(self._redirect),
|
||||
KEY_GRANT_TYPE: 'authorization_code'
|
||||
}
|
||||
resp = None
|
||||
async with await self.drive.request("post", self.config.get(Setting.DRIVE_TOKEN_URL), data=data) as resp:
|
||||
return Creds.load(self.time, await resp.json(), id=self._client_id, secret=self._client_secret)
|
||||
|
||||
async def refresh(self, creds: Creds):
|
||||
if creds.secret is not None:
|
||||
return await self._refresh_google(creds)
|
||||
else:
|
||||
return await self._refresh_default(creds)
|
||||
|
||||
async def _refresh_google(self, creds: Creds):
|
||||
data = {
|
||||
KEY_CLIENT_ID: creds.id,
|
||||
KEY_CLIENT_SECRET: creds.secret,
|
||||
KEY_REFRESH_TOKEN: creds.refresh_token,
|
||||
KEY_GRANT_TYPE: 'refresh_token'
|
||||
}
|
||||
async with await self.drive.request("post", self.config.get(Setting.DRIVE_REFRESH_URL), data=data) as resp:
|
||||
data = await resp.json()
|
||||
return Creds(
|
||||
self.time,
|
||||
id=creds.id,
|
||||
secret=creds.secret,
|
||||
access_token=ensureKey(KEY_ACCESS_TOKEN, data, CRED_OBJECT_NAME),
|
||||
refresh_token=creds.refresh_token,
|
||||
expiration=self._get_expiration(data),
|
||||
original_expiration=creds.original_expiration)
|
||||
|
||||
async def _refresh_default(self, creds: Creds):
|
||||
data = {
|
||||
KEY_CLIENT_ID: creds.id,
|
||||
KEY_REFRESH_TOKEN: creds.refresh_token,
|
||||
}
|
||||
token_paths = self.config.getTokenServers("/drive/refresh")
|
||||
last_error = None
|
||||
for url in token_paths:
|
||||
try:
|
||||
headers = {
|
||||
'addon_version': VERSION,
|
||||
'client': self.config.clientIdentifier()
|
||||
}
|
||||
async with self.session.post(str(url), headers=headers, json=data, timeout=ClientTimeout(total=self.config.get(Setting.EXCHANGER_TIMEOUT_SECONDS))) as resp:
|
||||
if resp.status < 400:
|
||||
return Creds.load(self.time, await resp.json(), original_expiration=creds.original_expiration)
|
||||
elif resp.status == 503:
|
||||
json = {}
|
||||
try:
|
||||
json = await resp.json()
|
||||
except BaseException:
|
||||
pass
|
||||
if "error" in json:
|
||||
if "invalid_grant" in json["error"]:
|
||||
raise GoogleCredentialsExpired()
|
||||
else:
|
||||
# Record the error, but still try other hosts
|
||||
last_error = CredRefreshGoogleError(json["error"])
|
||||
else:
|
||||
last_error = CredRefreshMyError("HTTP 503 from " + url.host)
|
||||
elif resp.status == 401:
|
||||
raise GoogleCredentialsExpired()
|
||||
else:
|
||||
try:
|
||||
extra = (await resp.json())["error"]
|
||||
except BaseException:
|
||||
extra = ""
|
||||
|
||||
# this is likely due to misconfiguration
|
||||
logger.warning("Got {0}:{1} from {2}, trying alternate server(s)...".format(resp.status, extra, url.host))
|
||||
last_error = CredRefreshMyError("HTTP {} {}".format(resp.status, extra))
|
||||
except ClientConnectorError:
|
||||
logger.warning("Unable to reach " + str(url.host) + ", trying alternate server(s)...")
|
||||
last_error = "Couldn't communicate with " + url.host
|
||||
except asyncio.exceptions.TimeoutError:
|
||||
logger.warning("Timed out communicating with " + str(url.host) + ", trying alternate server(s)...")
|
||||
last_error = "Timed out communicating with " + url.host
|
||||
logger.error("Unable to refresh credentials with Google Drive")
|
||||
if isinstance(last_error, str):
|
||||
raise CredRefreshMyError(last_error)
|
||||
elif isinstance(last_error, Exception):
|
||||
raise last_error
|
||||
else:
|
||||
raise Exception("Unexpected error type: " + str(last_error))
|
||||
|
||||
def refreshCredentials(self, refresh_token):
|
||||
return Creds(self.time, id=self._client_id, expiration=None, access_token=None, refresh_token=refresh_token, secret=self._client_secret)
|
||||
|
||||
def _get_expiration(self, data):
|
||||
return self.time.now() + timedelta(seconds=int(ensureKey(KEY_EXPIRES_IN, data, CRED_OBJECT_NAME)))
|
||||
@@ -0,0 +1,2 @@
|
||||
# flake8: noqa
|
||||
from .debug_server import DebugServer
|
||||
@@ -0,0 +1,18 @@
|
||||
from backup.config import Config, Setting, Startable
|
||||
from backup.logger import getLogger
|
||||
from injector import inject, singleton
|
||||
logger = getLogger(__name__)
|
||||
|
||||
|
||||
@singleton
|
||||
class DebugServer(Startable):
|
||||
@inject
|
||||
def __init__(self, config: Config):
|
||||
self._config = config
|
||||
|
||||
async def start(self):
|
||||
if self._config.get(Setting.DEBUGGER_PORT) is not None:
|
||||
import debugpy
|
||||
port = self._config.get(Setting.DEBUGGER_PORT)
|
||||
logger.info("Starting debugger on port {}".format(port))
|
||||
debugpy.listen(("0.0.0.0", port))
|
||||
@@ -0,0 +1,216 @@
|
||||
import asyncio
|
||||
import socket
|
||||
import aioping
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from aiohttp import ClientSession, ClientTimeout
|
||||
from injector import inject, singleton
|
||||
|
||||
from backup.config import Config, Setting, VERSION, _DEFAULTS, PRIVATE
|
||||
from backup.exceptions import KnownError
|
||||
from backup.util import GlobalInfo, Resolver
|
||||
from backup.time import Time
|
||||
from backup.worker import Worker
|
||||
from backup.logger import getLogger, getHistory
|
||||
from backup.ha import HaRequests, HaSource
|
||||
from backup.model import Coordinator, DestinationPrecache
|
||||
from yarl import URL
|
||||
|
||||
logger = getLogger(__name__)
|
||||
ERROR_LOG_LENGTH = 30
|
||||
|
||||
|
||||
@singleton
|
||||
class DebugWorker(Worker):
|
||||
@inject
|
||||
def __init__(self, time: Time, info: GlobalInfo, config: Config, resolver: Resolver, session: ClientSession, ha: HaRequests, coord: Coordinator, ha_source: HaSource, precache: DestinationPrecache):
|
||||
super().__init__("Debug Worker", self.doWork, time, interval=10)
|
||||
self.time = time
|
||||
self._info = info
|
||||
self.config = config
|
||||
self.ha = ha
|
||||
self.ha_source = ha_source
|
||||
self.coord = coord
|
||||
|
||||
self.last_dns_update = None
|
||||
self.dns_info = None
|
||||
|
||||
self.last_sent_error = None
|
||||
self.last_sent_error_time = None
|
||||
self._health = None
|
||||
self.resolver = resolver
|
||||
self.session = session
|
||||
self._last_server_check = None
|
||||
self._last_server_refresh = timedelta(days=1)
|
||||
self._precache = precache
|
||||
|
||||
async def doWork(self):
|
||||
if not self.last_dns_update or self.time.now() > self.last_dns_update + timedelta(hours=12):
|
||||
await self.updateDns()
|
||||
if not self._last_server_check or self.time.now() > self._last_server_check + self._last_server_refresh:
|
||||
await self.updateHealthCheck()
|
||||
if self.config.get(Setting.SEND_ERROR_REPORTS):
|
||||
try:
|
||||
await self.maybeSendErrorReport()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Once per day, query the health endpoint of the token server to see who is up.
|
||||
# This checks for broadcast messages for all users and also finds which token
|
||||
# servers are available.
|
||||
async def updateHealthCheck(self):
|
||||
headers = {
|
||||
'client': self.config.clientIdentifier(),
|
||||
'addon_version': VERSION
|
||||
}
|
||||
self._last_server_check = self.time.now()
|
||||
for host in self.config.getTokenServers():
|
||||
url = host.with_path("/health")
|
||||
try:
|
||||
async with self.session.get(url, headers=headers, timeout=ClientTimeout(total=10)) as resp:
|
||||
resp.raise_for_status()
|
||||
self._health = await resp.json()
|
||||
self._last_server_refresh = timedelta(days=1)
|
||||
return
|
||||
except: # noqa: E722
|
||||
# ignore any error and just try a different endpoint
|
||||
pass
|
||||
|
||||
# no good token host could be found, so reset it to the default and check again sooner.
|
||||
self._last_server_refresh = timedelta(minutes=1)
|
||||
|
||||
async def maybeSendErrorReport(self):
|
||||
error = self._info._last_error
|
||||
if error is not None:
|
||||
if isinstance(error, KnownError):
|
||||
error = error.code()
|
||||
else:
|
||||
error = logger.formatException(error)
|
||||
if error != self.last_sent_error:
|
||||
self.last_sent_error = error
|
||||
if error is not None:
|
||||
self.last_sent_error_time = self.time.now()
|
||||
package = await self.buildErrorReport(error)
|
||||
else:
|
||||
package = self.buildClearReport()
|
||||
logger.info("Sending error report (see settings to disable)")
|
||||
headers = {
|
||||
'client': self.config.clientIdentifier(),
|
||||
'addon_version': VERSION
|
||||
}
|
||||
url = URL(self.config.get(Setting.AUTHORIZATION_HOST)).with_path("/logerror")
|
||||
async with self.session.post(url, headers=headers, json=package):
|
||||
pass
|
||||
|
||||
async def updateDns(self):
|
||||
self.last_dns_update = self.time.now()
|
||||
try:
|
||||
# Resolve google's addresses
|
||||
self.dns_info = await self.getPingInfo()
|
||||
self._info.setDnsInfo(self.dns_info)
|
||||
except Exception as e:
|
||||
self.dns_info = logger.formatException(e)
|
||||
|
||||
async def buildErrorReport(self, error):
|
||||
config_special = {}
|
||||
for setting in Setting:
|
||||
if self.config.get(setting) != _DEFAULTS[setting]:
|
||||
if setting in PRIVATE:
|
||||
config_special[str(setting)] = "REDACTED"
|
||||
else:
|
||||
config_special[str(setting)] = self.config.get(setting)
|
||||
report = {}
|
||||
report['config'] = config_special
|
||||
report['time'] = self.formatDate(self.time.now())
|
||||
report['start_time'] = self.formatDate(self._info._start_time)
|
||||
report['addon_version'] = VERSION
|
||||
report['failure_time'] = self.formatDate(self._info._last_failure_time)
|
||||
report['failure_count'] = self._info._failures
|
||||
report['sync_last_start'] = self.formatDate(self._info._last_sync_start)
|
||||
report['sync_count'] = self._info._syncs
|
||||
report['sync_success_count'] = self._info._successes
|
||||
report['sync_last_success'] = self.formatDate(self._info._last_sync_success)
|
||||
report['upload_count'] = self._info._uploads
|
||||
report['upload_last_size'] = self._info._last_upload_size
|
||||
report['upload_last_attempt'] = self.formatDate(self._info._last_upload)
|
||||
report['next_sync'] = self.formatDate(self.coord.nextSyncAttempt())
|
||||
report['next_backup'] = self.formatDate(self.coord.nextBackupTime())
|
||||
report['next_cache_warm'] = self.formatDate(self._precache.getNextWarmDate())
|
||||
report['time_offset'] = self._time.offset.total_seconds()
|
||||
|
||||
report['debug'] = self._info.debug
|
||||
report['version'] = VERSION
|
||||
report['error'] = error
|
||||
report['client'] = self.config.clientIdentifier()
|
||||
|
||||
if self.ha_source.isInitialized() and self.ha_source.host_info and self.ha_source.super_info and self.ha_source.ha_info:
|
||||
report["super_version"] = self.ha_source.host_info.get('supervisor', "None")
|
||||
report["hassos_version"] = self.ha_source.host_info.get('hassos', "None")
|
||||
report["docker_version"] = self.ha_source.host_info.get('docker', "None")
|
||||
report["machine"] = self.ha_source.host_info.get('machine', "None")
|
||||
report["supervisor_channel"] = self.ha_source.host_info.get('channel', "None")
|
||||
report["arch"] = self.ha_source.super_info.get('arch', "None")
|
||||
report["timezone"] = self.ha_source.super_info.get('timezone', "None")
|
||||
report["ha_version"] = self.ha_source.ha_info.get('version', "None")
|
||||
else:
|
||||
report["super_version"] = "Uninitialized"
|
||||
report["arch"] = "Uninitialized"
|
||||
report["timezone"] = "Uninitialized"
|
||||
report["ha_version"] = "Uninitialized"
|
||||
report["backups"] = self.coord.buildBackupMetrics()
|
||||
return report
|
||||
|
||||
async def buildBugReportData(self, error):
|
||||
report = await self.buildErrorReport(error)
|
||||
report['addon_logs'] = "\n".join(b for a, b in list(getHistory(0, False))[-ERROR_LOG_LENGTH:])
|
||||
try:
|
||||
report['super_logs'] = "\n".join((await self.ha.getSuperLogs()).split("\n")[-ERROR_LOG_LENGTH:])
|
||||
except Exception as e:
|
||||
report['super_logs'] = logger.formatException(e)
|
||||
try:
|
||||
report['core_logs'] = "\n".join((await self.ha.getCoreLogs()).split("\n")[-ERROR_LOG_LENGTH:])
|
||||
except Exception as e:
|
||||
report['core_logs'] = logger.formatException(e)
|
||||
return report
|
||||
|
||||
def buildClearReport(self):
|
||||
duration = self.time.now() - self.last_sent_error_time
|
||||
report = {
|
||||
'duration': str(duration)
|
||||
}
|
||||
return report
|
||||
|
||||
def formatDate(self, date: datetime):
|
||||
if date is None:
|
||||
return "Never"
|
||||
else:
|
||||
return date.isoformat()
|
||||
|
||||
async def getPingInfo(self):
|
||||
who = self.config.get(Setting.DRIVE_HOST_NAME)
|
||||
ips = await self.resolve(who)
|
||||
results = {who: {}}
|
||||
tasks = {who: {}}
|
||||
for ip in ips:
|
||||
results[who][ip] = "Unknown"
|
||||
tasks[who][ip] = asyncio.create_task(aioping.ping(ip, timeout=self.config.get(Setting.PING_TIMEOUT)))
|
||||
|
||||
# ping each server
|
||||
for server in tasks.keys():
|
||||
for ip in tasks[server].keys():
|
||||
try:
|
||||
time = await tasks[server][ip]
|
||||
results[server][ip] = f"{round(time * 1000, 0)} ms"
|
||||
except Exception as e:
|
||||
results[server][ip] = str(e)
|
||||
return results
|
||||
|
||||
async def resolve(self, who: str):
|
||||
try:
|
||||
ret = [who]
|
||||
addresses = await self.resolver.resolve(who, 443, socket.AF_INET)
|
||||
for address in addresses:
|
||||
ret.append(address['host'])
|
||||
return ret
|
||||
except Exception:
|
||||
return [who]
|
||||
@@ -0,0 +1,5 @@
|
||||
# flake8: noqa
|
||||
from .driverequests import DriveRequests, RETRY_SESSION_ATTEMPTS, UPLOAD_SESSION_EXPIRATION_DURATION, URL_START_UPLOAD, OOB_CRED_CUTOFF
|
||||
from .drivesource import DriveSource, SOURCE_GOOGLE_DRIVE
|
||||
from .folderfinder import FolderFinder
|
||||
from .authcodequery import AuthCodeQuery
|
||||
@@ -0,0 +1,107 @@
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from backup.config import Config, Setting
|
||||
from backup.time import Time
|
||||
from backup.exceptions import GoogleCredGenerateError, KnownError, LogicError, ensureKey
|
||||
from aiohttp import ClientSession
|
||||
from injector import inject
|
||||
from .driverequests import DriveRequester
|
||||
from backup.logger import getLogger
|
||||
from backup.creds import Creds
|
||||
import asyncio
|
||||
|
||||
logger = getLogger(__name__)
|
||||
SCOPE = 'https://www.googleapis.com/auth/drive.file'
|
||||
|
||||
|
||||
class AuthCodeQuery:
|
||||
@inject
|
||||
def __init__(self, config: Config, session: ClientSession, time: Time, drive: DriveRequester):
|
||||
self.session = session
|
||||
self.config = config
|
||||
self.drive = drive
|
||||
self.time = time
|
||||
self.client_id: str = None
|
||||
self.client_secret: str = None
|
||||
self.device_code: str = None
|
||||
self.verification_url: str = None
|
||||
self.user_code: str = None
|
||||
self.check_interval: timedelta = timedelta(seconds=5)
|
||||
self.expiration: datetime = time.now()
|
||||
self.last_check = time.now()
|
||||
|
||||
async def requestCredentials(self, client_id: str, client_secret: str):
|
||||
self.client_id = client_id
|
||||
self.client_secret = client_secret
|
||||
request_data = {
|
||||
'client_id': self.client_id,
|
||||
'scope': SCOPE
|
||||
}
|
||||
resp = await self.session.post(self.config.get(Setting.DRIVE_DEVICE_CODE_URL), data=request_data, timeout=30)
|
||||
if resp.status != 200:
|
||||
raise GoogleCredGenerateError(f"Google responded with error status HTTP {resp.status}. Please verify your credentials are set up correctly.")
|
||||
data = await resp.json()
|
||||
self.device_code = str(ensureKey("device_code", data, "Google's authorization request"))
|
||||
self.verification_url = str(ensureKey("verification_url", data, "Google's authorization request"))
|
||||
self.user_code = str(ensureKey("user_code", data, "Google's authorization request"))
|
||||
self.expiration = self.time.now() + timedelta(seconds=int(ensureKey("expires_in", data, "Google's authorization request")))
|
||||
self.check_interval = timedelta(seconds=int(ensureKey("interval", data, "Google's authorization request")))
|
||||
|
||||
async def waitForPermission(self) -> Creds:
|
||||
if not self.device_code:
|
||||
raise LogicError("Please call requestCredentials() first")
|
||||
error_count = 0
|
||||
data = {
|
||||
'client_id': self.client_id,
|
||||
'client_secret': self.client_secret,
|
||||
'device_code': self.device_code,
|
||||
'grant_type': 'urn:ietf:params:oauth:grant-type:device_code'
|
||||
}
|
||||
while self.expiration > self.time.now():
|
||||
start = self.time.now()
|
||||
resp = None
|
||||
try:
|
||||
resp = await self.session.post(self.config.get(Setting.DRIVE_TOKEN_URL), data=data, timeout=self.check_interval.total_seconds())
|
||||
try:
|
||||
reply = await resp.json()
|
||||
except Exception:
|
||||
reply = {}
|
||||
if resp.status == 403:
|
||||
if reply.get("error", "") == "slow_down":
|
||||
# google wants us to chill out, so do that
|
||||
await asyncio.sleep(self.check_interval.total_seconds())
|
||||
else:
|
||||
# Google says no
|
||||
logger.error(f"Getting credentials from Google failed with HTTP 403 and error: {reply.get('error', 'unspecified')}")
|
||||
raise GoogleCredGenerateError("Google refused the request to connect your account, either because you rejected it or they were set up incorrectly.")
|
||||
elif resp.status == 428:
|
||||
# Google says PEBKAC
|
||||
logger.info(f"Waiting for you to authenticate with Google at {self.verification_url}")
|
||||
elif resp.status / 100 != 2:
|
||||
# Mysterious error
|
||||
logger.error(f"Getting credentials from Google failed with HTTP {resp.status} and error: {reply.get('error', 'unspecified')}")
|
||||
raise GoogleCredGenerateError("Failed unexpectedly while trying to reach Google. See the add-on logs for details.")
|
||||
else:
|
||||
# got the token, return it
|
||||
return Creds.load(self.time, reply, id=self.client_id, secret=self.client_secret)
|
||||
except KnownError:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error("Error while trying to retrieve credentials from Google")
|
||||
logger.printException(e)
|
||||
|
||||
# Allowing 10 errors is arbitrary, but prevents us from just erroring out forever in the background
|
||||
error_count += 1
|
||||
if error_count > 10:
|
||||
raise GoogleCredGenerateError("Failed unexpectedly too many times while attempting to reach Google. See the logs for details.")
|
||||
finally:
|
||||
if resp is not None:
|
||||
resp.release()
|
||||
|
||||
# Make sure we never query more than google says we should
|
||||
remainder = self.check_interval - (self.time.now() - start)
|
||||
if remainder > timedelta(seconds=0):
|
||||
await asyncio.sleep(remainder.total_seconds())
|
||||
|
||||
logger.error("Getting credentials from Google expired, please try again")
|
||||
raise GoogleCredGenerateError("Credentials expired while waiting for you to authorize with Google")
|
||||
@@ -0,0 +1,382 @@
|
||||
import io
|
||||
import math
|
||||
import re
|
||||
from typing import Any, Dict, Optional
|
||||
from urllib.parse import urlencode
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from aiohttp import ClientSession, ClientTimeout, ClientResponse
|
||||
from aiohttp.client_exceptions import ClientResponseError, ServerTimeoutError
|
||||
from injector import inject, singleton
|
||||
|
||||
from ..util import AsyncHttpGetter
|
||||
from ..config import Config, Setting
|
||||
from ..exceptions import (GoogleCredentialsExpired,
|
||||
GoogleSessionError, LogicError,
|
||||
ProtocolError, ensureKey, KnownTransient, GoogleTimeoutError, GoogleUnexpectedError)
|
||||
from backup.util import Backoff
|
||||
from backup.file import JsonFileSaver
|
||||
from ..time import Time
|
||||
from ..logger import getLogger
|
||||
from backup.creds import Creds, Exchanger, DriveRequester
|
||||
from datetime import timezone
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
MIME_TYPE = "application/tar"
|
||||
FOLDER_MIME_TYPE = 'application/vnd.google-apps.folder'
|
||||
FOLDER_NAME = 'Home Assistant Backups'
|
||||
DRIVE_VERSION = "v3"
|
||||
DRIVE_SERVICE = "drive"
|
||||
|
||||
SELECT_FIELDS = "id,name,appProperties,size,trashed,mimeType,modifiedTime,capabilities,parents,driveId"
|
||||
THUMBNAIL_MIME_TYPE = "image/png"
|
||||
QUERY_FIELDS = "nextPageToken,files(" + SELECT_FIELDS + ")"
|
||||
CREATE_FIELDS = SELECT_FIELDS
|
||||
URL_FILES = "/drive/v3/files/"
|
||||
URL_ABOUT = "/drive/v3/about"
|
||||
URL_START_UPLOAD = "/upload/drive/v3/files/?uploadType=resumable&supportsAllDrives=true"
|
||||
PAGE_SIZE = 100
|
||||
CHUNK_SIZE = 5 * 262144
|
||||
RANGE_RE = re.compile("^bytes=0-\\d+$")
|
||||
|
||||
BASE_CHUNK_SIZE = 256 * 1024 # Google's api requires uploading chunks in multiples of 256kb
|
||||
|
||||
# During upload, chunks get sized to complete upload after 10s so we can give status updates on progress.
|
||||
CHUNK_UPLOAD_TARGET_SECONDS = 10
|
||||
|
||||
# don't attempt to resume a session with than this many times consistant failures, just in case something is broken on Google's
|
||||
# end so we don't retry the same broken session forever. Because the addon eventually backs off to doing 1 attempt/hour, this will
|
||||
# cause uploads to fail and start over after about 4 days. This gets reset every time a chunk successfully uploads.
|
||||
# God be with you if your upload takes that long.
|
||||
RETRY_SESSION_ATTEMPTS = 100
|
||||
|
||||
# Google claims that an upload session becomes invalid after 7 days. I have not verified this, but probably better to call it
|
||||
# after 6 and restart the session.
|
||||
UPLOAD_SESSION_EXPIRATION_DURATION = timedelta(days=6)
|
||||
|
||||
|
||||
RATE_LIMIT_EXCEEDED = 403
|
||||
TOO_MANY_REQUESTS = 429
|
||||
|
||||
|
||||
# Defines the retry strategy for calls made to Drive
|
||||
# max # of time to retry and call to Drive
|
||||
DRIVE_MAX_RETRIES: int = 5
|
||||
# The initial backoff for drive retries.
|
||||
DRIVE_RETRY_INITIAL_SECONDS: int = 2
|
||||
# How uch longer to wait for each Drive service call (Exponential backoff)
|
||||
DRIVE_EXPONENTIAL_BACKOFF: int = 2
|
||||
|
||||
OOB_CRED_CUTOFF = datetime(2022, 3, 16, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
@singleton
|
||||
class DriveRequests():
|
||||
@inject
|
||||
def __init__(self, config: Config, time: Time, drive: DriveRequester, session: ClientSession, exchanger: Exchanger):
|
||||
self.session = session
|
||||
self.config = config
|
||||
self.time = time
|
||||
self.drive = drive
|
||||
self.creds: Optional[Creds] = None
|
||||
self.exchanger: Exchanger = exchanger
|
||||
|
||||
# Between attempts to upload, we keep track of the info needed to resume a resumable upload.
|
||||
self.last_attempt_metadata = None
|
||||
self.last_attempt_location = None
|
||||
self.last_attempt_count = 0
|
||||
self.last_attempt_start_time = None
|
||||
self.tryLoadCredentials()
|
||||
|
||||
async def _getHeaders(self):
|
||||
return {
|
||||
"Authorization": "Bearer " + await self.getToken(),
|
||||
"Client-Identifier": self.config.clientIdentifier()
|
||||
}
|
||||
|
||||
@property
|
||||
def might_be_oob_creds(self):
|
||||
"""Attempts to determine if the user might be using custom creds affected by google's OOB cred deprecation"""
|
||||
if not self.isCustomCreds():
|
||||
return False
|
||||
if self.creds.original_expiration is None:
|
||||
# These creds must be old, so assume they're affected
|
||||
return True
|
||||
try:
|
||||
return self.creds.original_expiration < OOB_CRED_CUTOFF
|
||||
except: # noqa: E722
|
||||
# Regardless of why this happens, assume they need to check
|
||||
return True
|
||||
|
||||
def isCustomCreds(self):
|
||||
return self.creds is not None and self.creds.id != self.config.get(Setting.DEFAULT_DRIVE_CLIENT_ID)
|
||||
|
||||
def _getAuthHeaders(self):
|
||||
return {
|
||||
"Client-Identifier": self.config.clientIdentifier()
|
||||
}
|
||||
|
||||
def enabled(self):
|
||||
return self.creds is not None and self.config.get(Setting.ENABLE_DRIVE_UPLOAD)
|
||||
|
||||
def _enabledCheck(self):
|
||||
if not self.enabled():
|
||||
raise LogicError(
|
||||
"Attempt to use Google Drive before credentials are configured")
|
||||
|
||||
def tryLoadCredentials(self):
|
||||
path = self.config.get(Setting.CREDENTIALS_FILE_PATH)
|
||||
if JsonFileSaver.exists(path):
|
||||
try:
|
||||
self.creds = Creds.load(self.time, JsonFileSaver.read(path))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def saveCredentials(self, creds: Creds):
|
||||
path = self.config.get(Setting.CREDENTIALS_FILE_PATH)
|
||||
if not creds:
|
||||
if JsonFileSaver.exists(path):
|
||||
JsonFileSaver.delete(path)
|
||||
self.creds = None
|
||||
return
|
||||
JsonFileSaver.write(path, creds.serialize())
|
||||
self.tryLoadCredentials()
|
||||
|
||||
async def getToken(self, refresh=False):
|
||||
if self.creds and not self.creds.is_expired and not refresh:
|
||||
return self.creds.access_token
|
||||
|
||||
# refresh the credentials
|
||||
logger.debug("Requesting refreshed Google Drive credentials")
|
||||
self.creds = await self.exchanger.refresh(self.creds)
|
||||
return self.creds.access_token
|
||||
|
||||
async def refreshToken(self):
|
||||
await self.getToken(refresh=True)
|
||||
|
||||
async def get(self, id):
|
||||
q = {
|
||||
"fields": SELECT_FIELDS,
|
||||
"supportsAllDrives": "true"
|
||||
}
|
||||
async with await self.retryRequest("GET", URL_FILES + id + "/?" + urlencode(q)) as response:
|
||||
return await response.json()
|
||||
|
||||
async def download(self, id, size):
|
||||
ret = AsyncHttpGetter(self.config.get(Setting.DRIVE_URL) + URL_FILES + id + "/?alt=media&supportsAllDrives=true",
|
||||
await self._getHeaders(),
|
||||
self.session,
|
||||
size=size,
|
||||
timeoutFactory=GoogleTimeoutError.factory,
|
||||
otherErrorFactory=GoogleUnexpectedError.factory,
|
||||
timeout=ClientTimeout(
|
||||
sock_connect=self.config.get(Setting.DOWNLOAD_TIMEOUT_SECONDS),
|
||||
sock_read=self.config.get(Setting.DOWNLOAD_TIMEOUT_SECONDS)),
|
||||
time=self.time)
|
||||
return ret
|
||||
|
||||
async def query(self, query):
|
||||
# SOMEDAY: Add a test for page size, test server support is needed too for continuation tokens
|
||||
continuation = None
|
||||
while True:
|
||||
q = {
|
||||
"q": query,
|
||||
"fields": QUERY_FIELDS,
|
||||
"pageSize": self.config.get(Setting.GOOGLE_DRIVE_PAGE_SIZE),
|
||||
"supportsAllDrives": "true",
|
||||
"includeItemsFromAllDrives": "true",
|
||||
"corpora": "allDrives"
|
||||
}
|
||||
if continuation:
|
||||
q["pageToken"] = continuation
|
||||
async with await self.retryRequest("GET", URL_FILES + "?" + urlencode(q)) as response:
|
||||
data = await response.json()
|
||||
for item in data['files']:
|
||||
yield item
|
||||
if "nextPageToken" not in data or len(data['nextPageToken']) <= 0:
|
||||
break
|
||||
else:
|
||||
continuation = data['nextPageToken']
|
||||
|
||||
async def update(self, id, update_metadata):
|
||||
async with await self.retryRequest("PATCH", URL_FILES + id + "/?supportsAllDrives=true", json=update_metadata):
|
||||
pass
|
||||
|
||||
async def delete(self, id):
|
||||
async with await self.retryRequest("DELETE", URL_FILES + id + "/?supportsAllDrives=true"):
|
||||
pass
|
||||
|
||||
async def getAboutInfo(self):
|
||||
q = {"fields": 'storageQuota,user'}
|
||||
async with await self.retryRequest("GET", URL_ABOUT + "?" + urlencode(q)) as resp:
|
||||
return await resp.json()
|
||||
|
||||
async def create(self, stream, metadata, mime_type):
|
||||
# Upload logic is complicated. See https://developers.google.com/drive/api/v3/manage-uploads#resumable
|
||||
total_size = stream.size()
|
||||
location = None
|
||||
if metadata == self.last_attempt_metadata and self.last_attempt_location is not None and self.last_attempt_count < RETRY_SESSION_ATTEMPTS and self.time.now() < self.last_attempt_start_time + UPLOAD_SESSION_EXPIRATION_DURATION:
|
||||
logger.debug(
|
||||
"Attempting to resume a previously failed upload where we left off")
|
||||
self.last_attempt_count += 1
|
||||
# Attempt to resume from a partially completed upload.
|
||||
headers = {
|
||||
"Content-Length": "0",
|
||||
"Content-Range": "bytes */{0}".format(total_size)
|
||||
}
|
||||
try:
|
||||
async with await self.retryRequest("PUT", self.last_attempt_location, headers=headers, patch_url=False) as initial:
|
||||
if initial.status == 308:
|
||||
# We can resume the upload, check where it left off
|
||||
if 'Range' in initial.headers:
|
||||
position = int(initial.headers["Range"][len("bytes=0-"):])
|
||||
stream.position(position + 1)
|
||||
else:
|
||||
# No range header in the response means no bytes have been uploaded yet.
|
||||
stream.position(0)
|
||||
logger.debug("Resuming upload at byte {0} of {1}".format(
|
||||
stream.position(), total_size))
|
||||
location = self.last_attempt_location
|
||||
else:
|
||||
logger.debug("Drive returned status code {0}, so we'll have to start the upload over again.".format(
|
||||
initial.status))
|
||||
except ClientResponseError as e:
|
||||
if e.status == 410:
|
||||
# Drive doesn't recognize the resume token, so we'll just have to start over.
|
||||
logger.debug("Drive upload session wasn't recognized, restarting upload from the beginning.")
|
||||
location = None
|
||||
self.last_attempt_location = None
|
||||
self.last_attempt_metadata = None
|
||||
raise GoogleUnexpectedError()
|
||||
if e.status == 404:
|
||||
logger.error("Drive upload session wasn't recognized (http 404), restarting upload from the beginning.")
|
||||
location = None
|
||||
self.last_attempt_location = None
|
||||
self.last_attempt_metadata = None
|
||||
raise GoogleUnexpectedError()
|
||||
else:
|
||||
raise
|
||||
|
||||
if location is None:
|
||||
# There is no session resume, so start a new one.
|
||||
logger.debug("Starting a new upload session with Google Drive")
|
||||
headers = {
|
||||
"X-Upload-Content-Type": mime_type,
|
||||
"X-Upload-Content-Length": str(total_size),
|
||||
}
|
||||
async with await self.retryRequest("POST", URL_START_UPLOAD, headers=headers, json=metadata) as initial:
|
||||
# Google returns a url in the header "Location", which is where subsequent requests to upload
|
||||
# the backup's bytes should be sent. Logic below handles uploading the file bytes in chunks.
|
||||
location = ensureKey(
|
||||
'Location', initial.headers, "Google Drive's Upload headers")
|
||||
self.last_attempt_count = 0
|
||||
stream.position(0)
|
||||
|
||||
# Keep track of the location in case the upload fails and we want to resume where we left off.
|
||||
# "metadata" is a durable fingerprint that uniquely identifies a backup, so we can use it to identify a
|
||||
# resumable partial upload in future retrys.
|
||||
self.last_attempt_location = location
|
||||
self.last_attempt_metadata = metadata
|
||||
self.last_attempt_start_time = self.time.now()
|
||||
|
||||
# Always start with the minimum chunk size and work up from there in case the last attempt
|
||||
# failed due to connectivity errors or ... whatever.
|
||||
current_chunk_size = BASE_CHUNK_SIZE
|
||||
while True:
|
||||
start = stream.position()
|
||||
data = await stream.read(current_chunk_size)
|
||||
chunk_size = len(data.getbuffer())
|
||||
if chunk_size == 0:
|
||||
raise LogicError(
|
||||
"Backup file stream ended prematurely while uploading to Google Drive")
|
||||
headers = {
|
||||
"Content-Length": str(chunk_size),
|
||||
"Content-Range": "bytes {0}-{1}/{2}".format(start, start + chunk_size - 1, total_size)
|
||||
}
|
||||
startTime = self.time.now()
|
||||
logger.debug("Sending {0} bytes to Google Drive".format(current_chunk_size))
|
||||
try:
|
||||
async with await self.retryRequest("PUT", location, headers=headers, data=data, patch_url=False) as partial:
|
||||
# Base the next chunk size on how long it took to send the last chunk.
|
||||
current_chunk_size = self._getNextChunkSize(
|
||||
current_chunk_size, (self.time.now() - startTime).total_seconds())
|
||||
|
||||
# any time a chunk gets uploaded, reset the retry counter. This lets very flaky connections
|
||||
# complete eventually after enough retrying.
|
||||
self.last_attempt_count = 1
|
||||
yield float(start + chunk_size) / float(total_size)
|
||||
if partial.status == 200 or partial.status == 201:
|
||||
# Upload completed, return the object json
|
||||
self.last_attempt_location = None
|
||||
self.last_attempt_metadata = None
|
||||
yield await self.get((await partial.json())['id'])
|
||||
break
|
||||
elif partial.status == 308:
|
||||
# Upload partially complete, seek to the new requested position
|
||||
range_bytes = ensureKey(
|
||||
"Range", partial.headers, "Google Drive's upload response headers")
|
||||
if not RANGE_RE.match(range_bytes):
|
||||
raise ProtocolError(
|
||||
"Range", partial.headers, "Google Drive's upload response headers")
|
||||
position = int(partial.headers["Range"][len("bytes=0-"):])
|
||||
stream.position(position + 1)
|
||||
else:
|
||||
partial.raise_for_status()
|
||||
except ClientResponseError as e:
|
||||
if math.floor(e.status / 100) == 4:
|
||||
# clear the cached session location URI, since a 4XX error
|
||||
# always means the upload session is no good anymore (AFAIK)
|
||||
self.last_attempt_location = None
|
||||
self.last_attempt_metadata = None
|
||||
|
||||
if e.status == 404:
|
||||
raise GoogleSessionError()
|
||||
else:
|
||||
raise e
|
||||
|
||||
def _getNextChunkSize(self, last_chunk_size, last_chunk_seconds):
|
||||
max = BASE_CHUNK_SIZE * math.floor(self.config.get(Setting.MAXIMUM_UPLOAD_CHUNK_BYTES) / BASE_CHUNK_SIZE)
|
||||
if max < BASE_CHUNK_SIZE:
|
||||
max = BASE_CHUNK_SIZE
|
||||
if last_chunk_seconds <= 0:
|
||||
return max
|
||||
next_chunk = CHUNK_UPLOAD_TARGET_SECONDS * last_chunk_size / last_chunk_seconds
|
||||
if next_chunk >= max:
|
||||
return max
|
||||
if next_chunk < BASE_CHUNK_SIZE:
|
||||
return BASE_CHUNK_SIZE
|
||||
return math.floor(next_chunk / BASE_CHUNK_SIZE) * BASE_CHUNK_SIZE
|
||||
|
||||
async def createFolder(self, metadata):
|
||||
async with await self.retryRequest("POST", URL_FILES + "?supportsAllDrives=true", json=metadata) as resp:
|
||||
return await resp.json()
|
||||
|
||||
async def retryRequest(self, method, url, auth_headers: Optional[Dict[str, str]] = None, headers: Optional[Dict[str, str]] = None, json: Optional[Dict[str, Any]] = None, data: Any = None, cred_retry: bool = True, patch_url: bool = True) -> ClientResponse:
|
||||
backoff = Backoff(base=DRIVE_RETRY_INITIAL_SECONDS, attempts=DRIVE_MAX_RETRIES)
|
||||
if patch_url:
|
||||
url = self.config.get(Setting.DRIVE_URL) + url
|
||||
while True:
|
||||
headers_to_use = await self._getHeaders()
|
||||
if headers:
|
||||
headers_to_use.update(headers)
|
||||
if self.config.get(Setting.TRACE_REQUESTS):
|
||||
logger.trace("Making Google Drive request: " + url)
|
||||
try:
|
||||
data_to_use = data
|
||||
if isinstance(data_to_use, io.BytesIO):
|
||||
# This is a pretty low-down dirty hack, but it works and lets us reuse the byte stream.
|
||||
# aiohttp complains if you pass it a large byte object
|
||||
data_to_use = io.BytesIO(data_to_use.getbuffer())
|
||||
data_to_use.seek(0)
|
||||
return await self.drive.request(method, url, headers=headers_to_use, json=json, data=data_to_use)
|
||||
except GoogleCredentialsExpired:
|
||||
# Get fresh credentials, then retry right away.
|
||||
logger.debug("Google Drive credentials have expired. We'll retry with new ones.")
|
||||
await self.refreshToken()
|
||||
except KnownTransient as e:
|
||||
backoff.backoff(e)
|
||||
logger.error("{0}: we'll retry in {1} seconds".format(e.message(), backoff.peek()))
|
||||
await self.time.sleepAsync(backoff.peek())
|
||||
except ServerTimeoutError:
|
||||
raise GoogleTimeoutError()
|
||||
@@ -0,0 +1,282 @@
|
||||
from datetime import datetime
|
||||
from io import IOBase
|
||||
from asyncio import Event
|
||||
from typing import Dict
|
||||
|
||||
from aiohttp import ClientSession
|
||||
from aiohttp.client_exceptions import ClientResponseError
|
||||
from injector import inject, singleton
|
||||
|
||||
from ..util import AsyncHttpGetter, GlobalInfo
|
||||
from ..config import Config, Setting, CreateOptions
|
||||
from ..const import SOURCE_GOOGLE_DRIVE
|
||||
from ..exceptions import (BackupFolderInaccessible,
|
||||
ExistingBackupFolderError,
|
||||
GoogleDrivePermissionDenied, LogicError)
|
||||
from ..model.backups import (PROP_NOTE, PROP_PROTECTED, PROP_RETAINED, PROP_TYPE, PROP_VERSION)
|
||||
from ..time import Time
|
||||
from .driverequests import DriveRequests
|
||||
from .folderfinder import FolderFinder
|
||||
from .thumbnail import THUMBNAIL_IMAGE
|
||||
from ..model import BackupDestination, DriveBackup, Backup
|
||||
from ..logger import getLogger
|
||||
from ..creds.creds import Creds
|
||||
from backup.const import NECESSARY_OLD_BACKUP_NAME, NECESSARY_OLD_BACKUP_PLURAL_NAME, NECESSARY_PROP_KEY_SLUG, NECESSARY_PROP_KEY_DATE, NECESSARY_PROP_KEY_NAME
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
MIME_TYPE = "application/tar"
|
||||
THUMBNAIL_MIME_TYPE = "image/png"
|
||||
FOLDER_MIME_TYPE = 'application/vnd.google-apps.folder'
|
||||
FOLDER_NAME = 'Home Assistant Backups'
|
||||
FOLDER_CACHE_SECONDS = 30
|
||||
DRIVE_MAX_PROPERTY_LENGTH = 120
|
||||
|
||||
|
||||
@singleton
|
||||
class DriveSource(BackupDestination):
|
||||
# SOMEDAY: read backups all in one big batch request, then sort the folder and child addons from that. Would need to add test verifying the "current" backup directory is used instead of the "latest"
|
||||
@inject
|
||||
def __init__(self, config: Config, time: Time, drive_requests: DriveRequests, info: GlobalInfo, session: ClientSession, folderfinder: FolderFinder):
|
||||
super().__init__()
|
||||
self.session = session
|
||||
self.config = config
|
||||
self.drivebackend: DriveRequests = drive_requests
|
||||
self.time = time
|
||||
self.folder_finder = folderfinder
|
||||
self._info = info
|
||||
self._uploadedAtLeastOneChunk = False
|
||||
self._drive_info = None
|
||||
self._cred_trigger = Event()
|
||||
|
||||
def saveCreds(self, creds: Creds) -> None:
|
||||
logger.info("Saving new Google Drive credentials")
|
||||
self.drivebackend.saveCredentials(creds)
|
||||
self.trigger()
|
||||
self._cred_trigger.set()
|
||||
|
||||
async def debug_wait_for_credentials(self):
|
||||
await self._cred_trigger.wait()
|
||||
self._cred_trigger.clear()
|
||||
|
||||
def isCustomCreds(self):
|
||||
return self.drivebackend.isCustomCreds()
|
||||
|
||||
@property
|
||||
def might_be_oob_creds(self) -> bool:
|
||||
return self.drivebackend.might_be_oob_creds
|
||||
|
||||
def name(self) -> str:
|
||||
return SOURCE_GOOGLE_DRIVE
|
||||
|
||||
def title(self) -> str:
|
||||
return "Google Drive"
|
||||
|
||||
def maxCount(self) -> None:
|
||||
return self.config.get(Setting.MAX_BACKUPS_IN_GOOGLE_DRIVE)
|
||||
|
||||
def upload(self) -> bool:
|
||||
return self.config.get(Setting.ENABLE_DRIVE_UPLOAD)
|
||||
|
||||
def enabled(self) -> bool:
|
||||
return self.drivebackend.enabled()
|
||||
|
||||
def needsConfiguration(self) -> bool:
|
||||
if not self.config.get(Setting.ENABLE_DRIVE_UPLOAD):
|
||||
return False
|
||||
return super().needsConfiguration()
|
||||
|
||||
def freeSpace(self):
|
||||
if self._drive_info and self._drive_info.get("storageQuota") is not None and not self.folder_finder.currentIsSharedDrive():
|
||||
info = self._drive_info.get("storageQuota")
|
||||
if 'limit' in info and 'usage' in info:
|
||||
return int(info.get("limit")) - int((info.get("usage")))
|
||||
return super().freeSpace()
|
||||
|
||||
async def create(self, options: CreateOptions) -> DriveBackup:
|
||||
raise LogicError("Backups can't be created in Drive")
|
||||
|
||||
def checkBeforeChanges(self):
|
||||
existing = self.folder_finder.getExisting()
|
||||
if existing:
|
||||
raise ExistingBackupFolderError(
|
||||
existing.get('id'), existing.get('name'))
|
||||
|
||||
def icon(self) -> str:
|
||||
return "google-drive"
|
||||
|
||||
def isWorking(self):
|
||||
return self._uploadedAtLeastOneChunk
|
||||
|
||||
def detail(self):
|
||||
if self._drive_info and 'user' in self._drive_info and 'emailAddress' in self._drive_info['user']:
|
||||
return f'{self._drive_info["user"]["emailAddress"]}'
|
||||
else:
|
||||
return super().detail()
|
||||
|
||||
async def get(self, allow_retry=True) -> Dict[str, DriveBackup]:
|
||||
parent = await self.getFolderId()
|
||||
try:
|
||||
self._drive_info = await self.drivebackend.getAboutInfo()
|
||||
except Exception as e:
|
||||
# This is just used to get the remaining space in Drive, which is a
|
||||
# nice to have. Just log the error to debug if we can't get it
|
||||
logger.debug("Unable to retrieve Google Drive storage info: " + str(e))
|
||||
backups: Dict[str, DriveBackup] = {}
|
||||
try:
|
||||
async for child in self.drivebackend.query("'{}' in parents".format(parent)):
|
||||
properties = child.get('appProperties')
|
||||
if properties and NECESSARY_PROP_KEY_DATE in properties and NECESSARY_PROP_KEY_SLUG in properties and not child['trashed']:
|
||||
backup = DriveBackup(child)
|
||||
backups[backup.slug()] = backup
|
||||
except ClientResponseError as e:
|
||||
if e.status == 404:
|
||||
# IIUC, 404 on create can only mean that the parent id isn't valid anymore.
|
||||
if not self.config.get(Setting.SPECIFY_BACKUP_FOLDER) and allow_retry:
|
||||
self.folder_finder.deCache()
|
||||
await self.folder_finder.create()
|
||||
return await self.get(False)
|
||||
raise BackupFolderInaccessible(parent)
|
||||
raise e
|
||||
except GoogleDrivePermissionDenied:
|
||||
# This should always mean we lost permission on the backup folder, but at least it still exists.
|
||||
if not self.config.get(Setting.SPECIFY_BACKUP_FOLDER) and allow_retry:
|
||||
self.folder_finder.deCache()
|
||||
await self.folder_finder.create()
|
||||
return await self.get(False)
|
||||
raise BackupFolderInaccessible(parent)
|
||||
return backups
|
||||
|
||||
async def delete(self, backup: Backup):
|
||||
item = self._validateBackup(backup)
|
||||
if item.canDeleteDirectly():
|
||||
logger.info("Deleting '{}' From Google Drive".format(item.name()))
|
||||
await self.drivebackend.delete(item.id())
|
||||
else:
|
||||
logger.info("Trashing '{}' in Google Drive".format(item.name()))
|
||||
await self.drivebackend.update(item.id(), {"trashed": True})
|
||||
backup.removeSource(self.name())
|
||||
|
||||
async def save(self, backup: Backup, source: AsyncHttpGetter) -> DriveBackup:
|
||||
retain = backup.getOptions() and backup.getOptions().retain_sources.get(self.name(), False)
|
||||
parent_id = await self.getFolderId()
|
||||
if backup.note() is not None:
|
||||
desc = backup.note()
|
||||
else:
|
||||
desc = 'A Home Assistant backup file uploaded by Home Assistant Google Drive Backup'
|
||||
file_metadata = {
|
||||
'name': str(backup.name()) + ".tar",
|
||||
'parents': [parent_id],
|
||||
'description': desc,
|
||||
'appProperties': {
|
||||
NECESSARY_PROP_KEY_SLUG: backup.slug(),
|
||||
NECESSARY_PROP_KEY_DATE: str(backup.date()),
|
||||
PROP_TYPE: str(backup.backupType()),
|
||||
PROP_VERSION: str(backup.version()),
|
||||
PROP_PROTECTED: str(backup.protected()),
|
||||
PROP_RETAINED: str(retain),
|
||||
},
|
||||
'contentHints': {
|
||||
'indexableText': 'Home Assistant hassio ' + NECESSARY_OLD_BACKUP_NAME + ' ' + NECESSARY_OLD_BACKUP_PLURAL_NAME + ' backup backups home assistant ' + desc,
|
||||
'thumbnail': {
|
||||
'image': THUMBNAIL_IMAGE,
|
||||
'mimeType': THUMBNAIL_MIME_TYPE
|
||||
}
|
||||
},
|
||||
'createdTime': self._timeToRfc3339String(backup.date()),
|
||||
'modifiedTime': self._timeToRfc3339String(backup.date())
|
||||
}
|
||||
|
||||
if backup.note() is not None:
|
||||
file_metadata['appProperties'][PROP_NOTE] = self.truncateAppProperty(PROP_NOTE, backup.note())
|
||||
file_metadata['appProperties'][NECESSARY_PROP_KEY_NAME] = self.truncateAppProperty(NECESSARY_PROP_KEY_NAME, str(backup.name()))
|
||||
|
||||
async with source:
|
||||
try:
|
||||
logger.info("Uploading '{}' to Google Drive".format(
|
||||
backup.name()))
|
||||
size = source.size()
|
||||
self._info.upload(size)
|
||||
backup.overrideStatus("Uploading {0}%", source)
|
||||
backup.setUploadSource(self.title(), source)
|
||||
async for progress in self.drivebackend.create(source, file_metadata, MIME_TYPE):
|
||||
self._uploadedAtLeastOneChunk = True
|
||||
if isinstance(progress, float):
|
||||
logger.debug("Uploading {1} {0:.2f}%".format(
|
||||
progress * 100, backup.name()))
|
||||
else:
|
||||
return DriveBackup(progress)
|
||||
raise LogicError(
|
||||
"Google Drive backup upload didn't return a completed item before exiting")
|
||||
except ClientResponseError as e:
|
||||
if e.status == 404:
|
||||
# IIUC, 404 on create can only mean that the parent id isn't valid anymore.
|
||||
raise BackupFolderInaccessible(parent_id)
|
||||
raise e
|
||||
except GoogleDrivePermissionDenied:
|
||||
# This should always mean we lost permission on the backup folder, since we could have only just
|
||||
# created the backup item on this request.
|
||||
raise BackupFolderInaccessible(parent_id)
|
||||
finally:
|
||||
backup.clearUploadSource()
|
||||
self._uploadedAtLeastOneChunk = False
|
||||
backup.clearStatus()
|
||||
|
||||
def truncateAppProperty(self, key: str, value: str):
|
||||
# Annoylingly, Drive properties can be a maximum of 124 bytes, in len(key + value) UTF8 encoded.
|
||||
# https://developers.google.com/drive/api/guides/properties
|
||||
# Is the extra indexing REALLY that expensive? Thats like some 1990's mainframe limitation.
|
||||
# Make sure we stay well under that limit
|
||||
if value is None:
|
||||
return value
|
||||
permitted = ""
|
||||
current = 0
|
||||
while current < len(value) and len(str(key + permitted + value[current]).encode('utf-8')) < DRIVE_MAX_PROPERTY_LENGTH:
|
||||
permitted += value[current]
|
||||
current += 1
|
||||
return permitted
|
||||
|
||||
async def read(self, backup: Backup) -> IOBase:
|
||||
item = self._validateBackup(backup)
|
||||
return await self.drivebackend.download(item.id(), item.size())
|
||||
|
||||
async def retain(self, backup: Backup, retain: bool) -> None:
|
||||
item = self._validateBackup(backup)
|
||||
if item.retained() == retain:
|
||||
return
|
||||
file_metadata: Dict[str, str] = {
|
||||
'appProperties': {
|
||||
PROP_RETAINED: str(retain),
|
||||
},
|
||||
}
|
||||
await self.drivebackend.update(item.id(), file_metadata)
|
||||
item.setRetained(retain)
|
||||
|
||||
async def note(self, backup, note: str) -> None:
|
||||
item = self._validateBackup(backup)
|
||||
truncated = self.truncateAppProperty(PROP_NOTE, note)
|
||||
file_metadata: Dict[str, str] = {
|
||||
'appProperties': {
|
||||
PROP_NOTE: truncated,
|
||||
},
|
||||
'description': note,
|
||||
}
|
||||
logger.debug(f"Adding a note to drive backup '{item.name()}'")
|
||||
await self.drivebackend.update(item.id(), file_metadata)
|
||||
item.setNote(truncated)
|
||||
|
||||
async def getFolderId(self):
|
||||
return await self.folder_finder.get()
|
||||
|
||||
def _validateBackup(self, backup: Backup) -> DriveBackup:
|
||||
drive_item: DriveBackup = backup.getSource(self.name())
|
||||
if not drive_item:
|
||||
raise LogicError(
|
||||
"Requested to do something with a backup from Google Drive, but the backup has no Google Drive source")
|
||||
return drive_item
|
||||
|
||||
def _timeToRfc3339String(self, time: datetime) -> str:
|
||||
return time.strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
async def _get(self, id):
|
||||
return await self.drivebackend.get(id)
|
||||
@@ -0,0 +1,204 @@
|
||||
from datetime import timedelta
|
||||
from typing import Any, Dict
|
||||
from backup.file import File
|
||||
from aiohttp.client_exceptions import ClientResponseError
|
||||
from injector import inject, singleton
|
||||
|
||||
from ..config import Config, Setting
|
||||
from ..exceptions import (BackupFolderInaccessible, BackupFolderMissingError,
|
||||
GoogleDrivePermissionDenied, LogInToGoogleDriveError)
|
||||
from ..time import Time
|
||||
from .driverequests import DriveRequests
|
||||
from ..logger import getLogger
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
FOLDER_MIME_TYPE = 'application/vnd.google-apps.folder'
|
||||
FOLDER_NAME = 'Home Assistant Backups'
|
||||
FOLDER_CACHE_SECONDS = 60 * 31 # 31 minutes
|
||||
|
||||
|
||||
@singleton
|
||||
class FolderFinder():
|
||||
@inject
|
||||
def __init__(self, config: Config, time: Time, drive_requests: DriveRequests):
|
||||
self.config = config
|
||||
self.drivebackend: DriveRequests = drive_requests
|
||||
self.time = time
|
||||
|
||||
# The cached folder id
|
||||
self._folderId = None
|
||||
|
||||
# When the fodler id was last cached
|
||||
self._folder_queryied_last = None
|
||||
|
||||
# These get set when an existing folder is found and should cause the UI to
|
||||
# prompt for what to do about it.
|
||||
self._existing_folder = None
|
||||
self._use_existing = None
|
||||
self._folder_details = None
|
||||
|
||||
def resolveExisting(self, val):
|
||||
if self._existing_folder:
|
||||
self._use_existing = val
|
||||
else:
|
||||
self._use_existing = None
|
||||
|
||||
def _isSharedDrive(self, folder):
|
||||
driveId = folder.get("driveId", None)
|
||||
return driveId and len(driveId) > 0
|
||||
|
||||
def currentIsSharedDrive(self):
|
||||
return self._folder_details and self._isSharedDrive(self._folder_details)
|
||||
|
||||
async def get(self):
|
||||
if self._existing_folder and self._use_existing is not None:
|
||||
if self._use_existing:
|
||||
await self.save(self._existing_folder)
|
||||
else:
|
||||
await self.create()
|
||||
self._use_existing = None
|
||||
if not self._folder_queryied_last or self._folder_queryied_last + timedelta(seconds=FOLDER_CACHE_SECONDS) < self.time.now():
|
||||
try:
|
||||
self._folderId = await self._readFolderId()
|
||||
except (BackupFolderMissingError, BackupFolderInaccessible):
|
||||
if not self.config.get(Setting.SPECIFY_BACKUP_FOLDER):
|
||||
# Search for a folder, they may have created one before
|
||||
self._existing_folder = await self._search()
|
||||
if self._existing_folder:
|
||||
self._folderId = self._existing_folder.get('id')
|
||||
else:
|
||||
# Create folder, since no other folder is available
|
||||
await self.create()
|
||||
else:
|
||||
raise
|
||||
self._folder_queryied_last = self.time.now()
|
||||
return self._folderId
|
||||
|
||||
def getExisting(self):
|
||||
return self._existing_folder
|
||||
|
||||
async def save(self, folder: Any) -> str:
|
||||
if not isinstance(folder, str):
|
||||
self._folder_details = folder
|
||||
folder = folder.get('id')
|
||||
else:
|
||||
self._folder_details = None
|
||||
logger.info("Saving backup folder: " + folder)
|
||||
File.write(self.config.get(Setting.FOLDER_FILE_PATH), folder)
|
||||
self._folderId = folder
|
||||
self._folder_queryied_last = self.time.now()
|
||||
self._existing_folder = None
|
||||
|
||||
def reset(self):
|
||||
if File.exists(self.config.get(Setting.FOLDER_FILE_PATH)):
|
||||
File.delete(self.config.get(Setting.FOLDER_FILE_PATH))
|
||||
self._folderId = None
|
||||
self._folder_queryied_last = None
|
||||
self._existing_folder = None
|
||||
|
||||
def getCachedFolder(self):
|
||||
return self._folderId
|
||||
|
||||
def deCache(self):
|
||||
self._folderId = None
|
||||
self._folder_queryied_last = None
|
||||
|
||||
async def _readFolderId(self) -> str:
|
||||
# First, check if we cached the drive folder
|
||||
if not File.exists(self.config.get(Setting.FOLDER_FILE_PATH)):
|
||||
raise BackupFolderMissingError()
|
||||
else:
|
||||
folder_id: str = File.read(self.config.get(Setting.FOLDER_FILE_PATH)).strip()
|
||||
if await self._verify(folder_id):
|
||||
return folder_id
|
||||
else:
|
||||
raise BackupFolderInaccessible(folder_id)
|
||||
|
||||
async def _search(self) -> str:
|
||||
folders = []
|
||||
|
||||
try:
|
||||
async for child in self.drivebackend.query("mimeType='" + FOLDER_MIME_TYPE + "'"):
|
||||
if self._isValidFolder(child):
|
||||
folders.append(child)
|
||||
except ClientResponseError as e:
|
||||
# 404 means the folder doesn't exist (maybe it got moved?)
|
||||
if e.status == 404:
|
||||
"Make Error"
|
||||
raise LogInToGoogleDriveError()
|
||||
else:
|
||||
raise e
|
||||
|
||||
if len(folders) == 0:
|
||||
return None
|
||||
|
||||
folders.sort(key=lambda c: Time.parse(c.get("modifiedTime")))
|
||||
# Found a folder, which means we're probably using the add-on from a
|
||||
# previous (or duplicate) installation. Record and return the id but don't
|
||||
# persist it until the user chooses to do so.
|
||||
folder = folders[len(folders) - 1]
|
||||
logger.info("Found " + folder.get('name'))
|
||||
return folder
|
||||
|
||||
async def _verify(self, id):
|
||||
if self.drivebackend.isCustomCreds():
|
||||
# If the user is using custom creds and specifying the backup folder, then chances are the
|
||||
# app doesn't have permission to access the parent folder directly. Ironically, we can still
|
||||
# query for children and add/remove backups. Not a huge deal, just
|
||||
# means we can't verify the folder still exists, isn't trashed, etc. Just let it be valid
|
||||
# and handle potential errors elsewhere.
|
||||
return True
|
||||
# Query drive for the folder to make sure it still exists and we have the right permission on it.
|
||||
try:
|
||||
folder = await self.drivebackend.get(id)
|
||||
if not self._isValidFolder(folder):
|
||||
logger.info("Provided backup folder {0} is invalid".format(id))
|
||||
return False
|
||||
self._folder_details = folder
|
||||
return True
|
||||
except ClientResponseError as e:
|
||||
if e.status == 404:
|
||||
# 404 means the folder doesn't exist (maybe it got moved?) but can also mean that we
|
||||
# just don't have permission to see the folder. Often we can still upload into it, so just
|
||||
# let it pass without further verification and let other error handling (on upload) identify problems.
|
||||
return True
|
||||
else:
|
||||
raise e
|
||||
except GoogleDrivePermissionDenied:
|
||||
# Lost permission on the backup folder
|
||||
return False
|
||||
|
||||
def _isValidFolder(self, folder) -> bool:
|
||||
try:
|
||||
caps = folder.get('capabilities')
|
||||
if folder.get('trashed'):
|
||||
return False
|
||||
elif not caps['canAddChildren']:
|
||||
return False
|
||||
elif not caps['canListChildren']:
|
||||
return False
|
||||
elif not caps.get('canDeleteChildren', False) and not caps.get('canRemoveChildren', False):
|
||||
if self._isSharedDrive(folder) and caps.get("canTrashChildren", False):
|
||||
# Allow folders in shared drives if you can still trash items inside it.
|
||||
return True
|
||||
return False
|
||||
elif folder.get("mimeType") != FOLDER_MIME_TYPE:
|
||||
return False
|
||||
except Exception:
|
||||
return False
|
||||
return True
|
||||
|
||||
async def create(self) -> str:
|
||||
logger.info('Creating folder "{}" in "My Drive"'.format(FOLDER_NAME))
|
||||
file_metadata: Dict[str, str] = {
|
||||
'name': FOLDER_NAME,
|
||||
'mimeType': FOLDER_MIME_TYPE,
|
||||
'appProperties': {
|
||||
"backup_folder": "true",
|
||||
},
|
||||
}
|
||||
folder = await self.drivebackend.createFolder(file_metadata)
|
||||
self._folder_details = folder
|
||||
await self.save(folder)
|
||||
return folder.get('id')
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,2 @@
|
||||
# flake8: noqa
|
||||
from .exceptions import UnknownNetworkStorageError, InactiveNetworkStorageError, GoogleCredGenerateError, SupervisorUnexpectedError, SupervisorTimeoutError, GoogleUnexpectedError, SupervisorFileSystemError, SupervisorPermissionError, LogInToGoogleDriveError, KnownTransient, GoogleInternalError, GoogleRateLimitError, CredRefreshGoogleError, CredRefreshMyError, BackupFolderInaccessible, BackupFolderMissingError, DeleteMutlipleBackupsError, DriveQuotaExceeded, ensureKey, ExistingBackupFolderError, UserCancelledError, UploadFailed, SupervisorConnectionError, BackupPasswordKeyInvalid, BackupInProgress, SimulatedError, ProtocolError, PleaseWait, NotUploadable, NoBackup, LowSpaceError, LogicError, KnownError, InvalidConfigurationValue, HomeAssistantDeleteError, GoogleTimeoutError, GoogleSessionError, GoogleInternalError, GoogleDrivePermissionDenied, GoogleDnsFailure, GoogleCredentialsExpired, GoogleCantConnect, ExistingBackupFolderError
|
||||
+479
@@ -0,0 +1,479 @@
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
from ..const import (DRIVE_FOLDER_URL_FORMAT, ERROR_BACKUP_FOLDER_INACCESSIBLE,
|
||||
ERROR_BACKUP_FOLDER_MISSING, ERROR_BAD_PASSWORD_KEY,
|
||||
ERROR_CREDS_EXPIRED, ERROR_DRIVE_FULL,
|
||||
ERROR_EXISTING_FOLDER, ERROR_GOOGLE_CONNECT, ERROR_GOOGLE_CRED_PROCESS,
|
||||
ERROR_GOOGLE_DNS, ERROR_GOOGLE_INTERNAL,
|
||||
ERROR_GOOGLE_SESSION, ERROR_GOOGLE_TIMEOUT,
|
||||
ERROR_HA_DELETE_ERROR, ERROR_INVALID_CONFIG, ERROR_LOGIC,
|
||||
ERROR_LOW_SPACE, ERROR_MULTIPLE_DELETES, ERROR_NO_BACKUP,
|
||||
ERROR_NOT_UPLOADABLE, ERROR_PLEASE_WAIT, ERROR_PROTOCOL,
|
||||
ERROR_BACKUP_IN_PROGRESS, ERROR_UPLOAD_FAILED, LOG_IN_TO_DRIVE,
|
||||
SUPERVISOR_PERMISSION, ERROR_GOOGLE_UNEXPECTED, ERROR_SUPERVISOR_TIMEOUT, ERROR_SUPERVISOR_UNEXPECTED, ERROR_SUPERVISOR_FILE_SYSTEM,
|
||||
UNKONWN_NETWORK_STORAGE, INACTIVE_NETWORK_STORAGE)
|
||||
|
||||
|
||||
def ensureKey(key, target, name):
|
||||
if key not in target:
|
||||
raise ProtocolError(key, name, target)
|
||||
return target[key]
|
||||
|
||||
|
||||
class KnownError(Exception, ABC):
|
||||
@abstractmethod
|
||||
def message(self) -> str:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def code(self) -> str:
|
||||
pass
|
||||
|
||||
def httpStatus(self) -> int:
|
||||
return 500
|
||||
|
||||
def data(self):
|
||||
return {}
|
||||
|
||||
def retrySoon(self):
|
||||
return True
|
||||
|
||||
|
||||
class KnownTransient(KnownError):
|
||||
pass
|
||||
|
||||
|
||||
class SimulatedError(KnownError):
|
||||
def __init__(self, code=None):
|
||||
self._code = code
|
||||
|
||||
def code(self):
|
||||
return self._code
|
||||
|
||||
def message(self):
|
||||
return "Gave code " + str(self._code)
|
||||
|
||||
|
||||
class LogicError(KnownError):
|
||||
def __init__(self, message=None):
|
||||
self._message = message
|
||||
|
||||
def message(self):
|
||||
return self._message
|
||||
|
||||
def code(self):
|
||||
return ERROR_LOGIC
|
||||
|
||||
|
||||
class ProtocolError(KnownError):
|
||||
def __init__(self, parameter=None, object_name=None, debug_object=None):
|
||||
self._parameter = parameter
|
||||
self._object_name = object_name
|
||||
self._debug_object = debug_object
|
||||
|
||||
def message(self):
|
||||
if self._object_name:
|
||||
return "Required key '{0}' was missing from {1}".format(self._parameter, self._object_name)
|
||||
else:
|
||||
return self._parameter
|
||||
|
||||
def code(self):
|
||||
return ERROR_PROTOCOL
|
||||
|
||||
|
||||
class BackupInProgress(KnownError):
|
||||
def message(self):
|
||||
return "A backup is already in progress"
|
||||
|
||||
def code(self):
|
||||
return ERROR_BACKUP_IN_PROGRESS
|
||||
|
||||
|
||||
class BackupPasswordKeyInvalid(KnownError):
|
||||
def message(self):
|
||||
return "Couldn't find your backup password in your secrets file. Please check your settings."
|
||||
|
||||
def code(self):
|
||||
return ERROR_BAD_PASSWORD_KEY
|
||||
|
||||
def retrySoon(self):
|
||||
return False
|
||||
|
||||
|
||||
class UploadFailed(KnownError):
|
||||
def message(self):
|
||||
return "Backup upload failed. Please check the supervisor logs for details."
|
||||
|
||||
def code(self):
|
||||
return ERROR_UPLOAD_FAILED
|
||||
|
||||
|
||||
class GoogleCredentialsExpired(KnownError):
|
||||
def message(self):
|
||||
return "Your Google Drive credentials have expired. Please reauthorize with Google Drive through the Web UI."
|
||||
|
||||
def code(self):
|
||||
return ERROR_CREDS_EXPIRED
|
||||
|
||||
def retrySoon(self):
|
||||
return False
|
||||
|
||||
|
||||
class NoBackup(KnownError):
|
||||
def message(self):
|
||||
return "The backup doesn't exist anymore"
|
||||
|
||||
def code(self):
|
||||
return ERROR_NO_BACKUP
|
||||
|
||||
|
||||
class NotUploadable(KnownError):
|
||||
def message(self):
|
||||
return "This backup can't be uploaded to Home Assistant yet"
|
||||
|
||||
def code(self):
|
||||
return ERROR_NOT_UPLOADABLE
|
||||
|
||||
|
||||
class PleaseWait(KnownError):
|
||||
def message(self):
|
||||
return "Please wait until the sync is finished."
|
||||
|
||||
def code(self):
|
||||
return ERROR_PLEASE_WAIT
|
||||
|
||||
|
||||
class InvalidConfigurationValue(KnownError):
|
||||
def __init__(self, key=None, current=None):
|
||||
self.key = key
|
||||
self.current = current
|
||||
|
||||
def message(self):
|
||||
return "'{0}' isn't a valid value for {1}".format(str(self.current), str(self.key))
|
||||
|
||||
def code(self):
|
||||
return ERROR_INVALID_CONFIG
|
||||
|
||||
|
||||
# UI Handler Done and updated
|
||||
|
||||
class DeleteMutlipleBackupsError(KnownError):
|
||||
def __init__(self, delete_sources=None):
|
||||
self.delete_sources = delete_sources
|
||||
|
||||
def message(self):
|
||||
return "The add-on has been configured to delete more than one older backups. Please confirm this by visiting the add-on's web UI or by setting the config option 'confirm_multiple_deletes'=false in your add-on configuration."
|
||||
|
||||
def code(self):
|
||||
return ERROR_MULTIPLE_DELETES
|
||||
|
||||
def data(self):
|
||||
return self.delete_sources
|
||||
|
||||
def retrySoon(self):
|
||||
return False
|
||||
|
||||
|
||||
class DriveQuotaExceeded(KnownError):
|
||||
def message(self):
|
||||
return "Google Drive is out of space"
|
||||
|
||||
def code(self):
|
||||
return ERROR_DRIVE_FULL
|
||||
|
||||
def retrySoon(self):
|
||||
return False
|
||||
|
||||
|
||||
class GoogleDnsFailure(KnownError):
|
||||
def message(self):
|
||||
return "Unable to resolve host www.googleapis.com"
|
||||
|
||||
def code(self):
|
||||
return ERROR_GOOGLE_DNS
|
||||
|
||||
|
||||
class GoogleCantConnect(KnownError):
|
||||
def message(self):
|
||||
return "Unable to connect to www.googleapis.com"
|
||||
|
||||
def code(self):
|
||||
return ERROR_GOOGLE_CONNECT
|
||||
|
||||
|
||||
class GoogleInternalError(KnownTransient):
|
||||
def message(self):
|
||||
return "Google Drive returned an internal error (HTTP: 5XX)"
|
||||
|
||||
def code(self):
|
||||
return ERROR_GOOGLE_INTERNAL
|
||||
|
||||
|
||||
class GoogleTimeoutError(KnownError):
|
||||
def message(self):
|
||||
return "Timed out while trying to reach Google Drive"
|
||||
|
||||
def code(self):
|
||||
return ERROR_GOOGLE_TIMEOUT
|
||||
|
||||
@classmethod
|
||||
def factory(cls):
|
||||
return GoogleTimeoutError()
|
||||
|
||||
|
||||
class GoogleRateLimitError(KnownTransient):
|
||||
def message(self):
|
||||
return "The addon has made too many requests to Google Drive, and will back off"
|
||||
|
||||
def code(self):
|
||||
return "google_rate_limit"
|
||||
|
||||
|
||||
class GoogleSessionError(KnownError):
|
||||
def message(self):
|
||||
return "Upload session with Google Drive expired. The upload could not complete."
|
||||
|
||||
def code(self):
|
||||
return ERROR_GOOGLE_SESSION
|
||||
|
||||
|
||||
class HomeAssistantDeleteError(KnownError):
|
||||
def message(self):
|
||||
return "Home Assistant refused to delete the backup."
|
||||
|
||||
def code(self):
|
||||
return ERROR_HA_DELETE_ERROR
|
||||
|
||||
|
||||
class ExistingBackupFolderError(KnownError):
|
||||
def __init__(self, existing_id: str = None, existing_name: str = None):
|
||||
self.existing_id = existing_id
|
||||
self.existing_name = existing_name
|
||||
|
||||
def message(self):
|
||||
return "A backup folder already exists. Please visit the add-on Web UI to select where to backup."
|
||||
|
||||
def code(self):
|
||||
return ERROR_EXISTING_FOLDER
|
||||
|
||||
def data(self):
|
||||
return {
|
||||
"existing_url#href": DRIVE_FOLDER_URL_FORMAT.format(self.existing_id),
|
||||
"existing_name": self.existing_name
|
||||
}
|
||||
|
||||
def retrySoon(self):
|
||||
return False
|
||||
|
||||
|
||||
class BackupFolderMissingError(KnownError):
|
||||
def message(self):
|
||||
return "Please visit the add-on Web UI to select where to backup."
|
||||
|
||||
def code(self):
|
||||
return ERROR_BACKUP_FOLDER_MISSING
|
||||
|
||||
def retrySoon(self):
|
||||
return False
|
||||
|
||||
|
||||
class BackupFolderInaccessible(KnownError):
|
||||
def __init__(self, existing_id: str = None):
|
||||
self.existing_id = existing_id
|
||||
|
||||
def message(self):
|
||||
return "The choosen backup folder has become inaccessible. Please visit the addon web UI to select a backup folder."
|
||||
|
||||
def data(self):
|
||||
return {
|
||||
"existing_url#href": DRIVE_FOLDER_URL_FORMAT.format(self.existing_id)
|
||||
}
|
||||
|
||||
def code(self):
|
||||
return ERROR_BACKUP_FOLDER_INACCESSIBLE
|
||||
|
||||
|
||||
class GoogleDrivePermissionDenied(KnownError):
|
||||
def message(self):
|
||||
return "Google Drive denied the request due to permissions."
|
||||
|
||||
def code(self):
|
||||
return "google_drive_permissions"
|
||||
|
||||
|
||||
class LowSpaceError(KnownError):
|
||||
def __init__(self, pct_used=None, space_remaining=None):
|
||||
self.pct_used = pct_used
|
||||
self.space_remaining = space_remaining
|
||||
|
||||
def message(self):
|
||||
return "Your backup folder is low on disk space. Backups can't be created until space is available."
|
||||
|
||||
def code(self):
|
||||
return ERROR_LOW_SPACE
|
||||
|
||||
def data(self):
|
||||
return {
|
||||
"pct_used": self.pct_used,
|
||||
"space_remaining": self.space_remaining
|
||||
}
|
||||
|
||||
|
||||
class SupervisorConnectionError(KnownError):
|
||||
def message(self):
|
||||
return "The addon couldn't connect to the supervisor. Backups can't continue until the supervisor is responding."
|
||||
|
||||
def code(self):
|
||||
return "supervisor_connection"
|
||||
|
||||
|
||||
class UserCancelledError(KnownError):
|
||||
def message(self):
|
||||
return "Sync was cancelled by you"
|
||||
|
||||
def code(self):
|
||||
return "cancelled"
|
||||
|
||||
def retrySoon(self):
|
||||
return False
|
||||
|
||||
|
||||
class CredRefreshGoogleError(KnownError):
|
||||
def __init__(self, from_google=None):
|
||||
self.from_google = from_google
|
||||
|
||||
def message(self):
|
||||
return "Couldn't refresh your credentials with Google because: '{}'".format(self.from_google)
|
||||
|
||||
def code(self):
|
||||
return "token_refresh_google_error"
|
||||
|
||||
def data(self):
|
||||
return {
|
||||
"from_google": self.from_google
|
||||
}
|
||||
|
||||
|
||||
class CredRefreshMyError(KnownError):
|
||||
def __init__(self, reason: str = None):
|
||||
self.reason = reason
|
||||
|
||||
def message(self):
|
||||
return "Couldn't refresh Google Drive credentials because: {}".format(self.reason)
|
||||
|
||||
def code(self):
|
||||
return "token_refresh_my_error"
|
||||
|
||||
def data(self):
|
||||
return {
|
||||
"reason": self.reason
|
||||
}
|
||||
|
||||
|
||||
class LogInToGoogleDriveError(KnownError):
|
||||
def message(self):
|
||||
return "Please visit drive.google.com to activate your Google Drive account."
|
||||
|
||||
def code(self):
|
||||
return LOG_IN_TO_DRIVE
|
||||
|
||||
def retrySoon(self):
|
||||
return False
|
||||
|
||||
|
||||
class SupervisorPermissionError(KnownError):
|
||||
def message(self):
|
||||
return "The supervisor is rejecting requests from the addon. Please visit the web-UI for guidance"
|
||||
|
||||
def code(self):
|
||||
return SUPERVISOR_PERMISSION
|
||||
|
||||
def retrySoon(self):
|
||||
return True
|
||||
|
||||
|
||||
class GoogleUnexpectedError(KnownError):
|
||||
def message(self):
|
||||
return "Google gave an unexpected response"
|
||||
|
||||
def code(self):
|
||||
return ERROR_GOOGLE_UNEXPECTED
|
||||
|
||||
@classmethod
|
||||
def factory(cls):
|
||||
return GoogleUnexpectedError()
|
||||
|
||||
|
||||
class SupervisorTimeoutError(KnownError):
|
||||
def message(self):
|
||||
return "A request to the supervisor timed out"
|
||||
|
||||
def code(self):
|
||||
return ERROR_SUPERVISOR_TIMEOUT
|
||||
|
||||
@classmethod
|
||||
def factory(cls):
|
||||
return SupervisorTimeoutError()
|
||||
|
||||
|
||||
class SupervisorUnexpectedError(KnownError):
|
||||
def message(self):
|
||||
return "The supervisor gave an unexpected response"
|
||||
|
||||
def code(self):
|
||||
return ERROR_SUPERVISOR_UNEXPECTED
|
||||
|
||||
@classmethod
|
||||
def factory(cls):
|
||||
return SupervisorUnexpectedError()
|
||||
|
||||
|
||||
class SupervisorFileSystemError(KnownError):
|
||||
def message(self):
|
||||
return "The host file system is read-only. Please restart Home Assistant and verify you have enough free space."
|
||||
|
||||
def code(self):
|
||||
return ERROR_SUPERVISOR_FILE_SYSTEM
|
||||
|
||||
|
||||
class GoogleCredGenerateError(KnownError):
|
||||
def __init__(self, message):
|
||||
self._msg = message
|
||||
|
||||
def message(self):
|
||||
return self._msg
|
||||
|
||||
def code(self):
|
||||
return ERROR_GOOGLE_CRED_PROCESS
|
||||
|
||||
|
||||
class UnknownNetworkStorageError(KnownError):
|
||||
def __init__(self, name: str="Unkown"):
|
||||
self.name = name
|
||||
|
||||
def message(self):
|
||||
return f"The network storage '{self.name}' isn't recognized. Please visit the add-on web UI to select different storage or use the local disk."
|
||||
|
||||
def code(self):
|
||||
return UNKONWN_NETWORK_STORAGE
|
||||
|
||||
def data(self):
|
||||
return {
|
||||
"storage_name": self.name
|
||||
}
|
||||
|
||||
|
||||
class InactiveNetworkStorageError(KnownError):
|
||||
def __init__(self, name: str="Unkown"):
|
||||
self.name = name
|
||||
|
||||
def message(self):
|
||||
return f"The network storage '{self.name}' isn't ready. The network share must be available before it can be used for a backup."
|
||||
|
||||
def code(self):
|
||||
return INACTIVE_NETWORK_STORAGE
|
||||
|
||||
def data(self):
|
||||
return {
|
||||
"storage_name": self.name
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
from .jsonfilesaver import JsonFileSaver
|
||||
from .file import File
|
||||
@@ -0,0 +1,76 @@
|
||||
import os
|
||||
from backup.logger import getLogger
|
||||
from os.path import exists
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
|
||||
class File:
|
||||
"""
|
||||
The envrionment Home Assistant runs in is notorious for disk-related failures, often from running completely out of space and SD card corruption.
|
||||
Both of these can leave the addon in a state where the files it need to run are either corrupted or empty. This class attempts to mitigate that
|
||||
by writing all config files twice, first to a backup file and then to the "real" file path. Then when reading it will check both locations to try
|
||||
and find a copy of the file that isn't corrupted or deleted.
|
||||
This avoids a number of common failures, namely:
|
||||
- A power failure while writing a file can leave it empty or malformed.
|
||||
- Overwriting a file while the disk is full can truncateit without writing the new data
|
||||
- HD corruption cna make a file malformed, but its less likely to affect both files.
|
||||
"""
|
||||
@classmethod
|
||||
def _read(cls, path):
|
||||
with open(path, "r") as f:
|
||||
return f.read()
|
||||
|
||||
@classmethod
|
||||
def read(cls, path):
|
||||
try:
|
||||
data = File._read(path)
|
||||
if len(data) == 0:
|
||||
logger.error(f"The configuration file {path} had an invalid format. This could be caused by hard drive corruption or an unstable power event. We'll attempt to load from a backup file instead.")
|
||||
backup = File._backup_path(path)
|
||||
if not exists(backup):
|
||||
logger.error("Unable to locate a backup path")
|
||||
raise
|
||||
return File._read(backup)
|
||||
else:
|
||||
return data
|
||||
except FileNotFoundError:
|
||||
logger.error(f"The configuration file {path} was not found. This could be caused by hard drive corruption or an unstable power event. We'll attempt to load from a backup file instead.")
|
||||
backup = File._backup_path(path)
|
||||
if not exists(backup):
|
||||
logger.error("Unable to locate a backup path")
|
||||
raise
|
||||
return File._read(backup)
|
||||
|
||||
@classmethod
|
||||
def _write(cls, path, data):
|
||||
with open(path, "w") as f:
|
||||
f.write(data)
|
||||
|
||||
@classmethod
|
||||
def write(cls, path, data):
|
||||
# Crete the backup (recovery) file first. This ensures its present if the subsequent write is corrupted.
|
||||
File._write(File._backup_path(path), data)
|
||||
File._write(path, data)
|
||||
|
||||
@classmethod
|
||||
def exists(cls, path):
|
||||
if exists(path):
|
||||
return True
|
||||
return exists(File._backup_path(path))
|
||||
|
||||
@classmethod
|
||||
def delete(sels, path):
|
||||
if exists(File._backup_path(path)):
|
||||
os.remove(File._backup_path(path))
|
||||
if exists(path):
|
||||
os.remove(path)
|
||||
|
||||
@classmethod
|
||||
def _backup_path(cls, path):
|
||||
return path + ".backup"
|
||||
|
||||
@classmethod
|
||||
def touch(cls, file):
|
||||
with open(file, "w"):
|
||||
pass
|
||||
@@ -0,0 +1,71 @@
|
||||
import json
|
||||
import os
|
||||
from backup.logger import getLogger
|
||||
from os.path import exists
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
|
||||
class JsonFileSaver:
|
||||
"""
|
||||
The envrionment Home Assistant runs in is notorious for disk-related failures, often from running completely out of space and SD card corruption.
|
||||
Both of these can leave the addon in a state where the files it need to run are either corrupted or empty. This class attempts to mitigate that
|
||||
by writing all config files twice, first to a backup file and then to the "real" file path. Then when reading it will check both locations to try
|
||||
and find a copy of the file that isn't corrupted or deleted.
|
||||
This avoids a number of common failures, namely:
|
||||
- A power failure while writing a file can leave it empty or malformed.
|
||||
- Overwriting a file while the disk is full can truncateit without writing the new data
|
||||
- HD corruption cna make a file malformed, but its less likely to affect both files.
|
||||
"""
|
||||
@classmethod
|
||||
def _read(cls, path):
|
||||
with open(path, "r") as f:
|
||||
return json.load(f)
|
||||
|
||||
@classmethod
|
||||
def read(cls, path):
|
||||
try:
|
||||
return JsonFileSaver._read(path)
|
||||
except json.decoder.JSONDecodeError:
|
||||
logger.error(f"The configuration file {path} had an invalid format. This could be caused by hard drive corruption or an unstable power event. We'll attempt to load from a backup file instead.")
|
||||
backup = JsonFileSaver._backup_path(path)
|
||||
if not exists(backup):
|
||||
logger.error("Unable to locate a backup path")
|
||||
raise
|
||||
return JsonFileSaver._read(backup)
|
||||
except FileNotFoundError:
|
||||
logger.error(f"The configuration file {path} was not found. This could be caused by hard drive corruption or an unstable power event. We'll attempt to load from a backup file instead.")
|
||||
backup = JsonFileSaver._backup_path(path)
|
||||
if not exists(backup):
|
||||
logger.error("Unable to locate a backup path")
|
||||
raise
|
||||
return JsonFileSaver._read(backup)
|
||||
|
||||
@classmethod
|
||||
def _write(cls, path, data):
|
||||
with open(path, "w") as f:
|
||||
json.dump(data, f, indent=4)
|
||||
|
||||
@classmethod
|
||||
def write(cls, path, data):
|
||||
# Crete the backup (rcovery) file first. This ensures its present if the subsequent write is corrupted.
|
||||
JsonFileSaver._write(JsonFileSaver._backup_path(path), data)
|
||||
JsonFileSaver._write(path, data)
|
||||
|
||||
@classmethod
|
||||
def exists(cls, path):
|
||||
if exists(path):
|
||||
return True
|
||||
return exists(JsonFileSaver._backup_path(path))
|
||||
|
||||
@classmethod
|
||||
def delete(sels, path):
|
||||
if exists(JsonFileSaver._backup_path(path)):
|
||||
os.remove(JsonFileSaver._backup_path(path))
|
||||
if exists(path):
|
||||
os.remove(path)
|
||||
|
||||
@classmethod
|
||||
def _backup_path(cls, path):
|
||||
return path + ".backup"
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
# flake8: noqa
|
||||
from .hasource import HaSource, HABackup, PendingBackup, SOURCE_HA
|
||||
from .haupdater import HaUpdater
|
||||
from .harequests import HaRequests, EVENT_BACKUP_END, EVENT_BACKUP_START, VERSION_BACKUP_PATH
|
||||
from .backupname import BackupName, BACKUP_NAME_KEYS
|
||||
from .password import Password
|
||||
from .addon_stopper import AddonStopper
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
from backup.config import Config, Setting
|
||||
from backup.file import JsonFileSaver
|
||||
from backup.worker import Worker
|
||||
from backup.exceptions import SupervisorFileSystemError
|
||||
from .harequests import HaRequests
|
||||
from injector import inject, singleton
|
||||
from backup.time import Time
|
||||
from backup.logger import getLogger
|
||||
from datetime import timedelta
|
||||
from asyncio import Lock
|
||||
|
||||
LOGGER = getLogger(__name__)
|
||||
CHECK_DURATION = timedelta(seconds=60)
|
||||
ATTR_STATE = "state"
|
||||
ATTR_WATCHDOG = "watchdog"
|
||||
ATTR_NAME = "name"
|
||||
STATE_STOPPED = "stopped"
|
||||
STATE_STARTED = "started"
|
||||
|
||||
STATES_STOPPED = ["stopped", "unknown", "error"]
|
||||
|
||||
|
||||
@singleton
|
||||
class AddonStopper(Worker):
|
||||
@inject
|
||||
def __init__(self, config: Config, requests: HaRequests, time: Time):
|
||||
super().__init__("StartandStopTimer", self.check, time, 10)
|
||||
self.requests = requests
|
||||
self.config = config
|
||||
self.time = time
|
||||
self.must_start = set()
|
||||
self.must_enable_watchdog = set()
|
||||
self.stop_start_check_time = time.now()
|
||||
self._backing_up = False
|
||||
self.allow_run = False
|
||||
self.lock = Lock()
|
||||
|
||||
async def start(self, schedule=True):
|
||||
if schedule:
|
||||
await super().start()
|
||||
path = self.config.get(Setting.STOP_ADDON_STATE_PATH)
|
||||
if JsonFileSaver.exists(path):
|
||||
data = JsonFileSaver.read(path)
|
||||
self.must_enable_watchdog = set(data.get("watchdog", []))
|
||||
self.must_start = set(data.get("start", []))
|
||||
|
||||
def allowRun(self):
|
||||
if not self.allow_run:
|
||||
for slug in self.config.get(Setting.STOP_ADDONS).split(','):
|
||||
if len(slug) == 0:
|
||||
continue
|
||||
self.must_start.add(slug)
|
||||
self.allow_run = True
|
||||
|
||||
def isBackingUp(self, backingUp):
|
||||
self._backing_up = backingUp
|
||||
|
||||
async def stopAddons(self, self_slug):
|
||||
async with self.lock:
|
||||
self._backing_up = True
|
||||
for slug in self.config.get(Setting.STOP_ADDONS).split(','):
|
||||
if slug == self_slug or len(slug) == 0:
|
||||
# Don't ask the supervisor to stop yourself. That would be BAD.
|
||||
continue
|
||||
try:
|
||||
info = await self.requests.getAddonInfo(slug)
|
||||
if info.get(ATTR_STATE, None) == STATE_STARTED:
|
||||
if info.get(ATTR_WATCHDOG, False):
|
||||
try:
|
||||
LOGGER.info("Temporarily disabling watchdog for addon '%s'", info.get(ATTR_NAME, slug))
|
||||
await self.requests.updateAddonOptions(slug, {ATTR_WATCHDOG: False})
|
||||
self.must_enable_watchdog.add(slug)
|
||||
except Exception as e:
|
||||
LOGGER.error("Unable to disable watchdog for addon {0}".format(info.get(ATTR_NAME, slug)))
|
||||
LOGGER.printException(e)
|
||||
try:
|
||||
LOGGER.info("Stopping addon '%s'", info.get(ATTR_NAME, slug))
|
||||
await self.requests.stopAddon(slug)
|
||||
self.must_start.add(slug)
|
||||
except Exception as e:
|
||||
LOGGER.error("Unable to stop addon '{0}'".format(info.get(ATTR_NAME, slug)))
|
||||
LOGGER.printException(e)
|
||||
except Exception as e:
|
||||
LOGGER.error("Unable to lookup info for addon '{0}', please check your configuration".format(slug))
|
||||
LOGGER.printException(e)
|
||||
self._save()
|
||||
|
||||
async def startAddons(self):
|
||||
self._backing_up = False
|
||||
self.stop_start_check_time = self.time.now() + CHECK_DURATION
|
||||
await self.check()
|
||||
|
||||
async def check(self):
|
||||
async with self.lock:
|
||||
if self._backing_up:
|
||||
return
|
||||
if not self.allow_run:
|
||||
return
|
||||
changes = False
|
||||
if len(self.must_start) > 0:
|
||||
for slug in list(self.must_start):
|
||||
try:
|
||||
info = await self.requests.getAddonInfo(slug)
|
||||
state = info.get(ATTR_STATE, None)
|
||||
if info.get(ATTR_STATE, None) in STATES_STOPPED:
|
||||
LOGGER.info("Starting addon '%s'", info.get(ATTR_NAME, slug))
|
||||
await self.requests.startAddon(slug)
|
||||
self.must_start.remove(slug)
|
||||
changes = True
|
||||
elif info.get(ATTR_STATE, None) == STATE_STARTED and self.time.now() > self.stop_start_check_time:
|
||||
# Give up on restarting it, looks like it was never stopped
|
||||
self.must_start.remove(slug)
|
||||
changes = True
|
||||
else:
|
||||
LOGGER.error(f"Addon '{info.get(ATTR_NAME, slug)} had unrecognized state {state}'. The addon will most likely be unable to automatically restart this addon.", )
|
||||
except Exception as e:
|
||||
LOGGER.error("Unable to start addon '%s'", slug)
|
||||
LOGGER.printException(e)
|
||||
self.must_start.remove(slug)
|
||||
changes = True
|
||||
|
||||
if len(self.must_enable_watchdog) > 0:
|
||||
for slug in list(self.must_enable_watchdog):
|
||||
if slug in self.must_start:
|
||||
# Wait until we're done trying to start the addon before re-enabling the watchdog, otherwise the supervisor complains
|
||||
continue
|
||||
try:
|
||||
info = await self.requests.getAddonInfo(slug)
|
||||
if not info.get(ATTR_WATCHDOG, True):
|
||||
LOGGER.info("Re-enabling watchdog for addon '%s'", info.get(ATTR_NAME, slug))
|
||||
await self.requests.updateAddonOptions(slug, {ATTR_WATCHDOG: True})
|
||||
except Exception as e:
|
||||
LOGGER.error("Unable to re-enable watchdog for addon '%s'", slug)
|
||||
LOGGER.printException(e)
|
||||
self.must_enable_watchdog.remove(slug)
|
||||
changes = True
|
||||
if changes:
|
||||
self._save()
|
||||
|
||||
def _save(self):
|
||||
try:
|
||||
path = self.config.get(Setting.STOP_ADDON_STATE_PATH)
|
||||
data = {"start": list(self.must_start), "watchdog": list(self.must_enable_watchdog)}
|
||||
JsonFileSaver.write(path, data)
|
||||
except OSError:
|
||||
raise SupervisorFileSystemError()
|
||||
@@ -0,0 +1,39 @@
|
||||
from datetime import datetime
|
||||
|
||||
from ..logger import getLogger
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
BACKUP_NAME_KEYS = {
|
||||
"{type}": lambda backup_type, now_local, host_info: backup_type,
|
||||
"{year}": lambda backup_type, now_local, host_info: now_local.strftime("%Y"),
|
||||
"{year_short}": lambda backup_type, now_local, host_info: now_local.strftime("%y"),
|
||||
"{weekday}": lambda backup_type, now_local, host_info: now_local.strftime("%A"),
|
||||
"{weekday_short}": lambda backup_type, now_local, host_info: now_local.strftime("%a"),
|
||||
"{month}": lambda backup_type, now_local, host_info: now_local.strftime("%m"),
|
||||
"{month_long}": lambda backup_type, now_local, host_info: now_local.strftime("%B"),
|
||||
"{month_short}": lambda backup_type, now_local, host_info: now_local.strftime("%b"),
|
||||
"{ms}": lambda backup_type, now_local, host_info: now_local.strftime("%f"),
|
||||
"{day}": lambda backup_type, now_local, host_info: now_local.strftime("%d"),
|
||||
"{hr24}": lambda backup_type, now_local, host_info: now_local.strftime("%H"),
|
||||
"{hr12}": lambda backup_type, now_local, host_info: now_local.strftime("%I"),
|
||||
"{min}": lambda backup_type, now_local, host_info: now_local.strftime("%M"),
|
||||
"{sec}": lambda backup_type, now_local, host_info: now_local.strftime("%S"),
|
||||
"{ampm}": lambda backup_type, now_local, host_info: now_local.strftime("%p"),
|
||||
"{version_ha}": lambda backup_type, now_local, host_info: str(host_info.get('homeassistant', 'Unknown')),
|
||||
"{version_hassos}": lambda backup_type, now_local, host_info: str(host_info.get('hassos', 'Unknown')),
|
||||
"{version_super}": lambda backup_type, now_local, host_info: str(host_info.get('supervisor', 'Unknown')),
|
||||
"{date}": lambda backup_type, now_local, host_info: now_local.strftime("%x"),
|
||||
"{time}": lambda backup_type, now_local, host_info: now_local.strftime("%X"),
|
||||
"{datetime}": lambda backup_type, now_local, host_info: now_local.strftime("%c"),
|
||||
"{isotime}": lambda backup_type, now_local, host_info: now_local.isoformat(),
|
||||
"{hostname}": lambda backup_type, now_local, host_info: str(host_info.get('hostname', 'Unknown')),
|
||||
}
|
||||
|
||||
|
||||
class BackupName():
|
||||
def resolve(self, backup_type: str, template: str, now_local: datetime, host_info) -> str:
|
||||
for key in BACKUP_NAME_KEYS:
|
||||
template = template.replace(key, BACKUP_NAME_KEYS[key](
|
||||
backup_type, now_local, host_info))
|
||||
return template
|
||||
@@ -0,0 +1,346 @@
|
||||
import os
|
||||
from typing import Any, Dict
|
||||
|
||||
from aiohttp import ClientSession, ClientTimeout
|
||||
from aiohttp.client_exceptions import ClientResponseError, ClientConnectorError
|
||||
from injector import inject
|
||||
from asyncio.exceptions import TimeoutError
|
||||
|
||||
from ..util import AsyncHttpGetter
|
||||
from ..config import Config, Setting, Version
|
||||
from ..exceptions import HomeAssistantDeleteError, SupervisorConnectionError, SupervisorPermissionError, SupervisorTimeoutError, SupervisorUnexpectedError
|
||||
from ..model import HABackup
|
||||
from ..logger import getLogger
|
||||
from ..util import DataCache
|
||||
from backup.time import Time
|
||||
from backup.const import NECESSARY_OLD_BACKUP_PLURAL_NAME, NECESSARY_OLD_SUPERVISOR_URL
|
||||
from yarl import URL
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
HEADER_TOKEN = "X-Supervisor-Token"
|
||||
|
||||
NOTIFICATION_ID = "backup_broken"
|
||||
EVENT_BACKUP_START = "backup_started"
|
||||
EVENT_BACKUP_END = "backup_ended"
|
||||
|
||||
VERSION_BACKUP_PATH = Version.parse("2021.8")
|
||||
VERSION_MOUNT_INFO = Version.parse("2023.6")
|
||||
|
||||
|
||||
def supervisor_call(func):
|
||||
async def wrap_and_call(*args, **kwargs):
|
||||
try:
|
||||
return await func(*args, **kwargs)
|
||||
except ClientConnectorError:
|
||||
raise SupervisorConnectionError()
|
||||
except TimeoutError:
|
||||
raise SupervisorConnectionError()
|
||||
except ClientResponseError as e:
|
||||
if e.status == 403:
|
||||
raise SupervisorPermissionError()
|
||||
raise
|
||||
return wrap_and_call
|
||||
|
||||
|
||||
class HaRequests():
|
||||
"""
|
||||
Stores logic for interacting with the supervisor add-on API
|
||||
"""
|
||||
@inject
|
||||
def __init__(self, config: Config, session: ClientSession, time: Time, data_cache: DataCache):
|
||||
self.config: Config = config
|
||||
self.cache = {}
|
||||
self.session = session
|
||||
self._time = time
|
||||
self._data_cache = data_cache
|
||||
|
||||
# default the supervisor versio to using the "most featured" when it can't be parsed.
|
||||
self._super_version = VERSION_MOUNT_INFO
|
||||
|
||||
def getSupervisorURL(self) -> URL:
|
||||
if len(self.config.get(Setting.SUPERVISOR_URL)) > 0:
|
||||
return URL(self.config.get(Setting.SUPERVISOR_URL))
|
||||
if 'SUPERVISOR_TOKEN' in os.environ:
|
||||
return URL("http://supervisor")
|
||||
else:
|
||||
return URL(NECESSARY_OLD_SUPERVISOR_URL)
|
||||
|
||||
def _getBackupPath(self):
|
||||
if self.supportsBackupPaths():
|
||||
return "backups"
|
||||
return NECESSARY_OLD_BACKUP_PLURAL_NAME
|
||||
|
||||
def supportsBackupPaths(self):
|
||||
return not self._super_version or self._super_version >= VERSION_BACKUP_PATH
|
||||
|
||||
@property
|
||||
def supportsMountInfo(self):
|
||||
return not self._super_version or self._super_version >= VERSION_MOUNT_INFO
|
||||
|
||||
@supervisor_call
|
||||
async def createBackup(self, info):
|
||||
if 'folders' in info or 'addons' in info:
|
||||
url = self.getSupervisorURL().with_path("{0}/new/partial".format(self._getBackupPath()))
|
||||
else:
|
||||
url = self.getSupervisorURL().with_path("{0}/new/full".format(self._getBackupPath()))
|
||||
return await self._postHassioData(url, info, timeout=ClientTimeout(total=self.config.get(Setting.PENDING_BACKUP_TIMEOUT_SECONDS)))
|
||||
|
||||
@supervisor_call
|
||||
async def auth(self, user: str, password: str) -> None:
|
||||
await self._postHassioData(self.getSupervisorURL().with_path("auth"), {"username": user, "password": password}, headers=self._altAuthHeaders())
|
||||
|
||||
@supervisor_call
|
||||
async def upload(self, stream):
|
||||
url = self.getSupervisorURL().with_path("{0}/new/upload".format(self._getBackupPath()))
|
||||
return await self._postHassioData(url, data=stream)
|
||||
|
||||
@supervisor_call
|
||||
async def delete(self, slug) -> None:
|
||||
if slug in self.cache:
|
||||
del self.cache[slug]
|
||||
try:
|
||||
if self.supportsBackupPaths():
|
||||
delete_url = self.getSupervisorURL().with_path("{1}/{0}".format(slug, self._getBackupPath()))
|
||||
await self._sendHassioData("delete", delete_url, {})
|
||||
else:
|
||||
delete_url = self.getSupervisorURL().with_path("{1}/{0}/remove".format(slug, self._getBackupPath()))
|
||||
await self._sendHassioData("post", delete_url, {})
|
||||
except ClientResponseError as e:
|
||||
if e.status == 400:
|
||||
raise HomeAssistantDeleteError()
|
||||
raise e
|
||||
|
||||
@supervisor_call
|
||||
async def startAddon(self, slug) -> None:
|
||||
url = self.getSupervisorURL().with_path("addons/{0}/start".format(slug))
|
||||
await self._postHassioData(url, {})
|
||||
|
||||
@supervisor_call
|
||||
async def stopAddon(self, slug) -> None:
|
||||
url = self.getSupervisorURL().with_path("addons/{0}/stop".format(slug))
|
||||
await self._postHassioData(url, {})
|
||||
|
||||
@supervisor_call
|
||||
async def backup(self, slug):
|
||||
if slug in self.cache:
|
||||
info = self.cache[slug]
|
||||
else:
|
||||
info = await self._getHassioData(self.getSupervisorURL().with_path("{1}/{0}/info".format(slug, self._getBackupPath())))
|
||||
self.cache[slug] = info
|
||||
return HABackup(info, self._data_cache, self.config, self.config.isRetained(slug))
|
||||
|
||||
@supervisor_call
|
||||
async def backups(self):
|
||||
return await self._getHassioData(self.getSupervisorURL().with_path(self._getBackupPath()))
|
||||
|
||||
@supervisor_call
|
||||
async def haInfo(self):
|
||||
return await self._getHassioData(self.getSupervisorURL().with_path("core/info"))
|
||||
|
||||
@supervisor_call
|
||||
async def selfInfo(self) -> Dict[str, Any]:
|
||||
return await self.getAddonInfo("self")
|
||||
|
||||
@supervisor_call
|
||||
async def getAddonInfo(self, addon_slug) -> Dict[str, Any]:
|
||||
return await self._getHassioData(self.getSupervisorURL().with_path("addons/{0}/info".format(addon_slug)))
|
||||
|
||||
@supervisor_call
|
||||
async def getAddons(self) -> Dict[str, Any]:
|
||||
return await self._getHassioData(self.getSupervisorURL().with_path("addons"))
|
||||
|
||||
@supervisor_call
|
||||
async def hassosInfo(self) -> Dict[str, Any]:
|
||||
return await self._getHassioData(self.getSupervisorURL().with_path("hassos/info"))
|
||||
|
||||
@supervisor_call
|
||||
async def info(self) -> Dict[str, Any]:
|
||||
return await self._getHassioData(self.getSupervisorURL().with_path("info"))
|
||||
|
||||
@supervisor_call
|
||||
async def refreshBackups(self):
|
||||
url = self.getSupervisorURL().with_path("{0}/reload".format(self._getBackupPath()))
|
||||
return await self._postHassioData(url)
|
||||
|
||||
@supervisor_call
|
||||
async def supervisorInfo(self):
|
||||
url = self.getSupervisorURL().with_path("supervisor/info")
|
||||
info = await self._getHassioData(url)
|
||||
|
||||
# parse the supervisor version
|
||||
if 'version' in info:
|
||||
self._super_version = Version.parse(info['version'])
|
||||
return info
|
||||
|
||||
@supervisor_call
|
||||
async def mountInfo(self):
|
||||
if self.supportsMountInfo:
|
||||
url = self.getSupervisorURL().with_path("mounts")
|
||||
info = await self._getHassioData(url)
|
||||
return info
|
||||
else:
|
||||
return {
|
||||
"default_backup_mount": None,
|
||||
"mounts": []
|
||||
}
|
||||
|
||||
@supervisor_call
|
||||
async def restore(self, slug: str, password: str = None) -> None:
|
||||
url = self.getSupervisorURL().with_path("{1}/{0}/restore/full".format(slug, self._getBackupPath()))
|
||||
if password:
|
||||
await self._postHassioData(url, {'password': password})
|
||||
else:
|
||||
await self._postHassioData(url, {})
|
||||
|
||||
@supervisor_call
|
||||
async def download(self, slug) -> AsyncHttpGetter:
|
||||
url = self.getSupervisorURL().with_path("{1}/{0}/download".format(slug, self._getBackupPath()))
|
||||
ret = AsyncHttpGetter(url,
|
||||
self._getAuthHeaders(),
|
||||
self.session,
|
||||
timeoutFactory=SupervisorTimeoutError.factory,
|
||||
otherErrorFactory=SupervisorUnexpectedError.factory,
|
||||
timeout=ClientTimeout(sock_connect=self.config.get(Setting.DOWNLOAD_TIMEOUT_SECONDS),
|
||||
sock_read=self.config.get(Setting.DOWNLOAD_TIMEOUT_SECONDS)),
|
||||
time=self._time)
|
||||
return ret
|
||||
|
||||
@supervisor_call
|
||||
async def getSuperLogs(self):
|
||||
url = self.getSupervisorURL().with_path("supervisor/logs")
|
||||
async with self.session.get(url, headers=self._getAuthHeaders()) as resp:
|
||||
resp.raise_for_status()
|
||||
return await resp.text()
|
||||
|
||||
@supervisor_call
|
||||
async def getCoreLogs(self):
|
||||
url = self.getSupervisorURL().with_path("core/logs")
|
||||
async with self.session.get(url, headers=self._getAuthHeaders()) as resp:
|
||||
resp.raise_for_status()
|
||||
return await resp.text()
|
||||
|
||||
async def _validateHassioReply(self, resp) -> Dict[str, Any]:
|
||||
async with resp:
|
||||
resp.raise_for_status()
|
||||
details: Dict[str, Any] = await resp.json()
|
||||
if "result" not in details or details["result"] != "ok":
|
||||
if "result" in details:
|
||||
raise Exception("Hassio said: " + details["result"])
|
||||
else:
|
||||
raise Exception(
|
||||
"Malformed response from Hassio: " + str(details))
|
||||
|
||||
if "data" not in details:
|
||||
return {}
|
||||
if self.config.get(Setting.TRACE_REQUESTS):
|
||||
logger.trace("Hassio replied: %s", details)
|
||||
return details["data"]
|
||||
|
||||
async def getAddonLogo(self, slug: str):
|
||||
url = self.getSupervisorURL().with_path("addons/{0}/icon".format(slug))
|
||||
async with self.session.get(url, headers=self._getAuthHeaders()) as resp:
|
||||
resp.raise_for_status()
|
||||
return (resp.headers['Content-Type'], await resp.read())
|
||||
|
||||
def _getToken(self):
|
||||
configured = self.config.get(Setting.SUPERVISOR_TOKEN)
|
||||
if configured and len(configured) > 0:
|
||||
return configured
|
||||
if "SUPERVISOR_TOKEN" in os.environ:
|
||||
return os.environ.get("SUPERVISOR_TOKEN")
|
||||
# Older versions of the supervisor use a different name for the token.
|
||||
return os.environ.get("HASSIO_TOKEN")
|
||||
|
||||
def _getAuthHeaders(self):
|
||||
return {
|
||||
'Authorization': 'Bearer ' + self._getToken()
|
||||
}
|
||||
|
||||
def _altAuthHeaders(self):
|
||||
return {
|
||||
HEADER_TOKEN: self._getToken()
|
||||
}
|
||||
|
||||
@supervisor_call
|
||||
async def _getHassioData(self, url: URL) -> Dict[str, Any]:
|
||||
if self.config.get(Setting.TRACE_REQUESTS):
|
||||
logger.trace("Making Hassio request: " + str(url))
|
||||
return await self._validateHassioReply(await self.session.get(url, headers=self._getAuthHeaders()))
|
||||
|
||||
async def _postHassioData(self, url: URL, json=None, file=None, data=None, timeout=None, headers=None) -> Dict[str, Any]:
|
||||
return await self._sendHassioData("post", url, json, file, data, timeout, headers)
|
||||
|
||||
@supervisor_call
|
||||
async def _sendHassioData(self, method: str, url: URL, json=None, file=None, data=None, timeout=None, headers=None) -> Dict[str, Any]:
|
||||
if headers is None:
|
||||
headers = self._getAuthHeaders()
|
||||
if self.config.get(Setting.TRACE_REQUESTS):
|
||||
logger.trace("Making Hassio request: " + str(url))
|
||||
return await self._validateHassioReply(await self.session.request(method, url, headers=headers, json=json, data=data, timeout=timeout))
|
||||
|
||||
async def _postHaData(self, path: str, data: Dict[str, Any]) -> None:
|
||||
url = self.getSupervisorURL().with_path("/core/api/" + path)
|
||||
async with self.session.post(url, headers=self._getAuthHeaders(), json=data) as resp:
|
||||
resp.raise_for_status()
|
||||
|
||||
async def sendNotification(self, title: str, message: str) -> None:
|
||||
data: Dict[str, str] = {
|
||||
"title": title,
|
||||
"message": message,
|
||||
"notification_id": NOTIFICATION_ID
|
||||
}
|
||||
await self._postHaData("services/persistent_notification/create", data)
|
||||
|
||||
async def eventBackupStart(self, name, time):
|
||||
await self._sendEvent(EVENT_BACKUP_START, {
|
||||
'backup_name': name,
|
||||
'backup_time': str(time)
|
||||
})
|
||||
|
||||
async def eventBackupEnd(self, name, time, completed):
|
||||
await self._sendEvent(EVENT_BACKUP_END, {
|
||||
'completed': completed,
|
||||
'backup_name': name,
|
||||
'backup_time': str(time)
|
||||
})
|
||||
|
||||
async def _sendEvent(self, event_name: str, data: Dict[str, str]) -> None:
|
||||
await self._postHaData("events/" + event_name, data)
|
||||
|
||||
async def dismissNotification(self) -> None:
|
||||
data: Dict[str, str] = {
|
||||
"notification_id": NOTIFICATION_ID
|
||||
}
|
||||
await self._postHaData("services/persistent_notification/dismiss", data)
|
||||
|
||||
async def updateBackupStaleSensor(self, state: bool) -> None:
|
||||
if self.config.get(Setting.CALL_BACKUP_SNAPSHOT):
|
||||
data: Dict[str, Any] = {
|
||||
"state": state,
|
||||
"attributes": {
|
||||
"friendly_name": "Snapshots Stale",
|
||||
"device_class": "problem"
|
||||
}
|
||||
}
|
||||
await self._postHaData("states/binary_sensor." + NECESSARY_OLD_BACKUP_PLURAL_NAME + "_stale", data)
|
||||
else:
|
||||
data: Dict[str, Any] = {
|
||||
"state": state,
|
||||
"attributes": {
|
||||
"friendly_name": "Backups Stale",
|
||||
"device_class": "problem"
|
||||
}
|
||||
}
|
||||
await self._postHaData("states/binary_sensor.backups_stale", data)
|
||||
|
||||
@supervisor_call
|
||||
async def updateConfig(self, config) -> None:
|
||||
return await self._postHassioData(self.getSupervisorURL().with_path("addons/self/options"), {'options': config})
|
||||
|
||||
@supervisor_call
|
||||
async def updateAddonOptions(self, slug, options):
|
||||
return await self._postHassioData(self.getSupervisorURL().with_path("addons/{0}/options".format(slug)), options)
|
||||
|
||||
async def updateEntity(self, entity, data):
|
||||
await self._postHaData("states/" + entity, data)
|
||||
@@ -0,0 +1,574 @@
|
||||
import asyncio
|
||||
import aiohttp
|
||||
from datetime import timedelta
|
||||
from io import IOBase
|
||||
from threading import Lock, Thread
|
||||
from typing import Dict, List, Optional, Any, Union
|
||||
|
||||
from aiohttp.client_exceptions import ClientResponseError
|
||||
from injector import inject, singleton
|
||||
|
||||
from backup.util import AsyncHttpGetter, GlobalInfo, Estimator, DataCache, KEY_NOTE, KEY_LAST_SEEN, KEY_PENDING, KEY_NAME, KEY_CREATED, KEY_I_MADE_THIS, KEY_IGNORE
|
||||
from ..config import Config, Setting, CreateOptions, Startable, Version
|
||||
from ..const import SOURCE_HA
|
||||
from ..model import BackupSource, AbstractBackup, HABackup, Backup
|
||||
from ..exceptions import (LogicError, BackupInProgress, UnknownNetworkStorageError, InactiveNetworkStorageError,
|
||||
UploadFailed, ensureKey)
|
||||
from .harequests import HaRequests
|
||||
from .password import Password
|
||||
from .backupname import BackupName
|
||||
from ..time import Time
|
||||
from ..logger import getLogger, StandardLogger
|
||||
from backup.const import FOLDERS, NECESSARY_OLD_BACKUP_PLURAL_NAME
|
||||
from .addon_stopper import LOGGER, AddonStopper
|
||||
|
||||
logger: StandardLogger = getLogger(__name__)
|
||||
|
||||
class PendingBackup(AbstractBackup):
|
||||
def __init__(self, backupType, protected, options: CreateOptions, request_info, config, time):
|
||||
super().__init__(
|
||||
name=request_info['name'],
|
||||
slug="pending",
|
||||
date=options.when,
|
||||
size="pending",
|
||||
source=SOURCE_HA,
|
||||
backupType=backupType,
|
||||
version="",
|
||||
protected=protected,
|
||||
retained=False,
|
||||
uploadable=False,
|
||||
details=None,
|
||||
note=options.note,
|
||||
pending=True)
|
||||
self._config = config
|
||||
self._failed = False
|
||||
self._complete = False
|
||||
self._exception = None
|
||||
self._failed_at = None
|
||||
self.setOptions(options)
|
||||
self._request_info = request_info
|
||||
self._completed_slug = None
|
||||
self._time = time
|
||||
self._pending_subverted = False
|
||||
self._start_time = time.now()
|
||||
|
||||
def considerForPurge(self) -> bool:
|
||||
return False
|
||||
|
||||
def startTime(self):
|
||||
return self._start_time
|
||||
|
||||
def failed(self, exception, time):
|
||||
self._failed = True
|
||||
self._exception = exception
|
||||
self._failed_at = time
|
||||
|
||||
def getFailureTime(self):
|
||||
return self._failed_at
|
||||
|
||||
def complete(self, slug):
|
||||
self._complete = True
|
||||
self._completed_slug = slug
|
||||
|
||||
def setPendingUnknown(self):
|
||||
self._name = "Pending Backup"
|
||||
self._backupType = "unknown"
|
||||
self._protected = False
|
||||
self._pending_subverted = True
|
||||
self._note = None
|
||||
|
||||
def createdSlug(self):
|
||||
return self._completed_slug
|
||||
|
||||
def isComplete(self):
|
||||
return self._complete
|
||||
|
||||
def isFailed(self):
|
||||
return self._failed
|
||||
|
||||
def status(self):
|
||||
if self._complete:
|
||||
return "Created"
|
||||
if self._failed:
|
||||
return "Failed!"
|
||||
return "Pending"
|
||||
|
||||
def raiseIfNeeded(self):
|
||||
if self.isFailed():
|
||||
raise self._exception
|
||||
if self._pending_subverted:
|
||||
raise BackupInProgress()
|
||||
|
||||
def isStale(self):
|
||||
if self._pending_subverted:
|
||||
delta = timedelta(seconds=self._config.get(
|
||||
Setting.BACKUP_STALE_SECONDS))
|
||||
if self._time.now() > self.startTime() + delta:
|
||||
return True
|
||||
if not self.isFailed():
|
||||
return False
|
||||
delta = timedelta(seconds=self._config.get(
|
||||
Setting.FAILED_BACKUP_TIMEOUT_SECONDS))
|
||||
staleTime = self.getFailureTime() + delta
|
||||
return self._time.now() >= staleTime
|
||||
|
||||
def madeByTheAddon(self):
|
||||
return True
|
||||
|
||||
|
||||
@singleton
|
||||
class HaSource(BackupSource[HABackup], Startable):
|
||||
"""
|
||||
Stores logic for interacting with the supervisor add-on API
|
||||
"""
|
||||
@inject
|
||||
def __init__(self, config: Config, time: Time, ha: HaRequests, info: GlobalInfo, stopper: AddonStopper, estimator: Estimator, data_cache: DataCache):
|
||||
super().__init__()
|
||||
self.config: Config = config
|
||||
self._data_cache = data_cache
|
||||
self.backup_thread: Thread = None
|
||||
self.pending_backup_error: Optional[Exception] = None
|
||||
self.pending_backup_slug: Optional[str] = None
|
||||
self.self_info = None
|
||||
self.host_info = None
|
||||
self.ha_info = None
|
||||
self.super_info = None
|
||||
self.mount_info = {}
|
||||
self.lock: Lock = Lock()
|
||||
self.time = time
|
||||
self.harequests = ha
|
||||
self.last_slugs = set()
|
||||
self.retained = []
|
||||
self.cached_retention = {}
|
||||
self._info = info
|
||||
self.pending_options = {}
|
||||
self.stopper = stopper
|
||||
self.estimator = estimator
|
||||
self._addons = {}
|
||||
self._changes_from_last_query = False
|
||||
|
||||
# This lock should be used for _ANYTHING_ that interacts with self._pending_backup
|
||||
self._pending_backup_lock = asyncio.Lock()
|
||||
self.pending_backup: Optional[PendingBackup] = None
|
||||
self._pending_backup_task = None
|
||||
self._initialized = False
|
||||
|
||||
def isInitialized(self):
|
||||
return self._initialized
|
||||
|
||||
async def check(self) -> bool:
|
||||
pending = self.pending_backup
|
||||
if pending and pending.isStale():
|
||||
self.trigger()
|
||||
return await super().check()
|
||||
|
||||
def icon(self) -> str:
|
||||
return "home-assistant"
|
||||
|
||||
def name(self) -> str:
|
||||
return SOURCE_HA
|
||||
|
||||
def title(self) -> str:
|
||||
return "Home Assistant"
|
||||
|
||||
def maxCount(self) -> None:
|
||||
return self.config.get(Setting.MAX_BACKUPS_IN_HA)
|
||||
|
||||
def enabled(self) -> bool:
|
||||
return True
|
||||
|
||||
def freeSpace(self):
|
||||
return self.estimator.getBytesFree()
|
||||
|
||||
async def create(self, options: CreateOptions) -> HABackup:
|
||||
# Make sure instance info is up-to-date, for the backup name
|
||||
await self._refreshInfo()
|
||||
|
||||
# Set a default name if it was unspecified
|
||||
if options.name_template is None or len(options.name_template) == 0:
|
||||
options.name_template = self.config.get(Setting.BACKUP_NAME)
|
||||
|
||||
# Build the backup request json, get type, etc
|
||||
request, type_name, protected = self._buildBackupInfo(
|
||||
options)
|
||||
|
||||
async with self._pending_backup_lock:
|
||||
# Check if a backup is already in progress
|
||||
if self.pending_backup:
|
||||
if not self.pending_backup.isFailed() and not self.pending_backup.isComplete():
|
||||
logger.info("A backup was already in progress")
|
||||
raise BackupInProgress()
|
||||
|
||||
# try to stop addons
|
||||
await self.stopper.stopAddons(self.self_info['slug'])
|
||||
|
||||
# Create the backup palceholder object
|
||||
self.pending_backup = PendingBackup(
|
||||
type_name, protected, options, request, self.config, self.time)
|
||||
logger.info("Requesting a new backup")
|
||||
self._pending_backup_task = asyncio.create_task(self._requestAsync(
|
||||
self.pending_backup), name="Pending Backup Requester")
|
||||
await asyncio.wait({self._pending_backup_task}, timeout=self.config.get(Setting.NEW_BACKUP_TIMEOUT_SECONDS))
|
||||
self.pending_backup.raiseIfNeeded()
|
||||
|
||||
# There is not other backup in progress, so assume its been requested.
|
||||
pending = self._data_cache.backup(KEY_PENDING)
|
||||
pending[KEY_NAME] = request['name']
|
||||
pending[KEY_CREATED] = options.when.isoformat()
|
||||
pending[KEY_LAST_SEEN] = self.time.now().isoformat()
|
||||
self._data_cache.makeDirty()
|
||||
if self.pending_backup.isComplete():
|
||||
# It completed while we waited, so just query the new backup
|
||||
ret = await self.harequests.backup(self.pending_backup.createdSlug())
|
||||
if options.note is not None:
|
||||
ret.setNote(options.note)
|
||||
self.setDataCacheInfo(ret)
|
||||
self._data_cache.backup(ret.slug())[KEY_I_MADE_THIS] = True
|
||||
return ret
|
||||
else:
|
||||
return self.pending_backup
|
||||
|
||||
def _isHttp400(self, e):
|
||||
if isinstance(e, ClientResponseError):
|
||||
return e.status == 400
|
||||
return False
|
||||
|
||||
async def start(self):
|
||||
try:
|
||||
await self.init()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async def stop(self):
|
||||
if self._pending_backup_task:
|
||||
self._pending_backup_task.cancel()
|
||||
await asyncio.wait([self._pending_backup_task])
|
||||
|
||||
@property
|
||||
def needsSpaceCheck(self):
|
||||
if not self.harequests.supportsMountInfo:
|
||||
return True
|
||||
if self.config.get(Setting.BACKUP_STORAGE) == 'local-disk':
|
||||
return True
|
||||
if self.mount_info.get("default_backup_mount") is None and len(self.config.get(Setting.BACKUP_STORAGE)) == 0:
|
||||
return True
|
||||
return False
|
||||
|
||||
@property
|
||||
def query_had_changes(self):
|
||||
return self._changes_from_last_query
|
||||
|
||||
async def get(self) -> Dict[str, HABackup]:
|
||||
if not self._initialized:
|
||||
await self.init()
|
||||
else:
|
||||
# Always ensure the supervisor version is fresh before makign any other requests
|
||||
self.super_info = await self.harequests.supervisorInfo()
|
||||
slugs = set()
|
||||
retained = []
|
||||
backups: Dict[str, HABackup] = {}
|
||||
query = await self.harequests.backups()
|
||||
|
||||
# Different supervisor version use different names for the list of backups
|
||||
backup_list = []
|
||||
if NECESSARY_OLD_BACKUP_PLURAL_NAME in query:
|
||||
backup_list = query[NECESSARY_OLD_BACKUP_PLURAL_NAME]
|
||||
if 'backups' in query:
|
||||
backup_list = query['backups']
|
||||
|
||||
for backup in backup_list:
|
||||
slug = backup['slug']
|
||||
slugs.add(slug)
|
||||
item = await self.harequests.backup(slug)
|
||||
if slug in self.pending_options:
|
||||
item.setOptions(self.pending_options[slug])
|
||||
backups[slug] = item
|
||||
if item.retained():
|
||||
retained.append(item.slug())
|
||||
self.setDataCacheInfo(item)
|
||||
if self.pending_backup:
|
||||
async with self._pending_backup_lock:
|
||||
if self.pending_backup:
|
||||
if self.pending_backup.isStale():
|
||||
# The backup is stale, so just let it die.
|
||||
self._killPending()
|
||||
elif self.pending_backup.isComplete() and self.pending_backup.createdSlug() in backups:
|
||||
# Copy over options if we got the requested backup.
|
||||
backups[self.pending_backup.createdSlug()].setOptions(
|
||||
self.pending_backup.getOptions())
|
||||
if self.pending_backup.note() is not None:
|
||||
# Save the note with the now known slug
|
||||
await self.note(backups[self.pending_backup.createdSlug()], self.pending_backup.note())
|
||||
self._killPending()
|
||||
elif self.last_slugs.symmetric_difference(slugs).intersection(slugs):
|
||||
# New backup added, ignore pending backup.
|
||||
sorted = list(backups.values())
|
||||
sorted.sort(key=HABackup.date)
|
||||
if self.pending_backup.note() is not None and len(sorted) > 0:
|
||||
# Save the note with the newest backup
|
||||
await self.note(sorted[-1], self.pending_backup.note())
|
||||
self._killPending()
|
||||
if self.pending_backup:
|
||||
backups[self.pending_backup.slug()] = self.pending_backup
|
||||
for slug in retained:
|
||||
if not self.config.isRetained(slug):
|
||||
self.config.setRetained(slug, False)
|
||||
self._changes_from_last_query = self.last_slugs != slugs
|
||||
self.last_slugs = slugs
|
||||
return backups
|
||||
|
||||
def setDataCacheInfo(self, backup: HABackup):
|
||||
if backup.slug() not in self._data_cache.backups:
|
||||
# its a new backup, so we need to create a record for it
|
||||
pending = self._data_cache.backups.get(KEY_PENDING, {})
|
||||
pending_created = self.time.parse(pending.get(KEY_CREATED, self.time.now().isoformat()))
|
||||
|
||||
# If the backup has the same name as the one we created and it was created within a day
|
||||
# of the requested time, then assume the addon created it.
|
||||
self_created = backup.name() == pending.get(KEY_NAME, None) and abs((pending_created - backup.date()).total_seconds()) < timedelta(days=1).total_seconds()
|
||||
|
||||
stored_backup = self._data_cache.backup(backup.slug())
|
||||
stored_backup[KEY_I_MADE_THIS] = self_created
|
||||
stored_backup[KEY_CREATED] = backup.date().isoformat()
|
||||
stored_backup[KEY_NAME] = backup.name()
|
||||
stored_backup[KEY_NOTE] = backup.note()
|
||||
if self_created:
|
||||
# Remove the pending backup info from the cache so it doesn't get reused.
|
||||
del self._data_cache.backups[KEY_PENDING]
|
||||
# bump the last seen time
|
||||
self._data_cache.backup(backup.slug())[KEY_LAST_SEEN] = self.time.now().isoformat()
|
||||
self._data_cache.makeDirty()
|
||||
|
||||
async def delete(self, backup: Backup):
|
||||
slug = self._validateBackup(backup).slug()
|
||||
logger.info("Deleting '{0}' from Home Assistant".format(backup.name()))
|
||||
await self.harequests.delete(slug)
|
||||
backup.removeSource(self.name())
|
||||
|
||||
async def ignore(self, backup: Backup, ignore: bool):
|
||||
slug = self._validateBackup(backup).slug()
|
||||
logger.info("Updating ignore settings for '{0}'".format(backup.name()))
|
||||
self._data_cache.backup(slug)[KEY_IGNORE] = ignore
|
||||
self._data_cache.makeDirty()
|
||||
|
||||
async def note(self, backup, note: Union[str, None]) -> None:
|
||||
if isinstance(backup, HABackup):
|
||||
validated = backup
|
||||
else:
|
||||
validated = self._validateBackup(backup)
|
||||
logger.debug(f"Adding a note to ha backup '{validated.name()}'")
|
||||
if isinstance(validated, PendingBackup):
|
||||
# The ntoe will get set once the backup is created and we know the slug
|
||||
validated.setNote(note)
|
||||
else:
|
||||
self._data_cache.backup(validated.slug())[KEY_NOTE] = note
|
||||
self._data_cache.makeDirty()
|
||||
validated.setNote(note)
|
||||
return await super().note(backup, note)
|
||||
|
||||
async def save(self, backup: Backup, source: AsyncHttpGetter) -> HABackup:
|
||||
logger.info("Downloading '{0}'".format(backup.name()))
|
||||
self._info.upload(0)
|
||||
resp = None
|
||||
try:
|
||||
backup.overrideStatus("Loading {0}%", source)
|
||||
backup.setUploadSource(self.title(), source)
|
||||
async with source:
|
||||
with aiohttp.MultipartWriter('mixed') as mpwriter:
|
||||
mpwriter.append(source, {'CONTENT-TYPE': 'application/tar'})
|
||||
resp = await self.harequests.upload(mpwriter)
|
||||
backup.clearStatus()
|
||||
backup.clearUploadSource()
|
||||
except Exception as e:
|
||||
logger.printException(e)
|
||||
backup.overrideStatus("Failed!")
|
||||
backup.uploadFailure(logger.formatException(e))
|
||||
if resp and 'slug' in resp and resp['slug'] == backup.slug():
|
||||
self.config.setRetained(backup.slug(), True)
|
||||
return await self.harequests.backup(backup.slug())
|
||||
else:
|
||||
raise UploadFailed()
|
||||
|
||||
async def read(self, backup: Backup) -> IOBase:
|
||||
item = self._validateBackup(backup)
|
||||
return await self.harequests.download(item.slug())
|
||||
|
||||
async def retain(self, backup: Backup, retain: bool) -> None:
|
||||
item: HABackup = self._validateBackup(backup)
|
||||
item._retained = retain
|
||||
self.config.setRetained(backup.slug(), retain)
|
||||
|
||||
async def init(self):
|
||||
await self._refreshInfo()
|
||||
self._initialized = True
|
||||
|
||||
async def refresh(self):
|
||||
await self._refreshInfo()
|
||||
|
||||
async def _refreshInfo(self) -> None:
|
||||
try:
|
||||
self.self_info = await self.harequests.selfInfo()
|
||||
self.host_info = await self.harequests.info()
|
||||
self.ha_info = await self.harequests.haInfo()
|
||||
self.super_info = await self.harequests.supervisorInfo()
|
||||
self.mount_info = await self.harequests.mountInfo()
|
||||
|
||||
addon_info = ensureKey("addons", await self.harequests.getAddons(), "Supervisor Metadata")
|
||||
self.config.update(
|
||||
ensureKey("options", self.self_info, "addon metdata"))
|
||||
if self.config.mustSaveUpgradeChanges():
|
||||
LOGGER.info("The configuration format has changed in this version of the addon and your configuration will be automatically updated")
|
||||
options = {}
|
||||
for option in self.config.getAllConfig().keys():
|
||||
options[option.value] = self.config.get(option)
|
||||
await self.harequests.updateConfig(options)
|
||||
self.config.persistedChanges()
|
||||
|
||||
self._info.ha_port = ensureKey(
|
||||
"port", self.ha_info, "Home Assistant metadata")
|
||||
self._info.ha_ssl = ensureKey(
|
||||
"ssl", self.ha_info, "Home Assistant metadata")
|
||||
self._info.addons = addon_info
|
||||
self._info.slug = ensureKey(
|
||||
"slug", self.self_info, "addon metdata")
|
||||
self._info.url = self.getAddonUrl()
|
||||
|
||||
self._addons = {}
|
||||
for addon in addon_info:
|
||||
self._addons[addon.get('slug', "default")] = addon
|
||||
|
||||
self._info.addDebugInfo("self_info", self.self_info)
|
||||
self._info.addDebugInfo("host_info", self.host_info)
|
||||
self._info.addDebugInfo("ha_info", self.ha_info)
|
||||
self._info.addDebugInfo("super_info", self.super_info)
|
||||
except Exception as e:
|
||||
logger.debug("Failed to connect to supervisor")
|
||||
logger.debug(logger.formatException(e))
|
||||
raise e
|
||||
|
||||
def addonHasLogo(self, slug):
|
||||
return self._addons.get(slug, {}).get('logo', False)
|
||||
|
||||
def getAddonUrl(self):
|
||||
"""
|
||||
Returns the relative path to the add-on, for the purpose of linking to the add-on page from within Home Assistant.
|
||||
"""
|
||||
if self._info.slug is None:
|
||||
return ""
|
||||
return "/hassio/ingress/" + str(self._info.slug)
|
||||
|
||||
def getHostInfo(self):
|
||||
if not self.isInitialized():
|
||||
return {}
|
||||
return self.host_info
|
||||
|
||||
def getFullAddonUrl(self):
|
||||
if not self.isInitialized():
|
||||
return ""
|
||||
return self._haUrl() + "hassio/ingress/" + str(self._info.slug)
|
||||
|
||||
def getHomeAssistantUrl(self):
|
||||
if not self.isInitialized():
|
||||
return ""
|
||||
return self._haUrl()
|
||||
|
||||
def _haUrl(self):
|
||||
if self._info.ha_ssl:
|
||||
protocol = "https://"
|
||||
else:
|
||||
protocol = "http://"
|
||||
return "".join([protocol, "{host}:", str(self._info.ha_port), "/"])
|
||||
|
||||
def _validateBackup(self, backup) -> HABackup:
|
||||
item: HABackup = backup.getSource(self.name())
|
||||
if not item:
|
||||
raise LogicError(
|
||||
"Requested to do something with a backup from Home Assistant, but the backup has no Home Assistant source")
|
||||
return item
|
||||
|
||||
def _killPending(self):
|
||||
self.pending_backup = None
|
||||
if self._pending_backup_task and not self._pending_backup_task.done():
|
||||
self._pending_backup_task.cancel()
|
||||
|
||||
def postSync(self):
|
||||
self.stopper.allowRun()
|
||||
self.stopper.isBackingUp(self.pending_backup is not None)
|
||||
|
||||
async def _requestAsync(self, pending: PendingBackup, start=[]) -> None:
|
||||
try:
|
||||
result = await asyncio.wait_for(self.harequests.createBackup(pending._request_info), timeout=self.config.get(Setting.PENDING_BACKUP_TIMEOUT_SECONDS))
|
||||
slug = ensureKey(
|
||||
"slug", result, "supervisor's create backup response")
|
||||
pending.complete(slug)
|
||||
self.config.setRetained(
|
||||
slug, pending.getOptions().retain_sources.get(self.name(), False))
|
||||
logger.info("Backup finished")
|
||||
except Exception as e:
|
||||
if self._isHttp400(e):
|
||||
logger.warning("A backup was already in progress")
|
||||
pending.setPendingUnknown()
|
||||
else:
|
||||
logger.error("Backup failed:")
|
||||
logger.printException(e)
|
||||
pending.failed(e, self.time.now())
|
||||
finally:
|
||||
await self.stopper.startAddons()
|
||||
self.trigger()
|
||||
|
||||
def _buildBackupInfo(self, options: CreateOptions):
|
||||
addons: List[str] = []
|
||||
for addon in self.super_info.get('addons', {}):
|
||||
addons.append(addon['slug'])
|
||||
request_info: Dict[str, Any] = {
|
||||
'addons': [],
|
||||
'folders': []
|
||||
}
|
||||
folders = list(map(lambda f: f['slug'], FOLDERS))
|
||||
type_name = "Full"
|
||||
for folder in folders:
|
||||
if folder not in self.config.get(Setting.EXCLUDE_FOLDERS):
|
||||
request_info['folders'].append(folder)
|
||||
else:
|
||||
type_name = "Partial"
|
||||
for addon in addons:
|
||||
if addon not in self.config.get(Setting.EXCLUDE_ADDONS):
|
||||
request_info['addons'].append(addon)
|
||||
else:
|
||||
type_name = "Partial"
|
||||
if type_name == "Full":
|
||||
del request_info['addons']
|
||||
del request_info['folders']
|
||||
protected = False
|
||||
password = Password(self.config).resolve()
|
||||
if password:
|
||||
request_info['password'] = password
|
||||
name = BackupName().resolve(type_name, options.name_template,
|
||||
self.time.toLocal(options.when), self.host_info)
|
||||
request_info['name'] = name
|
||||
|
||||
if self.harequests.supportsMountInfo:
|
||||
# Validate the mount location and set it if necessary
|
||||
mount_name = self.config.get(Setting.BACKUP_STORAGE)
|
||||
|
||||
# Default is to use Home Assistant's default configured mount
|
||||
if not mount_name or len(mount_name) == 0:
|
||||
ha_default = self.mount_info.get("default_backup_mount", None)
|
||||
if ha_default:
|
||||
mount_name = ha_default
|
||||
else:
|
||||
mount_name = "local-disk"
|
||||
|
||||
if mount_name != "local-disk":
|
||||
# check to make sure the mount location is valid
|
||||
for mount in self.mount_info.get("mounts", []):
|
||||
if mount.get("name", None) == mount_name:
|
||||
if mount.get("state", False) != "active":
|
||||
raise InactiveNetworkStorageError(mount_name)
|
||||
request_info['location'] = mount_name
|
||||
break
|
||||
if request_info.get('location', None) is None:
|
||||
raise UnknownNetworkStorageError(mount_name)
|
||||
else:
|
||||
request_info['location'] = None
|
||||
return request_info, type_name, protected
|
||||
@@ -0,0 +1,189 @@
|
||||
from datetime import timedelta
|
||||
|
||||
from aiohttp.client_exceptions import ClientResponseError
|
||||
from injector import inject, singleton
|
||||
|
||||
from ..model import Coordinator, Backup
|
||||
from ..config import Config, Setting
|
||||
from ..util import GlobalInfo, Backoff, Estimator
|
||||
from .harequests import HaRequests
|
||||
from ..time import Time
|
||||
from ..worker import Worker
|
||||
from ..const import SOURCE_HA, SOURCE_GOOGLE_DRIVE
|
||||
from ..logger import getLogger
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
NOTIFICATION_TITLE = "Home Assistant Google Drive Backup is Having Trouble"
|
||||
NOTIFICATION_DESC_LINK = "The add-on is having trouble making backups and needs attention. Please visit the add-on [status page]({0}) for details."
|
||||
NOTIFICATION_DESC_STATIC = "The add-on is having trouble making backups and needs attention. Please visit the add-on status page for details."
|
||||
|
||||
MAX_BACKOFF = 60 * 5 # 5 minutes
|
||||
FIRST_BACKOFF = 60 # 1 minute
|
||||
|
||||
# Wait 5 minutes before logging
|
||||
NOTIFY_DELAY = 60 * 5 # 5 minute
|
||||
|
||||
OLD_BACKUP_ENTITY_NAME = "sensor.snapshot_backup"
|
||||
BACKUP_ENTITY_NAME = "sensor.backup_state"
|
||||
|
||||
REASSURING_MESSAGE = "Unable to reach Home Assistant (HTTP {0}). This is normal if Home Assistant is restarting. You will probably see some errors in the supervisor logs until it comes back online."
|
||||
|
||||
|
||||
@singleton
|
||||
class HaUpdater(Worker):
|
||||
@inject
|
||||
def __init__(self, requests: HaRequests, coordinator: Coordinator, config: Config, time: Time, global_info: GlobalInfo):
|
||||
self._config = config
|
||||
super().__init__("Sensor Updater", self.update, time, self.getInterval)
|
||||
self._time = time
|
||||
self._coordinator = coordinator
|
||||
self._requests: HaRequests = requests
|
||||
self._info = global_info
|
||||
self._notified = False
|
||||
self._backoff = Backoff(max=MAX_BACKOFF, base=FIRST_BACKOFF)
|
||||
self._first_error = None
|
||||
self._trigger_once = False
|
||||
|
||||
self._last_backup_update = None
|
||||
self.last_backup_update_time = time.now() - timedelta(days=1)
|
||||
self._config.subscribe(self.config_updated)
|
||||
self._last_interval = self.getInterval()
|
||||
|
||||
def config_updated(self):
|
||||
if self._last_interval != self.getInterval():
|
||||
self._wait_event.set()
|
||||
self._last_interval = self.getInterval()
|
||||
|
||||
def getInterval(self):
|
||||
return self._config.get(Setting.HA_REPORTING_INTERVAL_SECONDS)
|
||||
|
||||
async def update(self):
|
||||
try:
|
||||
if self._config.get(Setting.ENABLE_BACKUP_STALE_SENSOR):
|
||||
await self._requests.updateBackupStaleSensor('on' if self._stale() else 'off')
|
||||
if self._config.get(Setting.ENABLE_BACKUP_STATE_SENSOR):
|
||||
await self._maybeSendBackupUpdate()
|
||||
if self._config.get(Setting.NOTIFY_FOR_STALE_BACKUPS):
|
||||
if self._stale() and not self._notified:
|
||||
if self._info.url is None or len(self._info.url) == 0:
|
||||
message = NOTIFICATION_DESC_STATIC
|
||||
else:
|
||||
message = NOTIFICATION_DESC_LINK.format(self._info.url)
|
||||
await self._requests.sendNotification(NOTIFICATION_TITLE, message)
|
||||
self._notified = True
|
||||
elif not self._stale() and self._notified:
|
||||
await self._requests.dismissNotification()
|
||||
self._notified = False
|
||||
self._backoff.reset()
|
||||
self._first_error = None
|
||||
self._trigger_once = False
|
||||
except ClientResponseError as e:
|
||||
if self._first_error is None:
|
||||
self._first_error = self._time.now()
|
||||
if int(e.status / 100) == 5:
|
||||
if self._time.now() > self._first_error + timedelta(seconds=NOTIFY_DELAY):
|
||||
logger.error(
|
||||
"Unable to reach Home Assistant (HTTP {0}). This is normal if Home Assistant is restarting. You will probably see some errors in the supervisor logs until it comes back online.".format(e.status))
|
||||
else:
|
||||
logger.error("Trouble updating Home Assistant sensors.")
|
||||
self._last_backup_update = None
|
||||
await self._time.sleepAsync(self._backoff.backoff(e))
|
||||
except Exception as e:
|
||||
self._last_backup_update = None
|
||||
logger.error("Trouble updating Home Assistant sensors.")
|
||||
logger.printException(e)
|
||||
await self._time.sleepAsync(self._backoff.backoff(e))
|
||||
|
||||
async def _maybeSendBackupUpdate(self):
|
||||
update = self._buildBackupUpdate()
|
||||
if self._trigger_once or update != self._last_backup_update or self._time.now() > self.last_backup_update_time + timedelta(hours=1):
|
||||
if self._config.get(Setting.CALL_BACKUP_SNAPSHOT):
|
||||
await self._requests.updateEntity(OLD_BACKUP_ENTITY_NAME, update)
|
||||
else:
|
||||
await self._requests.updateEntity(BACKUP_ENTITY_NAME, update)
|
||||
self._last_backup_update = update
|
||||
self.last_backup_update_time = self._time.now()
|
||||
|
||||
def _stale(self):
|
||||
if self._info._first_sync:
|
||||
return False
|
||||
if self._info._last_error:
|
||||
return self._time.now() > self._info._last_success + timedelta(seconds=self._config.get(Setting.BACKUP_STALE_SECONDS))
|
||||
else:
|
||||
next_backup = self._coordinator.nextBackupTime(include_pending=False)
|
||||
if not next_backup:
|
||||
# no backups are configured
|
||||
return False
|
||||
|
||||
# Determine if a lot of time has passed since the last backup "should" have been made.
|
||||
warn_after = next_backup + timedelta(seconds=self._config.get(Setting.LONG_TERM_STALE_BACKUP_SECONDS))
|
||||
return self._time.now() >= warn_after
|
||||
|
||||
def _state(self):
|
||||
if self._stale():
|
||||
return "error"
|
||||
else:
|
||||
return "waiting" if self._info._first_sync else "backed_up"
|
||||
|
||||
def triggerRefresh(self):
|
||||
self._trigger_once = True
|
||||
|
||||
def _buildBackupUpdate(self):
|
||||
backups = list(filter(lambda s: not s.ignore(), self._coordinator.backups()))
|
||||
last = "Never"
|
||||
if len(backups) > 0:
|
||||
last = max(backups, key=lambda s: s.date()).date().isoformat()
|
||||
|
||||
def makeBackupData(backup: Backup):
|
||||
return {
|
||||
"name": backup.name(),
|
||||
"date": str(backup.date().isoformat()),
|
||||
"state": backup.status(),
|
||||
"size": backup.sizeString(),
|
||||
"slug": backup.slug()
|
||||
}
|
||||
ha_backups = list(filter(lambda s: s.getSource(SOURCE_HA) is not None, backups))
|
||||
drive_backups = list(filter(lambda s: s.getSource(SOURCE_GOOGLE_DRIVE) is not None, backups))
|
||||
|
||||
last_uploaded = "Never"
|
||||
if len(drive_backups) > 0:
|
||||
last_uploaded = max(drive_backups, key=lambda s: s.date()).date().isoformat()
|
||||
if self._config.get(Setting.CALL_BACKUP_SNAPSHOT):
|
||||
return {
|
||||
"state": self._state(),
|
||||
"attributes": {
|
||||
"friendly_name": "Snapshot State",
|
||||
"last_snapshot": last, # type: ignore
|
||||
"snapshots_in_google_drive": len(drive_backups),
|
||||
"snapshots_in_hassio": len(ha_backups),
|
||||
"snapshots_in_home_assistant": len(ha_backups),
|
||||
"size_in_google_drive": Estimator.asSizeString(sum(map(lambda v: v.sizeInt(), drive_backups))),
|
||||
"size_in_home_assistant": Estimator.asSizeString(sum(map(lambda v: v.sizeInt(), ha_backups))),
|
||||
"snapshots": list(map(makeBackupData, backups))
|
||||
}
|
||||
}
|
||||
else:
|
||||
source_metrics = self._coordinator.buildBackupMetrics()
|
||||
next = self._coordinator.nextBackupTime()
|
||||
if next is not None:
|
||||
next = next.isoformat()
|
||||
attr = {
|
||||
"friendly_name": "Backup State",
|
||||
"last_backup": last, # type: ignore
|
||||
"next_backup": next,
|
||||
"last_uploaded": last_uploaded,
|
||||
"backups_in_google_drive": len(drive_backups),
|
||||
"backups_in_home_assistant": len(ha_backups),
|
||||
"size_in_google_drive": Estimator.asSizeString(sum(map(lambda v: v.sizeInt(), drive_backups))),
|
||||
"size_in_home_assistant": Estimator.asSizeString(sum(map(lambda v: v.sizeInt(), ha_backups))),
|
||||
"backups": list(map(makeBackupData, backups))
|
||||
}
|
||||
if SOURCE_GOOGLE_DRIVE in source_metrics and 'free_space' in source_metrics[SOURCE_GOOGLE_DRIVE]:
|
||||
attr["free_space_in_google_drive"] = source_metrics[SOURCE_GOOGLE_DRIVE]['free_space']
|
||||
else:
|
||||
attr["free_space_in_google_drive"] = ""
|
||||
return {
|
||||
"state": self._state(),
|
||||
"attributes": attr
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
import os
|
||||
|
||||
import yaml
|
||||
|
||||
from ..config import Config, Setting
|
||||
from ..exceptions import BackupPasswordKeyInvalid
|
||||
from ..logger import getLogger
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
|
||||
class Password():
|
||||
def __init__(self, config: Config):
|
||||
self.config = config
|
||||
|
||||
def resolve(self, password=None):
|
||||
if password is None:
|
||||
password = self.config.get(Setting.BACKUP_PASSWORD)
|
||||
if len(password) == 0:
|
||||
return None
|
||||
if password.startswith("!secret "):
|
||||
if not os.path.isfile(self.config.get(Setting.SECRETS_FILE_PATH)):
|
||||
raise BackupPasswordKeyInvalid()
|
||||
with open(self.config.get(Setting.SECRETS_FILE_PATH)) as f:
|
||||
secrets_yaml = yaml.load(f, Loader=yaml.SafeLoader)
|
||||
key = password[len("!secret "):]
|
||||
if key not in secrets_yaml:
|
||||
raise BackupPasswordKeyInvalid()
|
||||
return str(secrets_yaml[key])
|
||||
else:
|
||||
return password
|
||||
@@ -0,0 +1,217 @@
|
||||
import logging
|
||||
from logging import LogRecord, Formatter, ERROR
|
||||
from traceback import TracebackException
|
||||
from colorlog import ColoredFormatter
|
||||
from os.path import join, abspath
|
||||
|
||||
HISTORY_SIZE = 1000
|
||||
PATH_BASE = abspath(join(__file__, "..", ".."))
|
||||
|
||||
logging.addLevelName(5, "TRACE")
|
||||
logging.TRACE = 5
|
||||
|
||||
|
||||
class HistoryHandler(logging.Handler):
|
||||
def __init__(self):
|
||||
super(HistoryHandler, self).__init__()
|
||||
self.history = [None] * HISTORY_SIZE
|
||||
self.history_index = 0
|
||||
|
||||
def reset(self):
|
||||
self.history = [None] * HISTORY_SIZE
|
||||
self.history_index = 0
|
||||
|
||||
def emit(self, record: LogRecord):
|
||||
self.history[self.history_index % HISTORY_SIZE] = record
|
||||
self.history_index += 1
|
||||
|
||||
def getHistory(self, start=0, html=False):
|
||||
end = self.history_index
|
||||
if end - start >= HISTORY_SIZE:
|
||||
start = end - HISTORY_SIZE
|
||||
for x in range(start, end):
|
||||
item = self.history[x % HISTORY_SIZE]
|
||||
if html:
|
||||
if item.levelno == logging.WARN:
|
||||
style = "console-warning"
|
||||
elif item.levelno == logging.ERROR:
|
||||
style = "console-error"
|
||||
elif item.levelno == logging.DEBUG:
|
||||
style = "console-debug"
|
||||
elif item.levelno == logging.CRITICAL:
|
||||
style = "console-critical"
|
||||
elif item.levelno == logging.FATAL:
|
||||
style = "console-fatal"
|
||||
elif item.levelno == logging.WARNING:
|
||||
style = "console-warning"
|
||||
elif item.levelno == logging.TRACE:
|
||||
style = "console-trace"
|
||||
else:
|
||||
style = "console-default"
|
||||
line = "<span class='" + style + \
|
||||
"'>" + self.format(item) + "</span>"
|
||||
yield (x + 1, line)
|
||||
else:
|
||||
yield (x + 1, self.format(item))
|
||||
|
||||
def getLast(self) -> LogRecord:
|
||||
return self.history[(self.history_index - 1) % HISTORY_SIZE]
|
||||
|
||||
|
||||
CONSOLE = logging.StreamHandler()
|
||||
CONSOLE.setLevel(logging.INFO)
|
||||
formatter_color = ColoredFormatter(
|
||||
'%(log_color)s%(asctime)s %(levelname)s %(message)s%(reset)s',
|
||||
datefmt='%m-%d %H:%M:%S',
|
||||
reset=True,
|
||||
log_colors={
|
||||
"DEBUG": "cyan",
|
||||
"INFO": "green",
|
||||
"WARNING": "yellow",
|
||||
"ERROR": "red",
|
||||
"CRITICAL": "red",
|
||||
"TRACE": "white",
|
||||
},
|
||||
)
|
||||
CONSOLE.setFormatter(formatter_color)
|
||||
|
||||
HISTORY = HistoryHandler()
|
||||
HISTORY.setLevel(logging.DEBUG)
|
||||
HISTORY.setFormatter(Formatter('%(asctime)s %(levelname)s [%(name)s] %(message)s', '%m-%d %H:%M:%S'))
|
||||
|
||||
|
||||
class StandardLogger(logging.Logger):
|
||||
def __init__(self, name):
|
||||
super().__init__(name)
|
||||
self.setLevel(logging.TRACE)
|
||||
self.addHandler(CONSOLE)
|
||||
self.addHandler(HISTORY)
|
||||
|
||||
def trace(self, msg, *args, **kwargs):
|
||||
self.log(logging.TRACE, msg, *args, **kwargs)
|
||||
|
||||
def printException(self, ex: Exception, level=ERROR):
|
||||
self.log(level, self.formatException(ex))
|
||||
|
||||
def formatException(self, e: Exception) -> str:
|
||||
trace = None
|
||||
if (hasattr(e, "__traceback__")):
|
||||
trace = e.__traceback__
|
||||
tbe = TracebackException(type(e), e, trace, limit=None)
|
||||
lines = list(self._format(tbe))
|
||||
return '\n%s' % ''.join(lines)
|
||||
|
||||
def _format(self, tbe):
|
||||
if (tbe.__context__ is not None and not tbe.__suppress_context__):
|
||||
yield from self._format(tbe.__context__)
|
||||
yield "Whose handling caused:\n"
|
||||
is_addon, stack = self._formatStack(tbe)
|
||||
yield from stack
|
||||
yield from tbe.format_exception_only()
|
||||
|
||||
def _formatStack(self, tbe):
|
||||
_RECURSIVE_CUTOFF = 3
|
||||
result = []
|
||||
last_file = None
|
||||
last_line = None
|
||||
last_name = None
|
||||
count = 0
|
||||
is_addon = False
|
||||
buffer = []
|
||||
for frame in tbe.stack:
|
||||
line_internal = True
|
||||
if (last_file is None or last_file != frame.filename or last_line is None or last_line != frame.lineno or last_name is None or last_name != frame.name):
|
||||
if count > _RECURSIVE_CUTOFF:
|
||||
count -= _RECURSIVE_CUTOFF
|
||||
result.append(
|
||||
f' [Previous line repeated {count} more '
|
||||
f'time{"s" if count > 1 else ""}]\n'
|
||||
)
|
||||
last_file = frame.filename
|
||||
last_line = frame.lineno
|
||||
last_name = frame.name
|
||||
count = 0
|
||||
count += 1
|
||||
if count > _RECURSIVE_CUTOFF:
|
||||
continue
|
||||
fileName = frame.filename
|
||||
pos = fileName.rfind(PATH_BASE)
|
||||
if pos >= 0:
|
||||
is_addon = True
|
||||
line_internal = False
|
||||
fileName = "addon" + \
|
||||
fileName[pos + len(PATH_BASE):]
|
||||
|
||||
pos = fileName.rfind("site-packages")
|
||||
if pos > 0:
|
||||
fileName = fileName[pos - 1:]
|
||||
|
||||
pos = fileName.rfind("python3.7")
|
||||
if pos > 0:
|
||||
fileName = fileName[pos - 1:]
|
||||
pass
|
||||
line = ' {}:{} ({})\n'.format(fileName, frame.lineno, frame.name)
|
||||
if line_internal:
|
||||
buffer.append(line)
|
||||
else:
|
||||
result.extend(self._compressFrames(buffer))
|
||||
buffer = []
|
||||
result.append(line)
|
||||
if count > _RECURSIVE_CUTOFF:
|
||||
count -= _RECURSIVE_CUTOFF
|
||||
result.append(
|
||||
f' [Previous line repeated {count} more '
|
||||
f'time{"s" if count > 1 else ""}]\n'
|
||||
)
|
||||
result.extend(self._compressFrames(buffer))
|
||||
return is_addon, result
|
||||
|
||||
def overrideLevel(self, console, history):
|
||||
CONSOLE.setLevel(console)
|
||||
HISTORY.setLevel(history)
|
||||
|
||||
def _compressFrames(self, buffer):
|
||||
if len(buffer) > 1:
|
||||
yield buffer[0]
|
||||
if len(buffer) == 3:
|
||||
yield buffer[1]
|
||||
elif len(buffer) > 2:
|
||||
yield " [{} hidden frames]\n".format(len(buffer) - 2)
|
||||
yield buffer[len(buffer) - 1]
|
||||
elif len(buffer) > 0:
|
||||
yield buffer[len(buffer) - 1]
|
||||
pass
|
||||
|
||||
|
||||
def getLogger(name):
|
||||
return StandardLogger(name)
|
||||
|
||||
|
||||
def getHistory(index, html):
|
||||
return HISTORY.getHistory(index, html)
|
||||
|
||||
|
||||
def getLast() -> LogRecord:
|
||||
return HISTORY.getLast()
|
||||
|
||||
|
||||
def reset() -> None:
|
||||
return HISTORY.reset()
|
||||
|
||||
|
||||
class TraceLogger(StandardLogger):
|
||||
def __init__(self, name):
|
||||
super().__init__(name)
|
||||
self.setLevel(logging.TRACE)
|
||||
|
||||
def log(self, lvl, msg, *args, **kwargs):
|
||||
super().log(logging.TRACE, msg, *args, **kwargs)
|
||||
|
||||
def info(self, *args, **kwargs):
|
||||
super().log(logging.TRACE, *args, **kwargs)
|
||||
|
||||
def error(self, *args, **kwargs):
|
||||
super().log(logging.TRACE, *args, **kwargs)
|
||||
|
||||
def warn(self, *args, **kwargs):
|
||||
super().log(logging.TRACE, *args, **kwargs)
|
||||
@@ -0,0 +1,13 @@
|
||||
# flake8: noqa
|
||||
from .backupscheme import GenerationalScheme, OldestScheme, GenConfig, BackupScheme
|
||||
from .coordinator import Coordinator
|
||||
from .model import BackupSource, BackupDestination, Model
|
||||
from .syncer import Scyncer
|
||||
from .backups import AbstractBackup, Backup
|
||||
from .drivebackup import DriveBackup
|
||||
from .dummybackup import DummyBackup
|
||||
from .dummybackupsource import DummyBackupSource
|
||||
from .habackup import HABackup
|
||||
from .simulatedsource import SimulatedSource
|
||||
from .precache import Precache
|
||||
from .destinationprecache import DestinationPrecache
|
||||
@@ -0,0 +1,318 @@
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, Optional, Union
|
||||
from dateutil.tz import tzutc
|
||||
from ..util import Estimator
|
||||
|
||||
from ..const import SOURCE_GOOGLE_DRIVE, SOURCE_HA
|
||||
from ..logger import getLogger
|
||||
from ..config import CreateOptions
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
PROP_TYPE = "type"
|
||||
PROP_VERSION = "version"
|
||||
PROP_PROTECTED = "protected"
|
||||
PROP_RETAINED = "retained"
|
||||
PROP_NOTE = "note"
|
||||
|
||||
DRIVE_KEY_TEXT = "Google Drive's backup metadata"
|
||||
HA_KEY_TEXT = "Home Assistant's backup metadata"
|
||||
|
||||
|
||||
class AbstractBackup():
|
||||
def __init__(self, name: str, slug: str, source: str, date: str, size: int, version: str, backupType: str, protected: bool, note=None, retained: bool = False, uploadable: bool = False, details={}, pending=False):
|
||||
self._options = None
|
||||
self._name = name
|
||||
self._slug = slug
|
||||
self._source = source
|
||||
self._date = date
|
||||
self._size = size
|
||||
self._retained = retained
|
||||
self._uploadable = uploadable
|
||||
self._details = details
|
||||
self._version = version
|
||||
self._backupType = backupType
|
||||
self._protected = protected
|
||||
self._ignore = False
|
||||
self._note = note
|
||||
self._pending = pending
|
||||
|
||||
def isPending(self):
|
||||
return self._pending
|
||||
|
||||
def setOptions(self, options: CreateOptions):
|
||||
self._options = options
|
||||
|
||||
def getOptions(self) -> CreateOptions:
|
||||
return self._options
|
||||
|
||||
def name(self) -> str:
|
||||
return self._name
|
||||
|
||||
def slug(self) -> str:
|
||||
return self._slug
|
||||
|
||||
def size(self) -> int:
|
||||
return self._size
|
||||
|
||||
def note(self) -> Union[str, None]:
|
||||
return self._note
|
||||
|
||||
def sizeInt(self) -> int:
|
||||
try:
|
||||
return int(self.size())
|
||||
except ValueError:
|
||||
return 0
|
||||
|
||||
def date(self) -> datetime:
|
||||
return self._date
|
||||
|
||||
def source(self) -> str:
|
||||
return self._source
|
||||
|
||||
def retained(self) -> str:
|
||||
return self._retained
|
||||
|
||||
def version(self):
|
||||
return self._version
|
||||
|
||||
def backupType(self):
|
||||
return self._backupType
|
||||
|
||||
def protected(self):
|
||||
return self._protected
|
||||
|
||||
def setRetained(self, retained):
|
||||
self._retained = retained
|
||||
|
||||
def uploadable(self) -> bool:
|
||||
return self._uploadable
|
||||
|
||||
def considerForPurge(self) -> bool:
|
||||
return not self.retained()
|
||||
|
||||
def setUploadable(self, uploadable):
|
||||
self._uploadable = uploadable
|
||||
|
||||
def details(self):
|
||||
return self._details
|
||||
|
||||
def setNote(self, note: Union[str, None]):
|
||||
self._note = note
|
||||
|
||||
def status(self):
|
||||
return None
|
||||
|
||||
def madeByTheAddon(self):
|
||||
return True
|
||||
|
||||
def ignore(self):
|
||||
return self._ignore
|
||||
|
||||
def setIgnore(self, ignore):
|
||||
self._ignore = ignore
|
||||
|
||||
|
||||
class Backup(object):
|
||||
"""
|
||||
Represents a Home Assistant backup stored on Google Drive, locally in
|
||||
Home Assistant, or a pending backup we expect to see show up later
|
||||
"""
|
||||
|
||||
def __init__(self, backup: Optional[AbstractBackup] = None):
|
||||
self.sources: Dict[str, AbstractBackup] = {}
|
||||
self._purgeNext: Dict[str, bool] = {}
|
||||
self._options = None
|
||||
self._status_override = None
|
||||
self._status_override_args = None
|
||||
self._state_detail = None
|
||||
self._upload_source = None
|
||||
self._upload_source_name = None
|
||||
self._upload_fail_info = None
|
||||
if backup is not None:
|
||||
self.addSource(backup)
|
||||
|
||||
def setOptions(self, options):
|
||||
self._options = options
|
||||
|
||||
def getOptions(self):
|
||||
return self._options
|
||||
|
||||
def updatePurge(self, source: str, purge: bool):
|
||||
self._purgeNext[source] = purge
|
||||
|
||||
def addSource(self, backup: AbstractBackup):
|
||||
self.sources[backup.source()] = backup
|
||||
if backup.getOptions() and not self.getOptions():
|
||||
self.setOptions(backup.getOptions())
|
||||
|
||||
def getStatusDetail(self):
|
||||
return self._state_detail
|
||||
|
||||
def setStatusDetail(self, info):
|
||||
self._state_detail = info
|
||||
|
||||
def removeSource(self, source):
|
||||
if source in self.sources:
|
||||
del self.sources[source]
|
||||
if source in self._purgeNext:
|
||||
del self._purgeNext[source]
|
||||
|
||||
def getPurges(self):
|
||||
return self._purgeNext
|
||||
|
||||
def uploadInfo(self):
|
||||
if not self._upload_source:
|
||||
return {}
|
||||
elif self._upload_source.progress() == 100:
|
||||
return {}
|
||||
else:
|
||||
return {
|
||||
'progress': self._upload_source.progress()
|
||||
}
|
||||
|
||||
def getSource(self, source: str):
|
||||
return self.sources.get(source, None)
|
||||
|
||||
def name(self):
|
||||
for backup in self.sources.values():
|
||||
return backup.name()
|
||||
return "error"
|
||||
|
||||
def note(self):
|
||||
longest = None
|
||||
for backup in self.sources.values():
|
||||
if backup.note() is not None and (longest is None or len(backup.note()) > len(longest)):
|
||||
longest = backup.note()
|
||||
return longest
|
||||
|
||||
def slug(self) -> str:
|
||||
for backup in self.sources.values():
|
||||
return backup.slug()
|
||||
return "error"
|
||||
|
||||
def size(self) -> int:
|
||||
for backup in self.sources.values():
|
||||
return backup.size()
|
||||
return 0
|
||||
|
||||
def sizeInt(self) -> int:
|
||||
for backup in self.sources.values():
|
||||
return backup.sizeInt()
|
||||
return 0
|
||||
|
||||
def backupType(self) -> str:
|
||||
for backup in self.sources.values():
|
||||
return backup.backupType()
|
||||
return "error"
|
||||
|
||||
def version(self) -> Union[str, None]:
|
||||
for backup in self.sources.values():
|
||||
if backup.version() is not None:
|
||||
return backup.version()
|
||||
return None
|
||||
|
||||
def details(self):
|
||||
for backup in self.sources.values():
|
||||
if backup.details() is not None:
|
||||
return backup.details()
|
||||
return {}
|
||||
|
||||
def getUploadInfo(self, time):
|
||||
if self._upload_source_name is None:
|
||||
return None
|
||||
ret = {
|
||||
'name': self._upload_source_name
|
||||
}
|
||||
if self._upload_fail_info:
|
||||
ret['failure'] = self._upload_fail_info
|
||||
elif self._upload_source is not None:
|
||||
ret['progress'] = self._upload_source.progress()
|
||||
ret['speed'] = self._upload_source.speed(timedelta(seconds=20))
|
||||
ret['total'] = self._upload_source.position()
|
||||
ret['started'] = time.formatDelta(self._upload_source.startTime())
|
||||
return ret
|
||||
|
||||
def protected(self) -> bool:
|
||||
for backup in self.sources.values():
|
||||
return backup.protected()
|
||||
return False
|
||||
|
||||
def ignore(self) -> bool:
|
||||
for backup in self.sources.values():
|
||||
if not backup.ignore():
|
||||
return False
|
||||
return True
|
||||
|
||||
def date(self) -> datetime:
|
||||
for backup in self.sources.values():
|
||||
return backup.date()
|
||||
return datetime.now(tzutc())
|
||||
|
||||
def sizeString(self) -> str:
|
||||
size_string = self.size()
|
||||
if type(size_string) == str:
|
||||
return size_string
|
||||
return Estimator.asSizeString(size_string)
|
||||
|
||||
def status(self) -> str:
|
||||
# TODO: Drive Specific
|
||||
if self._status_override is not None:
|
||||
return self._status_override.format(*self._status_override_args)
|
||||
|
||||
for backup in self.sources.values():
|
||||
status = backup.status()
|
||||
if status:
|
||||
return status
|
||||
|
||||
inDrive = self.getSource(SOURCE_GOOGLE_DRIVE) is not None
|
||||
inHa = self.getSource(SOURCE_HA) is not None
|
||||
|
||||
if inDrive and inHa:
|
||||
return "Backed Up"
|
||||
if inDrive:
|
||||
return "Drive Only"
|
||||
if inHa:
|
||||
return "HA Only"
|
||||
return "Deleted"
|
||||
|
||||
def isDeleted(self) -> bool:
|
||||
return len(self.sources) == 0
|
||||
|
||||
def overrideStatus(self, format, *args) -> None:
|
||||
self._status_override = format
|
||||
self._status_override_args = args
|
||||
|
||||
def setUploadSource(self, source_name: str, source):
|
||||
self._upload_source = source
|
||||
self._upload_source_name = source_name
|
||||
self._upload_fail_info = None
|
||||
|
||||
def clearUploadSource(self):
|
||||
self._upload_source = None
|
||||
self._upload_source_name = None
|
||||
self._upload_fail_info = None
|
||||
|
||||
def uploadFailure(self, info):
|
||||
self._upload_source = None
|
||||
self._upload_fail_info = info
|
||||
|
||||
def clearStatus(self):
|
||||
self._status_override = None
|
||||
self._status_override_args = None
|
||||
|
||||
def isPending(self):
|
||||
for backup in self.sources.values():
|
||||
if backup.isPending():
|
||||
return True
|
||||
return False
|
||||
|
||||
def __str__(self) -> str:
|
||||
return "<Slug: {0} {1} {2}>".format(self.slug(), " ".join(self.sources), self.date().isoformat())
|
||||
|
||||
def __format__(self, format_spec: str) -> str:
|
||||
return self.__str__()
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return self.__str__()
|
||||
@@ -0,0 +1,236 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from calendar import monthrange
|
||||
from datetime import datetime, timedelta, date
|
||||
from typing import List, Optional, Sequence, Set, Tuple, Any, Union
|
||||
|
||||
from .backups import Backup
|
||||
from backup.util import RangeLookup
|
||||
from ..time import Time
|
||||
from ..config import GenConfig
|
||||
from ..logger import getLogger
|
||||
logger = getLogger(__name__)
|
||||
|
||||
|
||||
class BackupScheme(ABC):
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def getOldest(self, backups: Sequence[Backup]) -> Tuple[str, Optional[Backup]]:
|
||||
pass
|
||||
|
||||
def handleNaming(self, backups: Sequence[Backup]) -> None:
|
||||
for backup in backups:
|
||||
backup.setStatusDetail(None)
|
||||
|
||||
|
||||
class DeleteAfterUploadScheme(BackupScheme):
|
||||
def __init__(self, source: str, destinations: List[str]):
|
||||
self.source = source
|
||||
self.destinations = destinations
|
||||
|
||||
def getOldest(self, backups: List[Backup]):
|
||||
consider = []
|
||||
for backup in backups:
|
||||
uploaded = True
|
||||
if backup.getSource(self.source) is None:
|
||||
# No source, so ignore it
|
||||
uploaded = False
|
||||
for destination in self.destinations:
|
||||
if backup.getSource(destination) is None:
|
||||
# its not in destination, so ignore it
|
||||
uploaded = False
|
||||
if uploaded:
|
||||
consider.append(backup)
|
||||
|
||||
# Delete the oldest first
|
||||
return OldestScheme().getOldest(consider)
|
||||
|
||||
|
||||
class OldestScheme(BackupScheme):
|
||||
def __init__(self, count=0):
|
||||
self.count = count
|
||||
|
||||
def getOldest(self, backups: Sequence[Backup]) -> Tuple[Any, Union[Backup, None]]:
|
||||
if len(backups) <= self.count:
|
||||
return None, None
|
||||
return "default", min(backups, default=None, key=lambda s: s.date())
|
||||
|
||||
def handleNaming(self, backups: Sequence[Backup]) -> None:
|
||||
for backup in backups:
|
||||
backup.setStatusDetail(None)
|
||||
|
||||
|
||||
class Partition(object):
|
||||
def __init__(self, start: datetime, end: datetime, prefer: datetime, time: Time, details=None, delete_only: bool = False):
|
||||
self.start: datetime = start
|
||||
self.end: datetime = end
|
||||
self.prefer: datetime = prefer
|
||||
self.time = time
|
||||
self.details = details
|
||||
self.selected = None
|
||||
self._delete_only_partitions = delete_only
|
||||
|
||||
def select(self, backups: List[Backup]) -> Optional[Backup]:
|
||||
options = list(RangeLookup(backups, lambda s: s.date()).matches(self.start, self.end - timedelta(milliseconds=1)))
|
||||
|
||||
searcher = lambda s: self.day(s.date()) == self.day(self.prefer)
|
||||
|
||||
preferred = list(filter(searcher, options))
|
||||
if len(preferred) > 0:
|
||||
# If there is a backup on the "preferred" day, then use the latest backup on that day
|
||||
self.selected = max(preferred, default=None, key=Backup.date)
|
||||
else:
|
||||
# Otherwise, use the earliest backup over the valid period.
|
||||
self.selected = min(options, default=None, key=Backup.date)
|
||||
return self.selected
|
||||
|
||||
def delta(self) -> timedelta:
|
||||
return self.end - self.start
|
||||
|
||||
def day(self, date: datetime):
|
||||
# TODO: this conversion isn't time-zone safe, but is ok because we only use it to compare local day to local day.
|
||||
local = self.time.toLocal(date)
|
||||
return datetime(day=local.day, month=local.month, year=local.year)
|
||||
|
||||
# True if the partition exists only to determine why a snapshot is getting deleted.
|
||||
@property
|
||||
def is_delete_only(self):
|
||||
return self._delete_only_partitions
|
||||
|
||||
def __hash__(self):
|
||||
"""Overrides the default implementation"""
|
||||
return hash(tuple(sorted(self.__dict__.items())))
|
||||
|
||||
|
||||
class GenerationalScheme(BackupScheme):
|
||||
def __init__(self, time: Time, config: GenConfig, count=0):
|
||||
self.count = count
|
||||
self.time: Time = time
|
||||
self.config = config
|
||||
|
||||
def _buildPartitions(self, backups_input):
|
||||
backups: List[Backup] = list(backups_input)
|
||||
|
||||
# build the list of dates we should partition by
|
||||
day_of_week = 3
|
||||
weekday_lookup = {
|
||||
'mon': 0,
|
||||
'tue': 1,
|
||||
'wed': 2,
|
||||
'thu': 3,
|
||||
'fri': 4,
|
||||
'sat': 5,
|
||||
'sun': 6,
|
||||
}
|
||||
if self.config.day_of_week in weekday_lookup:
|
||||
day_of_week = weekday_lookup[self.config.day_of_week]
|
||||
|
||||
last = self.time.toLocal(backups[len(backups) - 1].date())
|
||||
lookups: List[Partition] = []
|
||||
currentDay = self.day(last)
|
||||
if self.config.days > 0:
|
||||
for x in range(0, self.config.days + 1):
|
||||
nextDay = self.day(currentDay, add_days=1)
|
||||
lookups.append(
|
||||
Partition(currentDay, nextDay, currentDay, self.time, "Day {0} of {1}".format(x + 1, self.config.days), delete_only=(x >= self.config.days)))
|
||||
currentDay = self.day(currentDay, add_days=-1)
|
||||
|
||||
if self.config.weeks > 0:
|
||||
for x in range(0, self.config.weeks + 1):
|
||||
# Start at the first monday preceeding the last backup
|
||||
start = self.time.local(last.year, last.month, last.day)
|
||||
start = self.day(start, add_days=-1 * start.weekday())
|
||||
|
||||
# Move back x weeks
|
||||
start = self.day(start, add_days=-7 * x)
|
||||
end = self.day(start, add_days=7)
|
||||
|
||||
# Only consider backups from that week after the start day
|
||||
# TODO: should this actually "prefer" the day of week but start on monday?
|
||||
start = self.day(start, add_days=day_of_week)
|
||||
lookups.append(Partition(start, end, start, self.time, "Week {0} of {1}".format(x + 1, self.config.weeks), delete_only=(x >= self.config.weeks)))
|
||||
|
||||
if self.config.months > 0:
|
||||
for x in range(0, self.config.months + 1):
|
||||
year_offset = int(x / 12)
|
||||
month_offset = int(x % 12)
|
||||
if last.month - month_offset < 1:
|
||||
year_offset = year_offset + 1
|
||||
month_offset = month_offset - 12
|
||||
start = self.time.local(
|
||||
last.year - year_offset, last.month - month_offset, 1)
|
||||
weekday, days = monthrange(start.year, start.month)
|
||||
end = start + timedelta(days=days)
|
||||
lookups.append(Partition(
|
||||
start, end, start + timedelta(days=self.config.day_of_month - 1), self.time,
|
||||
"{0} ({1} of {2} months)".format(start.strftime("%B"), x + 1, self.config.months), delete_only=(x >= self.config.months)))
|
||||
|
||||
if self.config.years > 0:
|
||||
for x in range(0, self.config.years + 1):
|
||||
start = self.time.local(last.year - x, 1, 1)
|
||||
end = self.time.local(last.year - x + 1, 1, 1)
|
||||
lookups.append(Partition(
|
||||
start, end, start + timedelta(days=self.config.day_of_year - 1), self.time,
|
||||
"{0} ({1} of {2} years)".format(start.strftime("%Y"), x + 1, self.config.years), delete_only=(x >= self.config.years)))
|
||||
|
||||
# Keep track of which backups are being saved for which time period.
|
||||
for lookup in lookups:
|
||||
lookup.select(backups)
|
||||
return lookups
|
||||
|
||||
def getOldest(self, backups: Sequence[Backup]):
|
||||
if len(backups) == 0:
|
||||
return None, None
|
||||
|
||||
sorted = list(backups)
|
||||
sorted.sort(key=lambda s: s.date())
|
||||
|
||||
partitions = self._buildPartitions(sorted)
|
||||
keepers: Set[Backup] = set()
|
||||
for part in partitions:
|
||||
if part.selected is not None and not part.is_delete_only:
|
||||
keepers.add(part.selected)
|
||||
|
||||
extras = []
|
||||
for backup in sorted:
|
||||
if backup not in keepers:
|
||||
extras.append(backup)
|
||||
|
||||
if self.config.aggressive and len(extras) > 0:
|
||||
match = min(filter(lambda p: p.selected == extras[0], partitions), key=Partition.delta, default=None)
|
||||
if match is not None:
|
||||
return match, extras[0]
|
||||
return "default", extras[0]
|
||||
|
||||
if len(sorted) <= self.count and not self.config.aggressive:
|
||||
return "default", None
|
||||
elif (self.config.aggressive or len(sorted) > self.count) and len(extras) > 0:
|
||||
return "default", min(extras, default=None, key=lambda s: s.date())
|
||||
elif len(sorted) > self.count:
|
||||
# no non-keep is invalid, so delete the oldest keeper
|
||||
return "default", min(keepers, default=None, key=lambda s: s.date())
|
||||
return None, None
|
||||
|
||||
def handleNaming(self, backups: Sequence[Backup]) -> None:
|
||||
sorted = list(backups)
|
||||
sorted.sort(key=lambda s: s.date())
|
||||
for backup in sorted:
|
||||
backup.setStatusDetail(None)
|
||||
# Ignored snapshots should have their label cleared in case
|
||||
# it was added previosuly, but should not get new labels
|
||||
unignored = list(filter(lambda s: not s.ignore(), sorted))
|
||||
|
||||
if len(unignored) == 0:
|
||||
return
|
||||
for part in self._buildPartitions(unignored):
|
||||
if part.selected is not None:
|
||||
if part.selected.getStatusDetail() is None:
|
||||
part.selected.setStatusDetail([])
|
||||
part.selected.getStatusDetail().append(part.details)
|
||||
|
||||
def day(self, utc_datetime: datetime, add_days=0):
|
||||
local = self.time.toLocal(utc_datetime)
|
||||
|
||||
local_date = date.fromordinal(date(local.year, local.month, local.day).toordinal() + add_days)
|
||||
return self.time.localize(datetime(local_date.year, local_date.month, local_date.day, 0, 0))
|
||||
@@ -0,0 +1,356 @@
|
||||
from asyncio import CancelledError, Task, create_task, wait, Event
|
||||
from datetime import timedelta
|
||||
from threading import Lock
|
||||
from typing import Dict, List
|
||||
|
||||
from injector import inject, singleton
|
||||
|
||||
from backup.config import Config, Setting, CreateOptions, DurationParser
|
||||
from backup.exceptions import (KnownError, LogicError, NoBackup, PleaseWait,
|
||||
UserCancelledError)
|
||||
from backup.util import GlobalInfo, Backoff, Estimator
|
||||
from backup.time import Time
|
||||
from backup.worker import Trigger
|
||||
from backup.logger import getLogger
|
||||
from backup.creds.creds import Creds
|
||||
from .precache import Precache
|
||||
from .model import BackupSource, Model
|
||||
from .backups import AbstractBackup, Backup, SOURCE_HA
|
||||
from random import Random
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
|
||||
@singleton
|
||||
class Coordinator(Trigger):
|
||||
@inject
|
||||
def __init__(self, model: Model, time: Time, config: Config, global_info: GlobalInfo, estimator: Estimator):
|
||||
super().__init__()
|
||||
self._model = model
|
||||
self._precache: Precache = None
|
||||
self._time = time
|
||||
self._config = config
|
||||
self._lock: Lock = Lock()
|
||||
self._global_info: GlobalInfo = global_info
|
||||
self._sources: Dict[str, BackupSource] = {
|
||||
self._model.source.name(): self._model.source,
|
||||
self._model.dest.name(): self._model.dest
|
||||
}
|
||||
self._backoff = Backoff(initial=0, base=10, max=config.get(Setting.MAX_BACKOFF_SECONDS))
|
||||
self._estimator = estimator
|
||||
self._busy = False
|
||||
self._sync_task: Task = None
|
||||
self._sync_start = Event()
|
||||
self._sync_wait = Event()
|
||||
self._sync_wait.set()
|
||||
self._random = Random()
|
||||
self._random.seed()
|
||||
self._next_sync_offset = self._random.random()
|
||||
self._global_info.triggerBackupCooldown(timedelta(minutes=self._config.get(Setting.BACKUP_STARTUP_DELAY_MINUTES)))
|
||||
self.trigger()
|
||||
|
||||
def saveCreds(self, creds: Creds):
|
||||
if not self._model.dest.enabled():
|
||||
# Since this is the first time saving credentials (eg the addon was just enabled). Hold off on
|
||||
# automatic backups for a few minutes to give the user a little while to figure out whats going on.
|
||||
self._global_info.triggerBackupCooldown(timedelta(minutes=self._config.get(Setting.BACKUP_STARTUP_DELAY_MINUTES)))
|
||||
|
||||
self._model.dest.saveCreds(creds)
|
||||
self._global_info.credsSaved()
|
||||
|
||||
def setPrecache(self, precache: Precache):
|
||||
self._precache = precache
|
||||
|
||||
def name(self):
|
||||
return "Coordinator"
|
||||
|
||||
def enabled(self) -> bool:
|
||||
return self._model.enabled()
|
||||
|
||||
def isWaitingForStartup(self):
|
||||
return self._model.waiting_for_startup
|
||||
|
||||
def ignoreStartupDelay(self):
|
||||
self._model.ignore_startup_delay = True
|
||||
|
||||
async def check(self) -> bool:
|
||||
if self._time.now() >= self.nextSyncAttempt():
|
||||
self.reset()
|
||||
return True
|
||||
else:
|
||||
return await super().check()
|
||||
|
||||
async def sync(self):
|
||||
await self._withSoftLock(lambda: self._sync_wrapper())
|
||||
|
||||
def isSyncing(self):
|
||||
task = self._sync_task
|
||||
return task is not None and not task.done()
|
||||
|
||||
def isWorkingThroughUpload(self):
|
||||
return self.isSyncing() and self._model.isWorkingThroughUpload()
|
||||
|
||||
async def waitForSyncToFinish(self):
|
||||
task = self._sync_task
|
||||
if task is not None:
|
||||
await task
|
||||
|
||||
async def cancel(self):
|
||||
task = self._sync_task
|
||||
if task is not None and not task.done():
|
||||
task.cancel()
|
||||
self.clearCaches()
|
||||
await wait([task])
|
||||
|
||||
def nextSyncAttempt(self):
|
||||
if self._global_info._last_error is not None:
|
||||
# we had an error last
|
||||
failure = self._global_info._last_failure_time
|
||||
if failure is None:
|
||||
return self._time.now() - timedelta(minutes=1)
|
||||
return failure + timedelta(seconds=self._backoff.peek())
|
||||
else:
|
||||
scheduled = self._global_info._last_success
|
||||
if scheduled is None:
|
||||
scheduled = self._time.now() - timedelta(minutes=1)
|
||||
else:
|
||||
scheduled += timedelta(seconds=self.nextSyncCheckOffset())
|
||||
next_backup = self.nextBackupTime()
|
||||
if next_backup is None:
|
||||
return scheduled
|
||||
else:
|
||||
return min(self.nextBackupTime(), scheduled)
|
||||
|
||||
def nextSyncCheckOffset(self):
|
||||
"""Determines how long we shoudl wait from the last check the refresh the cache of backups from Google Drive and Home Assistant"""
|
||||
# If we always sync MAX_SYNC_INTERVAL_SECONDS secodns after the last
|
||||
# check, then the addon in aggregate puts a really high strain on google
|
||||
# on every hour and the addon's auth servers need to be provisioned for
|
||||
# a big peak, which is epxensive. Instead we add some randomness to the time interval.
|
||||
randomness_max = self._config.get(Setting.MAX_SYNC_INTERVAL_SECONDS) * self._config.get(Setting.DEFAULT_SYNC_INTERVAL_VARIATION)
|
||||
non_randomness = self._config.get(Setting.MAX_SYNC_INTERVAL_SECONDS) - randomness_max
|
||||
|
||||
# The offset should be stable between syncs, which gets controlled by updating _next_sync_offset on each good sync
|
||||
return self._next_sync_offset * randomness_max + non_randomness
|
||||
|
||||
def nextBackupTime(self, include_pending=True):
|
||||
return self._buildModel().nextBackup(self._time.now(), include_pending)
|
||||
|
||||
def buildBackupMetrics(self):
|
||||
info = {}
|
||||
for source in self._sources:
|
||||
source_class = self._sources[source]
|
||||
source_info = {
|
||||
'backups': 0,
|
||||
'retained': 0,
|
||||
'deletable': 0,
|
||||
'name': source,
|
||||
'title': source_class.title(),
|
||||
'latest': None,
|
||||
'max': source_class.maxCount(),
|
||||
'enabled': source_class.enabled(),
|
||||
'icon': source_class.icon(),
|
||||
'ignored': 0,
|
||||
'detail': source_class.detail()
|
||||
}
|
||||
size = 0
|
||||
ignored_size = 0
|
||||
latest = None
|
||||
for backup in self.backups():
|
||||
data: AbstractBackup = backup.getSource(source)
|
||||
if data is None:
|
||||
continue
|
||||
if data.ignore() and backup.ignore():
|
||||
source_info['ignored'] += 1
|
||||
if backup.ignore():
|
||||
ignored_size += backup.size()
|
||||
continue
|
||||
source_info['backups'] += 1
|
||||
if data.retained():
|
||||
source_info['retained'] += 1
|
||||
else:
|
||||
source_info['deletable'] += 1
|
||||
if latest is None or data.date() > latest:
|
||||
latest = data.date()
|
||||
size += int(data.sizeInt())
|
||||
if latest is not None:
|
||||
source_info['latest'] = self._time.asRfc3339String(latest)
|
||||
source_info['size'] = Estimator.asSizeString(size)
|
||||
source_info['ignored_size'] = Estimator.asSizeString(ignored_size)
|
||||
free_space = source_class.freeSpace()
|
||||
if free_space is not None and source_class.needsSpaceCheck:
|
||||
source_info['free_space'] = Estimator.asSizeString(free_space)
|
||||
info[source] = source_info
|
||||
return info
|
||||
|
||||
async def _sync_wrapper(self):
|
||||
self._sync_task = create_task(
|
||||
self._sync(), name="Internal sync worker")
|
||||
await wait([self._sync_task])
|
||||
|
||||
async def _sync(self):
|
||||
try:
|
||||
self._sync_start.set()
|
||||
await self._sync_wait.wait()
|
||||
logger.info("Syncing Backups")
|
||||
self._global_info.sync()
|
||||
self._estimator.refresh()
|
||||
await self._buildModel().sync(self._time.now())
|
||||
self._next_sync_offset = self._random.random()
|
||||
self._global_info.success()
|
||||
self._backoff.reset()
|
||||
self._global_info.setSkipSpaceCheckOnce(False)
|
||||
except BaseException as e:
|
||||
self.handleError(e)
|
||||
finally:
|
||||
if self._precache:
|
||||
# Any sync should invalidate the precache regardless of the outcome
|
||||
# so the next sync uses fresh data
|
||||
self.clearCaches()
|
||||
self._updateFreshness()
|
||||
|
||||
def handleError(self, e):
|
||||
if isinstance(e, CancelledError):
|
||||
e = UserCancelledError()
|
||||
if isinstance(e, KnownError):
|
||||
known: KnownError = e
|
||||
logger.error(known.message())
|
||||
if known.retrySoon():
|
||||
self._backoff.backoff(e)
|
||||
else:
|
||||
self._backoff.maxOut()
|
||||
else:
|
||||
logger.printException(e)
|
||||
self._backoff.backoff(e)
|
||||
self._global_info.failed(e)
|
||||
|
||||
text = DurationParser().format(timedelta(seconds=self._backoff.peek()))
|
||||
logger.info("I'll try again in {0}".format(text))
|
||||
|
||||
def backups(self) -> List[Backup]:
|
||||
ret = list(self._model.backups.values())
|
||||
ret.sort(key=lambda s: s.date())
|
||||
return ret
|
||||
|
||||
async def uploadBackups(self, slug):
|
||||
await self._withSoftLock(lambda: self._uploadBackup(slug))
|
||||
|
||||
async def _uploadBackup(self, slug):
|
||||
self.clearCaches()
|
||||
backup = self._ensureBackup(self._model.dest.name(), slug)
|
||||
backup_dest = backup.getSource(self._model.dest.name())
|
||||
backup_source = backup.getSource(self._model.source.name())
|
||||
if backup_source:
|
||||
raise LogicError("This backup already exists in Home Assistant")
|
||||
if not backup_dest:
|
||||
# Unreachable?
|
||||
raise LogicError("This backup isn't in Google Drive")
|
||||
created = await self._model.source.save(backup, await self._model.dest.read(backup))
|
||||
backup.addSource(created)
|
||||
self._updateFreshness()
|
||||
|
||||
async def startBackup(self, options: CreateOptions):
|
||||
return await self._withSoftLock(lambda: self._startBackup(options))
|
||||
|
||||
async def _startBackup(self, options: CreateOptions):
|
||||
self.clearCaches()
|
||||
model = self._buildModel()
|
||||
self._estimator.refresh()
|
||||
if model.source.needsSpaceCheck:
|
||||
self._estimator.checkSpace(self.backups())
|
||||
created = await self._buildModel().source.create(options)
|
||||
backup = Backup(created)
|
||||
self._model.backups[backup.slug()] = backup
|
||||
self._updateFreshness()
|
||||
self._estimator.refresh()
|
||||
return backup
|
||||
|
||||
def getBackup(self, slug):
|
||||
return self._ensureBackup(None, slug)
|
||||
|
||||
async def download(self, slug):
|
||||
self.clearCaches()
|
||||
backup = self._ensureBackup(None, slug)
|
||||
for source in self._sources.values():
|
||||
if not source.enabled():
|
||||
continue
|
||||
if backup.getSource(source.name()):
|
||||
return await source.read(backup)
|
||||
raise NoBackup()
|
||||
|
||||
async def retain(self, sources: Dict[str, bool], slug: str):
|
||||
self.clearCaches()
|
||||
for source in sources:
|
||||
backup = self._ensureBackup(source, slug)
|
||||
await self._ensureSource(source).retain(backup, sources[source])
|
||||
self._updateFreshness()
|
||||
|
||||
async def note(self, note: str, slug: str):
|
||||
self.clearCaches()
|
||||
backup = self._ensureBackup(None, slug)
|
||||
for source in backup.sources.keys():
|
||||
await self._ensureSource(source).note(backup, note)
|
||||
|
||||
async def delete(self, sources, slug):
|
||||
await self._withSoftLock(lambda: self._delete(sources, slug))
|
||||
|
||||
async def ignore(self, slug: str, ignore: bool):
|
||||
await self._withSoftLock(lambda: self._ignore(slug, ignore))
|
||||
|
||||
async def _delete(self, sources, slug):
|
||||
self.clearCaches()
|
||||
for source in sources:
|
||||
backup = self._ensureBackup(source, slug)
|
||||
await self._ensureSource(source).delete(backup)
|
||||
if backup.isDeleted():
|
||||
del self._model.backups[slug]
|
||||
self._updateFreshness()
|
||||
|
||||
async def _ignore(self, slug: str, ignore: bool):
|
||||
self.clearCaches()
|
||||
backup = self._ensureBackup(SOURCE_HA, slug)
|
||||
await self._ensureSource(SOURCE_HA).ignore(backup, ignore)
|
||||
|
||||
def _ensureBackup(self, source: str = None, slug=None) -> Backup:
|
||||
backup = self._buildModel().backups.get(slug)
|
||||
if not backup:
|
||||
raise NoBackup()
|
||||
if not source:
|
||||
return backup
|
||||
if not source:
|
||||
return backup
|
||||
if not backup.getSource(source):
|
||||
raise NoBackup()
|
||||
return backup
|
||||
|
||||
def _ensureSource(self, source):
|
||||
ret = self._sources.get(source)
|
||||
if ret and ret.enabled():
|
||||
return ret
|
||||
raise LogicError()
|
||||
|
||||
def _buildModel(self) -> Model:
|
||||
self._model.reinitialize(self._precache)
|
||||
return self._model
|
||||
|
||||
def _updateFreshness(self):
|
||||
purges = self._buildModel().getNextPurges()
|
||||
for backup in self._model.backups.values():
|
||||
for source in purges:
|
||||
if backup.getSource(source):
|
||||
backup.updatePurge(source, backup == purges[source])
|
||||
|
||||
def clearCaches(self):
|
||||
if self._precache:
|
||||
self._precache.clear()
|
||||
|
||||
async def _withSoftLock(self, callable):
|
||||
with self._lock:
|
||||
if self._busy:
|
||||
raise PleaseWait()
|
||||
self._busy = True
|
||||
try:
|
||||
return await callable()
|
||||
finally:
|
||||
with self._lock:
|
||||
self._busy = False
|
||||
+81
@@ -0,0 +1,81 @@
|
||||
from .coordinator import Coordinator
|
||||
from backup.worker import Worker
|
||||
from injector import inject, singleton
|
||||
from backup.time import Time
|
||||
from backup.logger import getLogger
|
||||
from backup.config import Config, Setting
|
||||
from .model import BackupDestination
|
||||
from .precache import Precache
|
||||
from random import Random
|
||||
from datetime import datetime, timedelta
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict
|
||||
from logging import DEBUG
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CacheItem:
|
||||
"""Class for keeping track of an item in inventory."""
|
||||
valid_until: datetime
|
||||
data: Any
|
||||
|
||||
|
||||
@singleton
|
||||
class DestinationPrecache(Worker, Precache):
|
||||
@inject
|
||||
def __init__(self, coord: Coordinator, time: Time, dest: BackupDestination, config: Config):
|
||||
super().__init__("Traffic Smoothing Cache", self.checkForSmoothing, time, 60)
|
||||
self._config = config
|
||||
self._coord = coord
|
||||
self._dest = dest
|
||||
self._offset = Random().random()
|
||||
self._cache: Dict[str, CacheItem] = {}
|
||||
self._last_error: datetime = None
|
||||
|
||||
async def checkForSmoothing(self):
|
||||
if self._config.get(Setting.CACHE_WARMUP_MAX_SECONDS) == 0:
|
||||
# disable cache warmup
|
||||
return
|
||||
try:
|
||||
self._coord.setPrecache(self)
|
||||
nextSync = self._coord.nextSyncAttempt()
|
||||
now = self._time.now()
|
||||
if nextSync <= now:
|
||||
# No reason to warm the cache if we should sync right now anyway
|
||||
return
|
||||
if self.cached(self._dest.name(), now):
|
||||
# A value is already cached, so don't do anything
|
||||
return
|
||||
if now >= self.getNextWarmDate():
|
||||
# Warm the cache
|
||||
logger.debug("Preemptively retrieving and caching info from the backup destination to avoid peak demand")
|
||||
data = await self._dest.get()
|
||||
validity = nextSync + timedelta(minutes=1)
|
||||
self._cache[self._dest.name()] = CacheItem(validity, data)
|
||||
self._offset = Random().random()
|
||||
except Exception as e:
|
||||
# Any error should make us avoid precaching for a solid day.
|
||||
logger.debug("Unable to precache data from backup destination")
|
||||
logger.printException(e, level=DEBUG)
|
||||
self._offset = Random().random()
|
||||
if self._config.get(Setting.CACHE_WARMUP_ERROR_TIMEOUT_SECONDS) != 0:
|
||||
self._last_error = self._time.now()
|
||||
|
||||
def getNextWarmDate(self):
|
||||
warm_date = self._coord.nextSyncAttempt() - timedelta(seconds=self._config.get(Setting.CACHE_WARMUP_MAX_SECONDS) * self._offset)
|
||||
if self._last_error:
|
||||
return max(warm_date, self._last_error + timedelta(self._config.get(Setting.CACHE_WARMUP_ERROR_TIMEOUT_SECONDS)))
|
||||
return warm_date
|
||||
|
||||
def cached(self, source: str, date: datetime) -> Any:
|
||||
cached = self._cache.get(source)
|
||||
if cached and cached.valid_until >= date:
|
||||
return cached.data
|
||||
return None
|
||||
|
||||
def clear(self):
|
||||
"""Clears any precached data"""
|
||||
self._cache = {}
|
||||
self._offset = Random().random()
|
||||
@@ -0,0 +1,74 @@
|
||||
from .backups import AbstractBackup
|
||||
from typing import Any, Dict
|
||||
|
||||
from ..const import SOURCE_GOOGLE_DRIVE, NECESSARY_PROP_KEY_SLUG, NECESSARY_PROP_KEY_DATE, NECESSARY_PROP_KEY_NAME, PROP_NOTE
|
||||
from ..exceptions import ensureKey
|
||||
from ..config import BoolValidator
|
||||
from ..time import Time
|
||||
from ..logger import getLogger
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
PROP_TYPE = "type"
|
||||
PROP_VERSION = "version"
|
||||
PROP_PROTECTED = "protected"
|
||||
PROP_RETAINED = "retained"
|
||||
DRIVE_KEY_TEXT = "Google Drive's backup metadata"
|
||||
|
||||
|
||||
class DriveBackup(AbstractBackup):
|
||||
|
||||
"""
|
||||
Represents a Home Assistant backup stored on Google Drive
|
||||
"""
|
||||
|
||||
def __init__(self, data: Dict[Any, Any]):
|
||||
props = ensureKey('appProperties', data, DRIVE_KEY_TEXT)
|
||||
retained = BoolValidator.strToBool(props.get(PROP_RETAINED, "False"))
|
||||
if NECESSARY_PROP_KEY_NAME in props:
|
||||
backup_name = ensureKey(NECESSARY_PROP_KEY_NAME, props, DRIVE_KEY_TEXT)
|
||||
else:
|
||||
backup_name = data['name'].replace(".tar", "")
|
||||
super().__init__(
|
||||
name=backup_name,
|
||||
slug=ensureKey(NECESSARY_PROP_KEY_SLUG, props, DRIVE_KEY_TEXT),
|
||||
date=Time.parse(
|
||||
ensureKey(NECESSARY_PROP_KEY_DATE, props, DRIVE_KEY_TEXT)),
|
||||
size=int(ensureKey("size", data, DRIVE_KEY_TEXT)),
|
||||
source=SOURCE_GOOGLE_DRIVE,
|
||||
backupType=props.get(PROP_TYPE, "?"),
|
||||
version=props.get(PROP_VERSION, None),
|
||||
protected=BoolValidator.strToBool(props.get(PROP_PROTECTED, "?")),
|
||||
retained=retained,
|
||||
uploadable=False,
|
||||
details=None,
|
||||
note=props.get(PROP_NOTE, None),
|
||||
pending=False)
|
||||
self._drive_data = data
|
||||
self._id = ensureKey('id', data, DRIVE_KEY_TEXT)
|
||||
|
||||
def id(self) -> str:
|
||||
return self._id
|
||||
|
||||
def canDeleteDirectly(self) -> str:
|
||||
caps = self._drive_data.get("capabilities", {})
|
||||
if caps.get('canDelete', False):
|
||||
return True
|
||||
|
||||
# check if the item is in a shared drive
|
||||
sharedId = self._drive_data.get("driveId")
|
||||
if sharedId and len(sharedId) > 0 and caps.get("canTrash", False):
|
||||
# Its in a shared drive and trashable, so trash won't exhaust quota
|
||||
return False
|
||||
|
||||
# We aren't certain we can trash or delete, so just make a try at deleting.
|
||||
return True
|
||||
|
||||
def __str__(self) -> str:
|
||||
return "<Drive: {0} Name: {1} Id: {2}>".format(self.slug(), self.name(), self.id())
|
||||
|
||||
def __format__(self, format_spec: str) -> str:
|
||||
return self.__str__()
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return self.__str__()
|
||||
@@ -0,0 +1,26 @@
|
||||
from .backups import Backup
|
||||
from .dummybackupsource import DummyBackupSource
|
||||
from ..logger import getLogger
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
|
||||
class DummyBackup(Backup):
|
||||
def __init__(self, name, date, source, slug, size=0, ignore=False, note=None):
|
||||
super().__init__(None)
|
||||
self._size = size
|
||||
self._ignore = ignore
|
||||
self._note = note
|
||||
self.addSource(DummyBackupSource(name, date, source, slug))
|
||||
|
||||
def size(self):
|
||||
return self._size
|
||||
|
||||
def ignore(self):
|
||||
return self._ignore
|
||||
|
||||
def note(self):
|
||||
if self._note is not None:
|
||||
return self._note
|
||||
else:
|
||||
return super().note()
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
from .backups import AbstractBackup
|
||||
from ..logger import getLogger
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
|
||||
class DummyBackupSource(AbstractBackup):
|
||||
def __init__(self, name, date, source, slug, retain=False):
|
||||
super().__init__(
|
||||
name=name,
|
||||
slug=slug,
|
||||
date=date,
|
||||
size=0,
|
||||
source=source,
|
||||
backupType="dummy",
|
||||
version="dummy_version",
|
||||
protected=True,
|
||||
retained=retain,
|
||||
uploadable=True,
|
||||
details={})
|
||||
@@ -0,0 +1,72 @@
|
||||
from typing import Any, Dict
|
||||
|
||||
from backup.const import SOURCE_HA
|
||||
from backup.exceptions import ensureKey
|
||||
from backup.time import Time
|
||||
from .backups import AbstractBackup
|
||||
from backup.logger import getLogger
|
||||
from backup.util import DataCache, KEY_I_MADE_THIS, KEY_IGNORE, KEY_NOTE
|
||||
from backup.config import Config, Setting
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
HA_KEY_TEXT = "Home Assistant's backup metadata"
|
||||
|
||||
|
||||
class HABackup(AbstractBackup):
|
||||
"""
|
||||
Represents a Home Assistant backup stored locally in Home Assistant
|
||||
"""
|
||||
|
||||
def __init__(self, data: Dict[str, Any], data_cache: DataCache, config: Config, retained=False):
|
||||
super().__init__(
|
||||
name=ensureKey('name', data, HA_KEY_TEXT),
|
||||
slug=ensureKey('slug', data, HA_KEY_TEXT),
|
||||
date=Time.parse(ensureKey('date', data, HA_KEY_TEXT)),
|
||||
size=float(ensureKey("size", data, HA_KEY_TEXT)) * 1024 * 1024,
|
||||
source=SOURCE_HA,
|
||||
backupType=ensureKey('type', data, HA_KEY_TEXT),
|
||||
version=ensureKey('homeassistant', data, HA_KEY_TEXT),
|
||||
protected=ensureKey('protected', data, HA_KEY_TEXT),
|
||||
retained=retained,
|
||||
uploadable=True,
|
||||
details=data,
|
||||
pending=False)
|
||||
self._data_cache = data_cache
|
||||
self._config = config
|
||||
|
||||
def madeByTheAddon(self):
|
||||
return self._data_cache.backup(self.slug()).get(KEY_I_MADE_THIS, False)
|
||||
|
||||
def note(self):
|
||||
parent = super().note()
|
||||
if parent is None:
|
||||
return self._data_cache.backup(self.slug()).get(KEY_NOTE, None)
|
||||
else:
|
||||
return parent
|
||||
|
||||
def ignore(self):
|
||||
override = self._data_cache.backup(self.slug()).get(KEY_IGNORE, None)
|
||||
if override is not None:
|
||||
return override
|
||||
if self.madeByTheAddon():
|
||||
return False
|
||||
if self._config.get(Setting.IGNORE_OTHER_BACKUPS):
|
||||
return True
|
||||
archive_count = len(self.details().get("addons", [])) + len(self.details().get("folders", []))
|
||||
if self.details().get("homeassistant", None) is not None:
|
||||
# Supervisor backup query API doesn't quite match the create API, if the HA config folder
|
||||
# is present in a backup then the Home Assistant version is present in its details
|
||||
archive_count += 1
|
||||
if archive_count == 1 and self._config.get(Setting.IGNORE_UPGRADE_BACKUPS):
|
||||
return True
|
||||
return super().ignore()
|
||||
|
||||
def __str__(self) -> str:
|
||||
return "<HA: {0} Name: {1} {2}>".format(self.slug(), self.name(), self.date().isoformat())
|
||||
|
||||
def __format__(self, format_spec: str) -> str:
|
||||
return self.__str__()
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return self.__str__()
|
||||
@@ -0,0 +1,402 @@
|
||||
from datetime import datetime, timedelta, date
|
||||
from io import IOBase
|
||||
from typing import Dict, Generic, List, Optional, Tuple, TypeVar
|
||||
|
||||
from injector import inject, singleton
|
||||
|
||||
from .backupscheme import GenerationalScheme, OldestScheme, DeleteAfterUploadScheme
|
||||
from backup.config import Config, Setting, CreateOptions
|
||||
from backup.exceptions import DeleteMutlipleBackupsError, SimulatedError
|
||||
from backup.util import GlobalInfo, Estimator, DataCache
|
||||
from .backups import AbstractBackup, Backup
|
||||
from .dummybackup import DummyBackup
|
||||
from .precache import Precache
|
||||
from backup.time import Time
|
||||
from backup.worker import Trigger
|
||||
from backup.logger import getLogger
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
T = TypeVar('T')
|
||||
|
||||
|
||||
class BackupSource(Trigger, Generic[T]):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
pass
|
||||
|
||||
def name(self) -> str:
|
||||
return "Unnamed"
|
||||
|
||||
def title(self) -> str:
|
||||
return "Default"
|
||||
|
||||
def enabled(self) -> bool:
|
||||
return True
|
||||
|
||||
def needsConfiguration(self) -> bool:
|
||||
return not self.enabled()
|
||||
|
||||
def upload(self) -> bool:
|
||||
return True
|
||||
|
||||
def icon(self) -> str:
|
||||
return "sd_card"
|
||||
|
||||
def freeSpace(self):
|
||||
return None
|
||||
|
||||
@property
|
||||
def needsSpaceCheck(self):
|
||||
return True
|
||||
|
||||
async def create(self, options: CreateOptions) -> T:
|
||||
pass
|
||||
|
||||
async def get(self) -> Dict[str, T]:
|
||||
pass
|
||||
|
||||
async def delete(self, backup: T):
|
||||
pass
|
||||
|
||||
async def ignore(self, backup: T, ignore: bool):
|
||||
pass
|
||||
|
||||
async def save(self, backup: AbstractBackup, bytes: IOBase) -> T:
|
||||
pass
|
||||
|
||||
async def read(self, backup: T) -> IOBase:
|
||||
pass
|
||||
|
||||
async def retain(self, backup: T, retain: bool) -> None:
|
||||
pass
|
||||
|
||||
async def note(self, backup, note: str) -> None:
|
||||
pass
|
||||
|
||||
def maxCount(self) -> None:
|
||||
return 0
|
||||
|
||||
def postSync(self) -> None:
|
||||
return
|
||||
|
||||
def detail(self) -> str:
|
||||
return ""
|
||||
|
||||
def isDestination(self) -> bool:
|
||||
return False
|
||||
|
||||
# Gets called after reading state but before any changes are made
|
||||
# to check for additional errors.
|
||||
def checkBeforeChanges(self) -> None:
|
||||
pass
|
||||
|
||||
|
||||
class BackupDestination(BackupSource):
|
||||
def isWorking(self):
|
||||
return False
|
||||
|
||||
@property
|
||||
def might_be_oob_creds(self) -> bool:
|
||||
return False
|
||||
|
||||
def isDestination(self) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
@singleton
|
||||
class Model():
|
||||
@inject
|
||||
def __init__(self, config: Config, time: Time, source: BackupSource, dest: BackupDestination, info: GlobalInfo, estimator: Estimator, data_cache: DataCache):
|
||||
self.config: Config = config
|
||||
self.time = time
|
||||
self.precache: Precache = None
|
||||
self.source: BackupSource = source
|
||||
self.dest: BackupDestination = dest
|
||||
self.reinitialize()
|
||||
self.backups: Dict[str, Backup] = {}
|
||||
self.firstSync = True
|
||||
self.info = info
|
||||
self.simulate_error = None
|
||||
self.estimator = estimator
|
||||
self.waiting_for_startup = False
|
||||
self.ignore_startup_delay = False
|
||||
self._data_cache = data_cache
|
||||
|
||||
def enabled(self):
|
||||
if self.source.needsConfiguration():
|
||||
return False
|
||||
if self.dest.needsConfiguration():
|
||||
return False
|
||||
return True
|
||||
|
||||
def allSources(self):
|
||||
return [self.source, self.dest]
|
||||
|
||||
def reinitialize(self, precache: Precache = None):
|
||||
self.precache = precache
|
||||
self._time_of_day: Optional[Tuple[int, int]] = self._parseTimeOfDay()
|
||||
|
||||
# SOMEDAY: this should be cached in config and regenerated on config updates, not here
|
||||
self.generational_config = self.config.getGenerationalConfig()
|
||||
|
||||
def getTimeOfDay(self):
|
||||
return self._time_of_day
|
||||
|
||||
def _nextBackup(self, now: datetime, last_backup: Optional[datetime]) -> Optional[datetime]:
|
||||
timeofDay = self.getTimeOfDay()
|
||||
if self.config.get(Setting.DAYS_BETWEEN_BACKUPS) <= 0:
|
||||
next = None
|
||||
elif self.dest.needsConfiguration():
|
||||
next = None
|
||||
elif not last_backup:
|
||||
# this isn't the cleanest logic, but the idea here is that if there are no backups,
|
||||
# then the backups shoudl be made right when the addon starts up.
|
||||
next = self.info.start_time
|
||||
elif not timeofDay:
|
||||
next = last_backup + timedelta(days=self.config.get(Setting.DAYS_BETWEEN_BACKUPS))
|
||||
else:
|
||||
newest_local: datetime = self.time.toLocal(last_backup)
|
||||
time_that_day_local = self.time.localize(datetime(newest_local.year, newest_local.month, newest_local.day, timeofDay[0], timeofDay[1]))
|
||||
if newest_local < time_that_day_local:
|
||||
# Latest backup is before the backup time for that day
|
||||
next = self.time.toUtc(time_that_day_local)
|
||||
else:
|
||||
# return the next backup after the delta
|
||||
next_date = date.fromordinal(int(date(newest_local.year, newest_local.month, newest_local.day).toordinal() + self.config.get(Setting.DAYS_BETWEEN_BACKUPS)))
|
||||
next_datetime_local = self.time.localize(datetime(next_date.year, next_date.month, next_date.day, timeofDay[0], timeofDay[1]))
|
||||
next = self.time.toUtc(next_datetime_local)
|
||||
|
||||
if next is None:
|
||||
self.waiting_for_startup = False
|
||||
return None
|
||||
|
||||
# Don't backup X minutes after startup, since that can put an unreasonable amount of strain on
|
||||
# the system while booting up.
|
||||
cooldown_minimum = self.info.backupCooldownTime()
|
||||
if next <= now and now < cooldown_minimum and not self.ignore_startup_delay:
|
||||
self.waiting_for_startup = True
|
||||
return cooldown_minimum
|
||||
elif self.ignore_startup_delay:
|
||||
self.waiting_for_startup = False
|
||||
return next
|
||||
elif cooldown_minimum > next:
|
||||
self.waiting_for_startup = cooldown_minimum > now
|
||||
return cooldown_minimum
|
||||
else:
|
||||
self.waiting_for_startup = False
|
||||
return next
|
||||
|
||||
def nextBackup(self, now: datetime, include_pending=True):
|
||||
latest = max(filter(lambda s: not s.ignore() and (not s.isPending() or include_pending), self.backups.values()),
|
||||
default=None, key=lambda s: s.date())
|
||||
if latest:
|
||||
latest = latest.date()
|
||||
return self._nextBackup(now, latest)
|
||||
|
||||
async def sync(self, now: datetime):
|
||||
if self.simulate_error is not None:
|
||||
if self.simulate_error.startswith("test"):
|
||||
raise Exception(self.simulate_error)
|
||||
else:
|
||||
raise SimulatedError(self.simulate_error)
|
||||
await self._syncBackups([self.source, self.dest], now)
|
||||
|
||||
self.source.checkBeforeChanges()
|
||||
self.dest.checkBeforeChanges()
|
||||
|
||||
if not self.dest.needsConfiguration():
|
||||
if self.source.enabled():
|
||||
await self._purge(self.source)
|
||||
if self.dest.enabled():
|
||||
await self._purge(self.dest)
|
||||
|
||||
# Delete any "ignored" backups that have expired
|
||||
if (self.config.get(Setting.IGNORE_OTHER_BACKUPS) or self.config.get(Setting.IGNORE_UPGRADE_BACKUPS)) and self.config.get(Setting.DELETE_IGNORED_AFTER_DAYS) > 0:
|
||||
cutoff = now - timedelta(days=self.config.get(Setting.DELETE_IGNORED_AFTER_DAYS))
|
||||
delete = []
|
||||
for backup in self.backups.values():
|
||||
if backup.ignore() and backup.date() < cutoff:
|
||||
delete.append(backup)
|
||||
for backup in delete:
|
||||
await self.deleteBackup(backup, self.source)
|
||||
|
||||
self._handleBackupDetails()
|
||||
next_backup = self.nextBackup(now)
|
||||
if next_backup and now >= next_backup and self.source.enabled() and not self.dest.needsConfiguration():
|
||||
if self.config.get(Setting.DELETE_BEFORE_NEW_BACKUP):
|
||||
await self._purge(self.source, pre_purge=True)
|
||||
await self.createBackup(CreateOptions(now, self.config.get(Setting.BACKUP_NAME)))
|
||||
await self._purge(self.source)
|
||||
self._handleBackupDetails()
|
||||
|
||||
if self.dest.enabled() and self.dest.upload():
|
||||
# get the backups we should upload
|
||||
uploads = []
|
||||
for backup in self.backups.values():
|
||||
if backup.getSource(self.source.name()) is not None and backup.getSource(self.source.name()).uploadable() and backup.getSource(self.dest.name()) is None and not backup.ignore():
|
||||
uploads.append(backup)
|
||||
uploads.sort(key=lambda s: s.date())
|
||||
uploads.reverse()
|
||||
for upload in uploads:
|
||||
# only upload if doing so won't result in it being deleted next
|
||||
dummy = DummyBackup(
|
||||
"", upload.date(), self.dest.name(), "dummy_slug_name")
|
||||
proposed = list(self.backups.values())
|
||||
proposed.append(dummy)
|
||||
if self._nextPurge(self.dest, proposed)[1] != dummy:
|
||||
if self.config.get(Setting.DELETE_BEFORE_NEW_BACKUP):
|
||||
await self._purge(self.dest, pre_purge=True)
|
||||
upload.addSource(await self.dest.save(upload, await self.source.read(upload)))
|
||||
await self._purge(self.dest)
|
||||
self._handleBackupDetails()
|
||||
else:
|
||||
break
|
||||
if self.config.get(Setting.DELETE_AFTER_UPLOAD):
|
||||
await self._purge(self.source)
|
||||
self._handleBackupDetails()
|
||||
self.source.postSync()
|
||||
self.dest.postSync()
|
||||
self._data_cache.saveIfDirty()
|
||||
|
||||
def isWorkingThroughUpload(self):
|
||||
return self.dest.isWorking()
|
||||
|
||||
async def createBackup(self, options):
|
||||
if not self.source.enabled():
|
||||
return
|
||||
|
||||
self.estimator.refresh()
|
||||
if self.source.needsSpaceCheck:
|
||||
self.estimator.checkSpace(list(self.backups.values()))
|
||||
created = await self.source.create(options)
|
||||
backup = Backup(created)
|
||||
self.backups[backup.slug()] = backup
|
||||
|
||||
async def deleteBackup(self, backup, source):
|
||||
if not backup.getSource(source.name()):
|
||||
return
|
||||
slug = backup.slug()
|
||||
await source.delete(backup)
|
||||
backup.removeSource(source.name())
|
||||
if backup.isDeleted():
|
||||
del self.backups[slug]
|
||||
|
||||
def getNextPurges(self):
|
||||
purges = {}
|
||||
for source in [self.source, self.dest]:
|
||||
purges[source.name()] = self._nextPurge(
|
||||
source, self.backups.values(), findNext=True)[1]
|
||||
return purges
|
||||
|
||||
def _parseTimeOfDay(self) -> Optional[Tuple[int, int]]:
|
||||
from_config = self.config.get(Setting.BACKUP_TIME_OF_DAY)
|
||||
if len(from_config) == 0:
|
||||
return None
|
||||
parts = from_config.split(":")
|
||||
if len(parts) != 2:
|
||||
return None
|
||||
try:
|
||||
hour: int = int(parts[0])
|
||||
minute: int = int(parts[1])
|
||||
if hour < 0 or minute < 0 or hour > 23 or minute > 59:
|
||||
return None
|
||||
return (hour, minute)
|
||||
except ValueError:
|
||||
# Parse error
|
||||
return None
|
||||
|
||||
async def _syncBackups(self, sources: List[BackupSource], now: datetime):
|
||||
for source in sources:
|
||||
if source.enabled():
|
||||
# check if we have the results from this source precached
|
||||
from_source: Dict[str, AbstractBackup] = None
|
||||
if self.precache is not None:
|
||||
from_source = self.precache.cached(source.name(), now)
|
||||
if not from_source:
|
||||
from_source = await source.get()
|
||||
else:
|
||||
from_source: Dict[str, AbstractBackup] = {}
|
||||
for backup in from_source.values():
|
||||
if backup.slug() not in self.backups:
|
||||
self.backups[backup.slug()] = Backup(backup)
|
||||
else:
|
||||
self.backups[backup.slug()].addSource(backup)
|
||||
for backup in list(self.backups.values()):
|
||||
if backup.slug() not in from_source:
|
||||
slug = backup.slug()
|
||||
backup.removeSource(source.name())
|
||||
if backup.isDeleted():
|
||||
del self.backups[slug]
|
||||
self.firstSync = False
|
||||
|
||||
def _buildDeleteScheme(self, source, findNext=False):
|
||||
count = source.maxCount()
|
||||
if findNext:
|
||||
count -= 1
|
||||
if source == self.source and self.config.get(Setting.DELETE_AFTER_UPLOAD):
|
||||
return DeleteAfterUploadScheme(source.name(), [self.dest.name()])
|
||||
elif self.generational_config:
|
||||
return GenerationalScheme(
|
||||
self.time, self.generational_config, count=count)
|
||||
else:
|
||||
return OldestScheme(count=count)
|
||||
|
||||
def _buildNamingScheme(self):
|
||||
source = max(filter(BackupSource.enabled, self.allSources()), key=BackupSource.maxCount)
|
||||
return self._buildDeleteScheme(source)
|
||||
|
||||
def _handleBackupDetails(self):
|
||||
self._buildNamingScheme().handleNaming(self.backups.values())
|
||||
|
||||
def _nextPurge(self, source: BackupSource, backups, findNext=False):
|
||||
"""
|
||||
Given a list of backups, decides if one should be purged.
|
||||
"""
|
||||
if not source.enabled() or len(backups) == 0:
|
||||
return None, None
|
||||
if source.maxCount() == 0 and source.isDestination():
|
||||
# When maxCount is zero for a destination, we should never delete from it.
|
||||
return None, None
|
||||
if source.maxCount() == 0 and not self.config.get(Setting.DELETE_AFTER_UPLOAD):
|
||||
return None, None
|
||||
|
||||
scheme = self._buildDeleteScheme(source, findNext=findNext)
|
||||
consider_purging = []
|
||||
for backup in backups:
|
||||
source_backup = backup.getSource(source.name())
|
||||
if source_backup is not None and source_backup.considerForPurge() and not backup.ignore():
|
||||
consider_purging.append(backup)
|
||||
if len(consider_purging) == 0:
|
||||
return None, None
|
||||
return scheme.getOldest(consider_purging)
|
||||
|
||||
async def _purge(self, source: BackupSource, pre_purge=False):
|
||||
while True:
|
||||
purge = self._getPurgeList(source, pre_purge)
|
||||
|
||||
reasons = set(map(lambda p: p[1], purge))
|
||||
if len(purge) <= 0:
|
||||
return
|
||||
if len(purge) != len(reasons) and (self.config.get(Setting.CONFIRM_MULTIPLE_DELETES) and not self.info.isPermitMultipleDeletes()):
|
||||
raise DeleteMutlipleBackupsError(self._getPurgeStats())
|
||||
await self.deleteBackup(purge[0][0], source)
|
||||
|
||||
def _getPurgeStats(self):
|
||||
ret = {}
|
||||
for source in [self.source, self.dest]:
|
||||
ret[source.name()] = len(self._getPurgeList(source))
|
||||
return ret
|
||||
|
||||
def _getPurgeList(self, source: BackupSource, pre_purge=False):
|
||||
if not source.enabled():
|
||||
return []
|
||||
candidates = list(self.backups.values())
|
||||
purges = []
|
||||
while True:
|
||||
reason, next_purge = self._nextPurge(source, candidates, findNext=pre_purge)
|
||||
if next_purge is None:
|
||||
return purges
|
||||
else:
|
||||
purges.append((next_purge, reason))
|
||||
candidates.remove(next_purge)
|
||||
@@ -0,0 +1,15 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
class Precache(ABC):
|
||||
@abstractmethod
|
||||
def cached(self, source: str, date: datetime) -> Any:
|
||||
"""For a given source and datetime, returns valid precached results if they're available"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def clear(self):
|
||||
"""Clears any cached results stored"""
|
||||
pass
|
||||
+129
@@ -0,0 +1,129 @@
|
||||
from .model import CreateOptions, BackupDestination
|
||||
from .backups import Backup
|
||||
from .dummybackupsource import DummyBackupSource
|
||||
from typing import Dict
|
||||
from io import IOBase
|
||||
from ..ha import BackupName
|
||||
from ..logger import getLogger
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
|
||||
class SimulatedSource(BackupDestination):
|
||||
def __init__(self, name, is_destination=False):
|
||||
self._name = name
|
||||
self.current: Dict[str, DummyBackupSource] = {}
|
||||
self.saved = []
|
||||
self.deleted = []
|
||||
self.created = []
|
||||
self._enabled = True
|
||||
self._upload = True
|
||||
self.index = 0
|
||||
self.max = 0
|
||||
self.backup_name = BackupName()
|
||||
self.host_info = {}
|
||||
self.backup_type = "Full"
|
||||
self.working = False
|
||||
self.needConfig = None
|
||||
self.is_destination = is_destination
|
||||
|
||||
def isDestination(self):
|
||||
return self.is_destination
|
||||
|
||||
def setEnabled(self, value):
|
||||
self._enabled = value
|
||||
return self
|
||||
|
||||
def needsConfiguration(self) -> bool:
|
||||
if self.needConfig is not None:
|
||||
return self.needConfig
|
||||
return super().needsConfiguration()
|
||||
|
||||
def setNeedsConfiguration(self, value: bool):
|
||||
self.needConfig = value
|
||||
|
||||
def setUpload(self, value):
|
||||
self._upload = value
|
||||
return self
|
||||
|
||||
def upload(self):
|
||||
return self._upload
|
||||
|
||||
def setMax(self, count):
|
||||
self.max = count
|
||||
return self
|
||||
|
||||
def isWorking(self):
|
||||
return self.working
|
||||
|
||||
def setIsWorking(self, value):
|
||||
self.working = value
|
||||
|
||||
def maxCount(self) -> None:
|
||||
return self.max
|
||||
|
||||
def insert(self, name, date, slug=None, retain=False):
|
||||
if slug is None:
|
||||
slug = name
|
||||
new_backup = DummyBackupSource(
|
||||
name,
|
||||
date,
|
||||
self._name,
|
||||
slug)
|
||||
self.current[new_backup.slug()] = new_backup
|
||||
return new_backup
|
||||
|
||||
def name(self) -> str:
|
||||
return self._name
|
||||
|
||||
def enabled(self) -> bool:
|
||||
return self._enabled
|
||||
|
||||
def nameSetup(self, type, host_info):
|
||||
self.backup_type = type
|
||||
self.host_info = host_info
|
||||
|
||||
async def create(self, options: CreateOptions) -> DummyBackupSource:
|
||||
assert self.enabled
|
||||
new_backup = DummyBackupSource(
|
||||
self.backup_name.resolve(
|
||||
self.backup_type, options.name_template, options.when, self.host_info),
|
||||
options.when,
|
||||
self._name,
|
||||
"{0}slug{1}".format(self._name, self.index))
|
||||
self.index += 1
|
||||
self.current[new_backup.slug()] = new_backup
|
||||
self.created.append(new_backup)
|
||||
return new_backup
|
||||
|
||||
async def get(self) -> Dict[str, DummyBackupSource]:
|
||||
assert self.enabled
|
||||
return self.current
|
||||
|
||||
async def delete(self, backup: Backup):
|
||||
assert self.enabled
|
||||
assert backup.getSource(self._name) is not None
|
||||
assert backup.getSource(self._name).source() is self._name
|
||||
assert backup.slug() in self.current
|
||||
slug = backup.slug()
|
||||
self.deleted.append(backup.getSource(self._name))
|
||||
backup.removeSource(self._name)
|
||||
del self.current[slug]
|
||||
|
||||
async def save(self, backup: Backup, bytes: IOBase = None) -> DummyBackupSource:
|
||||
assert self.enabled
|
||||
assert backup.slug() not in self.current
|
||||
new_backup = DummyBackupSource(
|
||||
backup.name(), backup.date(), self._name, backup.slug())
|
||||
backup.addSource(new_backup)
|
||||
self.current[new_backup.slug()] = new_backup
|
||||
self.saved.append(new_backup)
|
||||
return new_backup
|
||||
|
||||
async def read(self, backup: DummyBackupSource) -> IOBase:
|
||||
assert self.enabled
|
||||
return None
|
||||
|
||||
async def retain(self, backup: DummyBackupSource, retain: bool) -> None:
|
||||
assert self.enabled
|
||||
backup.getSource(self.name()).setRetained(retain)
|
||||
@@ -0,0 +1,36 @@
|
||||
from typing import List
|
||||
|
||||
from injector import inject, singleton
|
||||
|
||||
from .coordinator import Coordinator
|
||||
from backup.time import Time
|
||||
from backup.worker import Worker, Trigger
|
||||
from backup.logger import getLogger
|
||||
from backup.exceptions import PleaseWait
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
|
||||
@singleton
|
||||
class Scyncer(Worker):
|
||||
@inject
|
||||
def __init__(self, time: Time, coord: Coordinator, triggers: List[Trigger]):
|
||||
super().__init__("Sync Worker", self.checkforSync, time, 0.5)
|
||||
self.coord = coord
|
||||
self.triggers: List[Trigger] = triggers
|
||||
self._time = time
|
||||
|
||||
async def checkforSync(self):
|
||||
try:
|
||||
doSync = False
|
||||
for trigger in self.triggers:
|
||||
if await trigger.check():
|
||||
logger.debug("Sync requested by " + str(trigger.name()))
|
||||
doSync = True
|
||||
if doSync:
|
||||
while self.coord.isSyncing():
|
||||
await self._time.sleepAsync(3)
|
||||
await self.coord.sync()
|
||||
except PleaseWait:
|
||||
# Ignore this, since it means a sync already started (unavilable race condition)
|
||||
pass
|
||||
@@ -0,0 +1,87 @@
|
||||
import socket
|
||||
import sys
|
||||
import aiohttp
|
||||
import os
|
||||
from aiohttp import ClientSession
|
||||
from injector import Module, provider, singleton, multiprovider
|
||||
from typing import List
|
||||
|
||||
from backup.config import Config, Startable, Setting
|
||||
from backup.drive import DriveSource
|
||||
from backup.ha import HaSource, HaUpdater, AddonStopper
|
||||
from backup.model import BackupDestination, BackupSource, Scyncer
|
||||
from backup.util import Resolver
|
||||
from backup.model import Coordinator, Precache, DestinationPrecache
|
||||
from backup.worker import Trigger
|
||||
from backup.watcher import Watcher
|
||||
from backup.ui import UiServer, Restarter
|
||||
from backup.logger import getLogger
|
||||
from backup.debug import DebugServer
|
||||
from .debugworker import DebugWorker
|
||||
from .tracing_session import TracingSession
|
||||
logger = getLogger(__name__)
|
||||
|
||||
|
||||
class BaseModule(Module):
|
||||
'''
|
||||
A module shared between tests and main
|
||||
'''
|
||||
def __init__(self, override_dns=True):
|
||||
self._override_dns = override_dns
|
||||
|
||||
@multiprovider
|
||||
@singleton
|
||||
def getTriggers(self, coord: Coordinator, ha: HaSource, drive: DriveSource, watcher: Watcher, server: UiServer) -> List[Trigger]:
|
||||
return [coord, ha, drive, watcher, server]
|
||||
|
||||
@provider
|
||||
@singleton
|
||||
def getDrive(self, drive: DriveSource) -> BackupDestination:
|
||||
return drive
|
||||
|
||||
@provider
|
||||
@singleton
|
||||
def getHa(self, ha: HaSource) -> BackupSource:
|
||||
return ha
|
||||
|
||||
@provider
|
||||
@singleton
|
||||
def getPrecache(self, cache: DestinationPrecache) -> Precache:
|
||||
return cache
|
||||
|
||||
@multiprovider
|
||||
@singleton
|
||||
def getStartables(self, debug_server: DebugServer, ha_updater: HaUpdater, debugger: DebugWorker, ha_source: HaSource,
|
||||
server: UiServer, restarter: Restarter, syncer: Scyncer, watcher: Watcher, stopper: AddonStopper, precache: Precache) -> List[Startable]:
|
||||
# Order here matters, since its the order in which components of the addon are initialized.
|
||||
return [debug_server, ha_updater, debugger, ha_source, server, restarter, syncer, watcher, stopper, precache]
|
||||
|
||||
@provider
|
||||
@singleton
|
||||
def getSession(self, resolver: Resolver, config: Config) -> ClientSession:
|
||||
conn = None
|
||||
if self._override_dns:
|
||||
conn = aiohttp.TCPConnector(resolver=resolver, family=socket.AF_INET)
|
||||
return TracingSession(config, connector=conn)
|
||||
|
||||
|
||||
class MainModule(Module):
|
||||
@provider
|
||||
@singleton
|
||||
def getConfig(self) -> Config:
|
||||
alt_config = None
|
||||
index = 1
|
||||
for arg in sys.argv[1:]:
|
||||
if arg == "--config":
|
||||
alt_config = sys.argv[index + 1]
|
||||
break
|
||||
index += 1
|
||||
|
||||
if alt_config:
|
||||
config = Config.withFileOverrides(alt_config)
|
||||
elif "PYTEST_CURRENT_TEST" in os.environ:
|
||||
config = Config()
|
||||
else:
|
||||
config = Config.fromFile(Setting.CONFIG_FILE_PATH.default())
|
||||
logger.overrideLevel(config.get(Setting.CONSOLE_LOG_LEVEL), config.get(Setting.LOG_LEVEL))
|
||||
return config
|
||||
@@ -0,0 +1,4 @@
|
||||
# flake8: noqa
|
||||
from .server import Server
|
||||
from .errorstore import ErrorStore
|
||||
from .cloudlogger import CloudLogger
|
||||
@@ -0,0 +1,27 @@
|
||||
import aiorun
|
||||
from .server import Server
|
||||
from backup.config import Config
|
||||
from backup.module import BaseModule
|
||||
from injector import Injector
|
||||
from injector import provider, singleton
|
||||
|
||||
|
||||
class ServerModule(BaseModule):
|
||||
def __init__(self):
|
||||
super().__init__(override_dns=False)
|
||||
|
||||
@provider
|
||||
@singleton
|
||||
def getConfig(self) -> Config:
|
||||
return Config.fromEnvironment()
|
||||
|
||||
|
||||
async def main():
|
||||
module = ServerModule()
|
||||
injector = Injector(module)
|
||||
await injector.get(Server).start()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
print("Starting")
|
||||
aiorun.run(main())
|
||||
@@ -0,0 +1,28 @@
|
||||
import os
|
||||
import json
|
||||
from backup.logger import getLogger, StandardLogger
|
||||
from injector import inject, singleton
|
||||
from google.cloud import logging
|
||||
from google.auth.exceptions import DefaultCredentialsError
|
||||
|
||||
basic_logger = getLogger(__name__)
|
||||
|
||||
|
||||
@singleton
|
||||
class CloudLogger(StandardLogger):
|
||||
@inject
|
||||
def __init__(self):
|
||||
super().__init__(__name__)
|
||||
self.google_logger = None
|
||||
if os.environ.get('GOOGLE_APPLICATION_CREDENTIALS') is not None:
|
||||
try:
|
||||
google_logger_client = logging.Client()
|
||||
self.googler_logger = google_logger_client.logger("refresh_server")
|
||||
except DefaultCredentialsError:
|
||||
basic_logger.error("Unable to start Google Logger, no default credentials")
|
||||
|
||||
def log_struct(self, data):
|
||||
if self.google_logger is not None:
|
||||
self.google_logger.log_struct(data)
|
||||
else:
|
||||
basic_logger.info(json.dumps(data))
|
||||
@@ -0,0 +1,32 @@
|
||||
import firebase_admin
|
||||
from firebase_admin import credentials
|
||||
from firebase_admin import firestore
|
||||
from datetime import datetime
|
||||
from backup.config import Setting, Config
|
||||
from .cloudlogger import CloudLogger
|
||||
from injector import inject, singleton
|
||||
|
||||
|
||||
@singleton
|
||||
class ErrorStore():
|
||||
@inject
|
||||
def __init__(self, logger: CloudLogger, config: Config):
|
||||
try:
|
||||
cred = credentials.ApplicationDefault()
|
||||
firebase_admin.initialize_app(cred, {
|
||||
'projectId': config.get(Setting.SERVER_PROJECT_ID),
|
||||
})
|
||||
self.db = firestore.client()
|
||||
except Exception as e:
|
||||
logger.log_struct({
|
||||
"error": "unable to initialize firestore, errors will not be logged to firestore. If you are running this on a developer machine, this error is normal.",
|
||||
"exception": str(e)
|
||||
})
|
||||
self.db = None
|
||||
self.last_error = None
|
||||
|
||||
def store(self, error_data):
|
||||
if self.db is not None:
|
||||
doc_ref = self.db.collection(u'error_reports').document(error_data.get('client', "unknown") + "-" + datetime.now().isoformat())
|
||||
doc_ref.set(error_data)
|
||||
self.last_error = error_data
|
||||
@@ -0,0 +1,218 @@
|
||||
import json
|
||||
import aiohttp_jinja2
|
||||
import jinja2
|
||||
import base64
|
||||
from os.path import abspath, join
|
||||
from aiohttp.web import Application, json_response, Request, TCPSite, AppRunner, post, Response, static, get
|
||||
from aiohttp.client_exceptions import ClientResponseError, ClientConnectorError, ServerConnectionError, ServerDisconnectedError, ServerTimeoutError
|
||||
from aiohttp.web_exceptions import HTTPBadRequest, HTTPSeeOther
|
||||
from backup.creds import Exchanger
|
||||
from backup.config import Config, Setting, VERSION
|
||||
from backup.exceptions import GoogleCredentialsExpired, ensureKey, KnownError
|
||||
from injector import ClassAssistedBuilder, inject, singleton
|
||||
from .errorstore import ErrorStore
|
||||
from .cloudlogger import CloudLogger
|
||||
from yarl import URL
|
||||
from backup.config import Version
|
||||
from urllib.parse import unquote
|
||||
|
||||
NEW_AUTH_MINIMUM = Version(0, 101, 3)
|
||||
|
||||
|
||||
@singleton
|
||||
class Server():
|
||||
@inject
|
||||
def __init__(self,
|
||||
config: Config,
|
||||
exchanger_builder: ClassAssistedBuilder[Exchanger],
|
||||
logger: CloudLogger,
|
||||
error_store: ErrorStore):
|
||||
self.exchanger = exchanger_builder.build(
|
||||
client_id=config.get(Setting.DEFAULT_DRIVE_CLIENT_ID),
|
||||
client_secret=config.get(Setting.DEFAULT_DRIVE_CLIENT_SECRET),
|
||||
redirect=URL(config.get(Setting.AUTHORIZATION_HOST)).with_path("/drive/authorize"))
|
||||
self.logger = logger
|
||||
self.config = config
|
||||
self.error_store = error_store
|
||||
|
||||
def base_context(self, request: Request):
|
||||
return {
|
||||
'version': VERSION,
|
||||
'backgroundColor': request.query.get('bg', self.config.get(Setting.BACKGROUND_COLOR)),
|
||||
'accentColor': request.query.get('ac', self.config.get(Setting.ACCENT_COLOR)),
|
||||
'bmc_logo_path': "/static/" + VERSION + "/images/bmc.svg"
|
||||
}
|
||||
|
||||
async def authorize(self, request: Request):
|
||||
if 'redirectbacktoken' in request.query:
|
||||
version = Version.parse(request.query.get('version', "0"))
|
||||
token_url = request.query.get('redirectbacktoken')
|
||||
return_url = request.query.get('return', None)
|
||||
state = {
|
||||
'v': str(version),
|
||||
'token': token_url,
|
||||
'return': return_url,
|
||||
'bg': self.base_context(request).get('backgroundColor'),
|
||||
'ac': self.base_context(request).get('accentColor'),
|
||||
}
|
||||
# Someone is trying to authenticate with the add-on, direct them to the google auth url
|
||||
raise HTTPSeeOther(await self.exchanger.getAuthorizationUrl(json.dumps(state)))
|
||||
elif 'state' in request.query and 'code' in request.query:
|
||||
state = json.loads(unquote(request.query.get('state')))
|
||||
code = request.query.get('code')
|
||||
try:
|
||||
version = Version.parse(state["v"])
|
||||
creds = (await self.exchanger.exchange(code)).serialize(include_secret=False)
|
||||
|
||||
if version < NEW_AUTH_MINIMUM:
|
||||
# Redirect back to the addon, since this is the older addon
|
||||
url = URL(state['token']).with_query({'creds': json.dumps(creds)})
|
||||
raise HTTPSeeOther(url)
|
||||
|
||||
serialized_creds = str(base64.b64encode(json.dumps(creds).encode("utf-8")), "utf-8")
|
||||
url = URL(state['token']).with_query({
|
||||
'creds': serialized_creds,
|
||||
'host': state['return']})
|
||||
context = {
|
||||
**self.base_context(request),
|
||||
'redirect_url': str(url),
|
||||
'credentials_serialized': serialized_creds,
|
||||
}
|
||||
if 'bg' in state:
|
||||
context['backgroundColor'] = state['bg']
|
||||
if 'ac' in state:
|
||||
context['accentColor'] = state['ac']
|
||||
return aiohttp_jinja2.render_template(
|
||||
"authorize.jinja2",
|
||||
request,
|
||||
context)
|
||||
except Exception as e:
|
||||
if isinstance(e, HTTPSeeOther):
|
||||
# expected, pass this thorugh
|
||||
raise
|
||||
self.logError(request, e)
|
||||
content = "The server encountered an error while processing this request: " + str(e) + "<br/>"
|
||||
content += "Please <a href='https://github.com/sabeechen/hassio-google-drive-backup/issues'>file an issue</a> on Home Assistant Google Backup's GitHub page so I'm aware of this problem or attempt authorizing with Google Drive again."
|
||||
return Response(status=500, body=content)
|
||||
else:
|
||||
raise HTTPBadRequest()
|
||||
|
||||
async def error(self, request: Request):
|
||||
try:
|
||||
self.logReport(request, await request.json())
|
||||
except BaseException as e:
|
||||
self.logError(request, e)
|
||||
return Response()
|
||||
|
||||
async def refresh(self, request: Request):
|
||||
try:
|
||||
token = ensureKey('refresh_token', await request.json(), "the request payload")
|
||||
creds = self.exchanger.refreshCredentials(token)
|
||||
new_creds = await self.exchanger.refresh(creds)
|
||||
return json_response(new_creds.serialize(include_secret=False))
|
||||
except ClientResponseError as e:
|
||||
if e.status == 401:
|
||||
return json_response({
|
||||
"error": "expired"
|
||||
}, status=401)
|
||||
else:
|
||||
self.logError(request, e)
|
||||
return json_response({
|
||||
"error": "Google returned HTTP {}".format(e.status)
|
||||
}, status=503)
|
||||
except ClientConnectorError:
|
||||
return json_response({
|
||||
"error": "Couldn't connect to Google's servers"
|
||||
}, status=503)
|
||||
except ServerConnectionError:
|
||||
return json_response({
|
||||
"error": "Couldn't connect to Google's servers"
|
||||
}, status=503)
|
||||
except ServerDisconnectedError:
|
||||
return json_response({
|
||||
"error": "Couldn't connect to Google's servers"
|
||||
}, status=503)
|
||||
except ServerTimeoutError:
|
||||
return json_response({
|
||||
"error": "Google's servers timed out"
|
||||
}, status=503)
|
||||
except GoogleCredentialsExpired:
|
||||
return json_response({
|
||||
"error": "expired"
|
||||
}, status=401)
|
||||
except KnownError as e:
|
||||
return json_response({
|
||||
"error": e.message()
|
||||
}, status=503)
|
||||
except Exception as e:
|
||||
self.logError(request, e)
|
||||
return json_response({
|
||||
"error": str(e)
|
||||
}, status=500)
|
||||
|
||||
@aiohttp_jinja2.template('picker.jinja2')
|
||||
async def picker(self, request: Request):
|
||||
version = Version.parse(request.query.get('version', "0"))
|
||||
bg = request.query.get('bg', self.config.get(Setting.BACKGROUND_COLOR))
|
||||
ac = request.query.get('ac', self.config.get(Setting.ACCENT_COLOR))
|
||||
return {
|
||||
**self.base_context(request),
|
||||
"client_id": self.config.get(Setting.DEFAULT_DRIVE_CLIENT_ID),
|
||||
"developer_key": self.config.get(Setting.DRIVE_PICKER_API_KEY),
|
||||
"app_id": self.config.get(Setting.DEFAULT_DRIVE_CLIENT_ID).split("-")[0],
|
||||
'backgroundColor': bg,
|
||||
'accentColor': ac,
|
||||
"do_redirect": str(version < NEW_AUTH_MINIMUM).lower()
|
||||
}
|
||||
|
||||
@aiohttp_jinja2.template('server-index.jinja2')
|
||||
async def index(self, request: Request):
|
||||
return self.base_context(request)
|
||||
|
||||
async def health(self, request: Request):
|
||||
return json_response({
|
||||
'status': 'ok',
|
||||
'messages': []
|
||||
})
|
||||
|
||||
def buildApp(self, app):
|
||||
path = abspath(join(__file__, "..", "..", "static"))
|
||||
app.add_routes([
|
||||
static("/static/" + VERSION, path, append_version=True),
|
||||
static("/drive/static/" + VERSION, path, append_version=True),
|
||||
get("/drive/picker", self.picker),
|
||||
get("/", self.index),
|
||||
get("/drive/authorize", self.authorize),
|
||||
post("/drive/refresh", self.refresh),
|
||||
post("/logerror", self.error),
|
||||
get("/health", self.health)
|
||||
])
|
||||
aiohttp_jinja2.setup(app, loader=jinja2.FileSystemLoader(path))
|
||||
return app
|
||||
|
||||
async def start(self):
|
||||
runner = AppRunner(self.buildApp(Application()))
|
||||
await runner.setup()
|
||||
site = TCPSite(runner, "0.0.0.0", int(self.config.get(Setting.PORT)))
|
||||
await site.start()
|
||||
self.logger.info("Backup Auth Server Started")
|
||||
|
||||
def logError(self, request: Request, exception: Exception):
|
||||
data = self.getRequestInfo(request)
|
||||
data['exception'] = self.logger.formatException(exception)
|
||||
self.logger.log_struct(data)
|
||||
|
||||
def logReport(self, request, report):
|
||||
data = self.getRequestInfo(request)
|
||||
data['report'] = report
|
||||
self.logger.log_struct(data)
|
||||
self.error_store.store(data)
|
||||
|
||||
def getRequestInfo(self, request: Request):
|
||||
return {
|
||||
'client': request.headers.get('client', "unknown"),
|
||||
'version': request.headers.get('addon_version', "unknown"),
|
||||
'address': request.remote,
|
||||
'url': str(request.url),
|
||||
'length': request.content_length
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user