237 lines
9.4 KiB
Python
237 lines
9.4 KiB
Python
from abc import ABC, abstractmethod
|
|
from calendar import monthrange
|
|
from datetime import datetime, timedelta, date
|
|
from typing import List, Optional, Sequence, Set, Tuple, Any, Union
|
|
|
|
from .backups import Backup
|
|
from backup.util import RangeLookup
|
|
from ..time import Time
|
|
from ..config import GenConfig
|
|
from ..logger import getLogger
|
|
logger = getLogger(__name__)
|
|
|
|
|
|
class BackupScheme(ABC):
|
|
def __init__(self):
|
|
pass
|
|
|
|
@abstractmethod
|
|
def getOldest(self, backups: Sequence[Backup]) -> Tuple[str, Optional[Backup]]:
|
|
pass
|
|
|
|
def handleNaming(self, backups: Sequence[Backup]) -> None:
|
|
for backup in backups:
|
|
backup.setStatusDetail(None)
|
|
|
|
|
|
class DeleteAfterUploadScheme(BackupScheme):
|
|
def __init__(self, source: str, destinations: List[str]):
|
|
self.source = source
|
|
self.destinations = destinations
|
|
|
|
def getOldest(self, backups: List[Backup]):
|
|
consider = []
|
|
for backup in backups:
|
|
uploaded = True
|
|
if backup.getSource(self.source) is None:
|
|
# No source, so ignore it
|
|
uploaded = False
|
|
for destination in self.destinations:
|
|
if backup.getSource(destination) is None:
|
|
# its not in destination, so ignore it
|
|
uploaded = False
|
|
if uploaded:
|
|
consider.append(backup)
|
|
|
|
# Delete the oldest first
|
|
return OldestScheme().getOldest(consider)
|
|
|
|
|
|
class OldestScheme(BackupScheme):
|
|
def __init__(self, count=0):
|
|
self.count = count
|
|
|
|
def getOldest(self, backups: Sequence[Backup]) -> Tuple[Any, Union[Backup, None]]:
|
|
if len(backups) <= self.count:
|
|
return None, None
|
|
return "default", min(backups, default=None, key=lambda s: s.date())
|
|
|
|
def handleNaming(self, backups: Sequence[Backup]) -> None:
|
|
for backup in backups:
|
|
backup.setStatusDetail(None)
|
|
|
|
|
|
class Partition(object):
|
|
def __init__(self, start: datetime, end: datetime, prefer: datetime, time: Time, details=None, delete_only: bool = False):
|
|
self.start: datetime = start
|
|
self.end: datetime = end
|
|
self.prefer: datetime = prefer
|
|
self.time = time
|
|
self.details = details
|
|
self.selected = None
|
|
self._delete_only_partitions = delete_only
|
|
|
|
def select(self, backups: List[Backup]) -> Optional[Backup]:
|
|
options = list(RangeLookup(backups, lambda s: s.date()).matches(self.start, self.end - timedelta(milliseconds=1)))
|
|
|
|
searcher = lambda s: self.day(s.date()) == self.day(self.prefer)
|
|
|
|
preferred = list(filter(searcher, options))
|
|
if len(preferred) > 0:
|
|
# If there is a backup on the "preferred" day, then use the latest backup on that day
|
|
self.selected = max(preferred, default=None, key=Backup.date)
|
|
else:
|
|
# Otherwise, use the earliest backup over the valid period.
|
|
self.selected = min(options, default=None, key=Backup.date)
|
|
return self.selected
|
|
|
|
def delta(self) -> timedelta:
|
|
return self.end - self.start
|
|
|
|
def day(self, date: datetime):
|
|
# TODO: this conversion isn't time-zone safe, but is ok because we only use it to compare local day to local day.
|
|
local = self.time.toLocal(date)
|
|
return datetime(day=local.day, month=local.month, year=local.year)
|
|
|
|
# True if the partition exists only to determine why a snapshot is getting deleted.
|
|
@property
|
|
def is_delete_only(self):
|
|
return self._delete_only_partitions
|
|
|
|
def __hash__(self):
|
|
"""Overrides the default implementation"""
|
|
return hash(tuple(sorted(self.__dict__.items())))
|
|
|
|
|
|
class GenerationalScheme(BackupScheme):
|
|
def __init__(self, time: Time, config: GenConfig, count=0):
|
|
self.count = count
|
|
self.time: Time = time
|
|
self.config = config
|
|
|
|
def _buildPartitions(self, backups_input):
|
|
backups: List[Backup] = list(backups_input)
|
|
|
|
# build the list of dates we should partition by
|
|
day_of_week = 3
|
|
weekday_lookup = {
|
|
'mon': 0,
|
|
'tue': 1,
|
|
'wed': 2,
|
|
'thu': 3,
|
|
'fri': 4,
|
|
'sat': 5,
|
|
'sun': 6,
|
|
}
|
|
if self.config.day_of_week in weekday_lookup:
|
|
day_of_week = weekday_lookup[self.config.day_of_week]
|
|
|
|
last = self.time.toLocal(backups[len(backups) - 1].date())
|
|
lookups: List[Partition] = []
|
|
currentDay = self.day(last)
|
|
if self.config.days > 0:
|
|
for x in range(0, self.config.days + 1):
|
|
nextDay = self.day(currentDay, add_days=1)
|
|
lookups.append(
|
|
Partition(currentDay, nextDay, currentDay, self.time, "Day {0} of {1}".format(x + 1, self.config.days), delete_only=(x >= self.config.days)))
|
|
currentDay = self.day(currentDay, add_days=-1)
|
|
|
|
if self.config.weeks > 0:
|
|
for x in range(0, self.config.weeks + 1):
|
|
# Start at the first monday preceeding the last backup
|
|
start = self.time.local(last.year, last.month, last.day)
|
|
start = self.day(start, add_days=-1 * start.weekday())
|
|
|
|
# Move back x weeks
|
|
start = self.day(start, add_days=-7 * x)
|
|
end = self.day(start, add_days=7)
|
|
|
|
# Only consider backups from that week after the start day
|
|
# TODO: should this actually "prefer" the day of week but start on monday?
|
|
start = self.day(start, add_days=day_of_week)
|
|
lookups.append(Partition(start, end, start, self.time, "Week {0} of {1}".format(x + 1, self.config.weeks), delete_only=(x >= self.config.weeks)))
|
|
|
|
if self.config.months > 0:
|
|
for x in range(0, self.config.months + 1):
|
|
year_offset = int(x / 12)
|
|
month_offset = int(x % 12)
|
|
if last.month - month_offset < 1:
|
|
year_offset = year_offset + 1
|
|
month_offset = month_offset - 12
|
|
start = self.time.local(
|
|
last.year - year_offset, last.month - month_offset, 1)
|
|
weekday, days = monthrange(start.year, start.month)
|
|
end = start + timedelta(days=days)
|
|
lookups.append(Partition(
|
|
start, end, start + timedelta(days=self.config.day_of_month - 1), self.time,
|
|
"{0} ({1} of {2} months)".format(start.strftime("%B"), x + 1, self.config.months), delete_only=(x >= self.config.months)))
|
|
|
|
if self.config.years > 0:
|
|
for x in range(0, self.config.years + 1):
|
|
start = self.time.local(last.year - x, 1, 1)
|
|
end = self.time.local(last.year - x + 1, 1, 1)
|
|
lookups.append(Partition(
|
|
start, end, start + timedelta(days=self.config.day_of_year - 1), self.time,
|
|
"{0} ({1} of {2} years)".format(start.strftime("%Y"), x + 1, self.config.years), delete_only=(x >= self.config.years)))
|
|
|
|
# Keep track of which backups are being saved for which time period.
|
|
for lookup in lookups:
|
|
lookup.select(backups)
|
|
return lookups
|
|
|
|
def getOldest(self, backups: Sequence[Backup]):
|
|
if len(backups) == 0:
|
|
return None, None
|
|
|
|
sorted = list(backups)
|
|
sorted.sort(key=lambda s: s.date())
|
|
|
|
partitions = self._buildPartitions(sorted)
|
|
keepers: Set[Backup] = set()
|
|
for part in partitions:
|
|
if part.selected is not None and not part.is_delete_only:
|
|
keepers.add(part.selected)
|
|
|
|
extras = []
|
|
for backup in sorted:
|
|
if backup not in keepers:
|
|
extras.append(backup)
|
|
|
|
if self.config.aggressive and len(extras) > 0:
|
|
match = min(filter(lambda p: p.selected == extras[0], partitions), key=Partition.delta, default=None)
|
|
if match is not None:
|
|
return match, extras[0]
|
|
return "default", extras[0]
|
|
|
|
if len(sorted) <= self.count and not self.config.aggressive:
|
|
return "default", None
|
|
elif (self.config.aggressive or len(sorted) > self.count) and len(extras) > 0:
|
|
return "default", min(extras, default=None, key=lambda s: s.date())
|
|
elif len(sorted) > self.count:
|
|
# no non-keep is invalid, so delete the oldest keeper
|
|
return "default", min(keepers, default=None, key=lambda s: s.date())
|
|
return None, None
|
|
|
|
def handleNaming(self, backups: Sequence[Backup]) -> None:
|
|
sorted = list(backups)
|
|
sorted.sort(key=lambda s: s.date())
|
|
for backup in sorted:
|
|
backup.setStatusDetail(None)
|
|
# Ignored snapshots should have their label cleared in case
|
|
# it was added previosuly, but should not get new labels
|
|
unignored = list(filter(lambda s: not s.ignore(), sorted))
|
|
|
|
if len(unignored) == 0:
|
|
return
|
|
for part in self._buildPartitions(unignored):
|
|
if part.selected is not None:
|
|
if part.selected.getStatusDetail() is None:
|
|
part.selected.setStatusDetail([])
|
|
part.selected.getStatusDetail().append(part.details)
|
|
|
|
def day(self, utc_datetime: datetime, add_days=0):
|
|
local = self.time.toLocal(utc_datetime)
|
|
|
|
local_date = date.fromordinal(date(local.year, local.month, local.day).toordinal() + add_days)
|
|
return self.time.localize(datetime(local_date.year, local_date.month, local_date.day, 0, 0))
|