From 340ea53e0e71750ec7ec96fd25898bc73d0d4bcc Mon Sep 17 00:00:00 2001 From: SebClem Date: Fri, 2 Aug 2024 15:56:07 +0200 Subject: [PATCH] Add cleanup --- nextcloud_backup/backend/src/routes/action.ts | 20 ++++ .../src/services/homeAssistantService.ts | 94 +++++++++++++------ .../backend/src/services/orchestrator.ts | 6 ++ .../backend/src/services/webdavService.ts | 91 +++++++++++------- nextcloud_backup/backend/src/types/status.ts | 2 + .../components/statusBar/ActionComponent.vue | 13 ++- .../frontend/src/services/actionService.ts | 4 + 7 files changed, 164 insertions(+), 66 deletions(-) diff --git a/nextcloud_backup/backend/src/routes/action.ts b/nextcloud_backup/backend/src/routes/action.ts index cfbea9a..4f44e6c 100644 --- a/nextcloud_backup/backend/src/routes/action.ts +++ b/nextcloud_backup/backend/src/routes/action.ts @@ -2,6 +2,10 @@ import express from "express"; import { doBackupWorkflow } from "../services/orchestrator.js"; import { WorkflowType } from "../types/services/orchecstrator.js"; import logger from "../config/winston.js"; +import { clean as webdavClean } from "../services/webdavService.js"; +import { getBackupConfig } from "../services/backupConfigService.js"; +import { getWebdavConfig } from "../services/webdavConfigService.js"; +import { clean } from "../services/homeAssistantService.js"; const actionRouter = express.Router(); @@ -16,4 +20,20 @@ actionRouter.post("/backup", (req, res) => { res.sendStatus(202); }); +actionRouter.post("/clean", (req, res) => { + const backupConfig = getBackupConfig(); + const webdavConfig = getWebdavConfig(); + webdavClean(backupConfig, webdavConfig) + .then(() => { + return clean(backupConfig); + }) + .then(() => { + logger.info("All good !"); + }) + .catch(() => { + logger.error("Something wrong !"); + }); + res.sendStatus(202); +}); + export default actionRouter; diff --git a/nextcloud_backup/backend/src/services/homeAssistantService.ts b/nextcloud_backup/backend/src/services/homeAssistantService.ts index 3330b5c..b7eac5d 100644 --- a/nextcloud_backup/backend/src/services/homeAssistantService.ts +++ b/nextcloud_backup/backend/src/services/homeAssistantService.ts @@ -13,13 +13,15 @@ import { promisify } from "util"; import logger from "../config/winston.js"; import messageManager from "../tools/messageManager.js"; import * as statusTools from "../tools/status.js"; -import { BackupType } from "../types/services/backupConfig.js"; +import { + BackupType, + type BackupConfig, +} from "../types/services/backupConfig.js"; import type { NewBackupPayload } from "../types/services/ha_os_payload.js"; import type { AddonData, BackupData, BackupDetailModel, - BackupModel, CoreInfoBody, SupervisorResponse, } from "../types/services/ha_os_response.js"; @@ -161,7 +163,7 @@ function downloadSnapshot(id: string): Promise { } function delSnap(id: string) { - logger.info(`Deleting Home Assistant backup ${id}`); + logger.debug(`Deleting Home Assistant backup ${id}`); const option = { headers: { authorization: `Bearer ${token}` }, }; @@ -262,36 +264,66 @@ function createNewBackup( ); } -function clean(backups: BackupModel[], numberToKeep: number) { - const promises = []; - if (backups.length < numberToKeep) { - return; +function clean(backupConfig: BackupConfig) { + if (!backupConfig.autoClean.homeAssistant.enabled) { + logger.debug("Clean disabled for Home Assistant"); + return Promise.resolve(); } - backups.sort((a, b) => { - return Date.parse(b.date) - Date.parse(a.date); - }); - const toDel = backups.slice(numberToKeep); - for (const i of toDel) { - promises.push(delSnap(i.slug)); - } - logger.info("Local clean done."); - return Promise.allSettled(promises).then((values) => { - let errors = false; - for (const val of values) { - if (val.status == "rejected") { - // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - messageManager.error("Fail to delete backup", val.reason); - logger.error("Fail to delete backup"); - logger.error(val.reason); - errors = true; + logger.info("Clean for Home Assistant"); + const status = statusTools.getStatus(); + status.status = States.CLEAN_HA; + status.progress = -1; + statusTools.setStatus(status); + + const numberToKeep = backupConfig.autoClean.homeAssistant.nbrToKeep || 5; + return getBackups() + .then((response) => { + const backups = response.body.data.backups; + if (backups.length > numberToKeep) { + backups.sort((a, b) => { + return Date.parse(b.date) - Date.parse(a.date); + }); + const toDel = backups.slice(numberToKeep); + logger.debug(`Number of backup to clean: ${toDel.length}`); + const promises = toDel.map((value) => delSnap(value.slug)); + logger.info("Home Assistant clean done."); + return Promise.allSettled(promises); + } else { + logger.debug("Nothing to clean"); } - } - if (errors) { - messageManager.error("Fail to clean backups in Home Assistant"); - logger.error("Fail to clean backups in Home Assistant"); - return Promise.reject(new Error()); - } - }); + }) + .then( + (values) => { + const status = statusTools.getStatus(); + status.status = States.IDLE; + status.progress = undefined; + statusTools.setStatus(status); + + let errors = false; + for (const val of values || []) { + if (val.status == "rejected") { + messageManager.error("Fail to delete backup", val.reason as string); + logger.error("Fail to delete backup"); + logger.error(val.reason); + errors = true; + } + } + if (errors) { + messageManager.error("Fail to clean backups in Home Assistant"); + logger.error("Fail to clean backups in Home Assistant"); + return Promise.reject(new Error()); + } + return Promise.resolve(); + }, + (reason: RequestError) => { + logger.error("Fail to clean Home Assistant backup", reason.message); + messageManager.error( + "Fail to clean Home Assistant backup", + reason.message + ); + return Promise.reject(reason); + } + ); } function uploadSnapshot(path: string) { diff --git a/nextcloud_backup/backend/src/services/orchestrator.ts b/nextcloud_backup/backend/src/services/orchestrator.ts index 325706a..d69709c 100644 --- a/nextcloud_backup/backend/src/services/orchestrator.ts +++ b/nextcloud_backup/backend/src/services/orchestrator.ts @@ -92,6 +92,12 @@ export function doBackupWorkflow(type: WorkflowType) { .then(() => { return homeAssistantService.startAddons(addonsToStartStop); }) + .then(() => { + return homeAssistantService.clean(backupConfig); + }) + .then(() => { + return webDavService.clean(backupConfig, webdavConfig); + }) .then(() => { logger.info("Backup workflow finished successfully !"); messageManager.info( diff --git a/nextcloud_backup/backend/src/services/webdavService.ts b/nextcloud_backup/backend/src/services/webdavService.ts index 76ffd42..4e2cdfd 100644 --- a/nextcloud_backup/backend/src/services/webdavService.ts +++ b/nextcloud_backup/backend/src/services/webdavService.ts @@ -23,6 +23,7 @@ import { templateToRegexp } from "./backupConfigService.js"; import { getChunkEndpoint, getEndpoint } from "./webdavConfigService.js"; import { pipeline } from "stream/promises"; import { humanFileSize } from "../tools/toolbox.js"; +import type { BackupConfig } from "../types/services/backupConfig.js"; const CHUNK_SIZE = 5 * 1024 * 1024; // 5MiB Same as desktop client const CHUNK_NUMBER_SIZE = 5; // To add landing "0" @@ -201,6 +202,7 @@ function extractBackupInfo(backups: WebdavBackup[], template: string) { } export function deleteBackup(path: string, config: WebdavConfig) { + logger.debug(`Deleting Cloud backup ${path}`); const endpoint = getEndpoint(config); return got .delete(config.url + endpoint + path, { @@ -589,6 +591,7 @@ export function downloadFile( }, (reason: RequestError) => { if (fs.existsSync(tmp_file)) fs.unlinkSync(tmp_file); + logger.error("Fail to download Cloud backup", reason.message); messageManager.error("Fail to download Cloud backup", reason.message); const status = statusTools.getStatus(); status.status = States.IDLE; @@ -599,39 +602,59 @@ export function downloadFile( ); } -// clean() { -// let limit = settingsTools.getSettings().auto_clean_backup_keep; -// if (limit == null) limit = 5; -// return new Promise((resolve, reject) => { -// this.getFolderContent(this.getConf()?.back_dir + pathTools.auto) -// .then(async (contents: any) => { -// if (contents.length < limit) { -// resolve(undefined); -// return; -// } -// contents.sort((a: any, b: any) => { -// return a.date < b.date ? 1 : -1; -// }); +export function clean(backupConfig: BackupConfig, webdavConfig: WebdavConfig) { + if (!backupConfig.autoClean.webdav.enabled) { + logger.debug("Clean disabled for Cloud"); + return Promise.resolve(); + } + logger.info("Clean for cloud"); + const status = statusTools.getStatus(); + status.status = States.CLEAN_CLOUD; + status.progress = -1; + statusTools.setStatus(status); + const limit = backupConfig.autoClean.homeAssistant.nbrToKeep || 5; + return getBackups(pathTools.auto, webdavConfig, backupConfig.nameTemplate) + .then((backups) => { + if (backups.length > limit) { + const toDel = backups.splice(limit); + logger.debug(`Number of backup to clean: ${toDel.length}`); + const promises = toDel.map((value) => + deleteBackup(value.path, webdavConfig) + ); + return Promise.allSettled(promises); + } else { + logger.debug("Nothing to clean"); + } + }) + .then( + (values) => { + const status = statusTools.getStatus(); + status.status = States.IDLE; + status.progress = undefined; + statusTools.setStatus(status); -// const toDel = contents.slice(limit); -// for (const i in toDel) { -// await this.client?.deleteFile(toDel[i].filename); -// } -// logger.info("Cloud clean done."); -// resolve(undefined); -// }) -// .catch((error) => { -// const status = statusTools.getStatus(); -// status.status = "error"; -// status.error_code = 6; -// status.message = "Fail to clean Nexcloud (" + error + ") !"; -// statusTools.setStatus(status); -// logger.error(status.message); -// reject(status.message); -// }); -// }); -// } -// } + let errors = false; + for (const val of values || []) { + if (val.status == "rejected") { + messageManager.error("Fail to delete backup", val.reason); + logger.error("Fail to delete backup"); + logger.error(val.reason); + errors = true; + } + } -// const INSTANCE = new WebdavTools(); -// export default INSTANCE; + if (errors) { + messageManager.error("Fail to clean backups in Cloud"); + logger.error("Fail to clean backups in Cloud"); + return Promise.reject(new Error()); + } + + return Promise.resolve(); + }, + (reason: RequestError) => { + logger.error("Fail to clean cloud backup", reason.message); + messageManager.error("Fail to clean cloud backup", reason.message); + return Promise.reject(reason); + } + ); +} diff --git a/nextcloud_backup/backend/src/types/status.ts b/nextcloud_backup/backend/src/types/status.ts index e00c8ab..f14b223 100644 --- a/nextcloud_backup/backend/src/types/status.ts +++ b/nextcloud_backup/backend/src/types/status.ts @@ -9,6 +9,8 @@ export enum States { BKUP_UPLOAD_CLOUD = "BKUP_UPLOAD_CLOUD", STOP_ADDON = "STOP_ADDON", START_ADDON = "START_ADDON", + CLEAN_CLOUD = "CLEAN_CLOUD", + CLEAN_HA = "CLEAN_HA", } export interface Status { diff --git a/nextcloud_backup/frontend/src/components/statusBar/ActionComponent.vue b/nextcloud_backup/frontend/src/components/statusBar/ActionComponent.vue index ce2d482..e50d1b3 100644 --- a/nextcloud_backup/frontend/src/components/statusBar/ActionComponent.vue +++ b/nextcloud_backup/frontend/src/components/statusBar/ActionComponent.vue @@ -4,12 +4,13 @@ Backup Now + Clean diff --git a/nextcloud_backup/frontend/src/services/actionService.ts b/nextcloud_backup/frontend/src/services/actionService.ts index caf0aab..75042bb 100644 --- a/nextcloud_backup/frontend/src/services/actionService.ts +++ b/nextcloud_backup/frontend/src/services/actionService.ts @@ -3,3 +3,7 @@ import kyClient from "./kyClient"; export function backupNow() { return kyClient.post("action/backup"); } + +export function clean() { + return kyClient.post("action/clean"); +} \ No newline at end of file