From 1a1ebc252bcc4698ef639dfc9002148a60b0f66a Mon Sep 17 00:00:00 2001 From: Sebastien Clement Date: Wed, 4 Nov 2020 16:10:25 +0100 Subject: [PATCH] :hammer: Add ability to upload backed-up snapshot from Nextcloud to Home assistant --- .../opt/nextcloud_backup/package-lock.json | 33 ++ .../rootfs/opt/nextcloud_backup/package.json | 2 + .../rootfs/opt/nextcloud_backup/routes/api.js | 15 + .../nextcloud_backup/tools/hassioApiTools.js | 316 +++++++++++------- .../opt/nextcloud_backup/tools/webdavTools.js | 228 ++++++++----- .../nextcloud_backup/views/backupSnaps.ejs | 2 +- .../opt/nextcloud_backup/views/index.ejs | 17 +- 7 files changed, 404 insertions(+), 209 deletions(-) diff --git a/nextcloud_backup/rootfs/opt/nextcloud_backup/package-lock.json b/nextcloud_backup/rootfs/opt/nextcloud_backup/package-lock.json index 67aeab2..d5a49f0 100644 --- a/nextcloud_backup/rootfs/opt/nextcloud_backup/package-lock.json +++ b/nextcloud_backup/rootfs/opt/nextcloud_backup/package-lock.json @@ -164,6 +164,11 @@ "lodash": "^4.17.14" } }, + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" + }, "axios": { "version": "0.19.0", "resolved": "https://registry.npmjs.org/axios/-/axios-0.19.0.tgz", @@ -337,6 +342,14 @@ "text-hex": "1.0.x" } }, + "combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "requires": { + "delayed-stream": "~1.0.0" + } + }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -447,6 +460,11 @@ "object-keys": "^1.0.12" } }, + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" + }, "depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", @@ -856,6 +874,16 @@ } } }, + "form-data": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.0.tgz", + "integrity": "sha512-CKMFDglpbMi6PyN+brwB9Q/GOw0eAnsrEZDgcsH5Krhz5Od/haKHAX0NmQfha2zPPz0JpWzA7GJHGSnvCRLWsg==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + } + }, "forwarded": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", @@ -1986,6 +2014,11 @@ "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" }, + "uuid": { + "version": "8.3.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.1.tgz", + "integrity": "sha512-FOmRr+FmWEIG8uhZv6C2bTgEVXsHk08kE7mPlrBbEe+c3r9pjceVPgupIfNIhc4yx55H69OXANrUaSuu9eInKg==" + }, "v8-compile-cache": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.1.0.tgz", diff --git a/nextcloud_backup/rootfs/opt/nextcloud_backup/package.json b/nextcloud_backup/rootfs/opt/nextcloud_backup/package.json index 91a60a7..8cdf90b 100644 --- a/nextcloud_backup/rootfs/opt/nextcloud_backup/package.json +++ b/nextcloud_backup/rootfs/opt/nextcloud_backup/package.json @@ -12,10 +12,12 @@ "debug": "~2.6.9", "ejs": "~2.6.1", "express": "~4.16.1", + "form-data": "^3.0.0", "got": "^11.5.2", "http-errors": "~1.6.3", "moment": "^2.24.0", "morgan": "~1.9.1", + "uuid": "^8.3.1", "webdav": "^2.10.0", "winston": "^3.2.1" }, diff --git a/nextcloud_backup/rootfs/opt/nextcloud_backup/routes/api.js b/nextcloud_backup/rootfs/opt/nextcloud_backup/routes/api.js index 41ba8ff..05e012d 100644 --- a/nextcloud_backup/rootfs/opt/nextcloud_backup/routes/api.js +++ b/nextcloud_backup/rootfs/opt/nextcloud_backup/routes/api.js @@ -190,5 +190,20 @@ router.post('/clean-now', function(req, res, next){ }); +router.post('/restore', function(req, res, next){ + if(req.body['path'] != null){ + webdav.downloadFile(req.body['path'] ).then((path)=>{ + hassioApiTools.uploadSnapshot(path); + }); + res.status(200); + res.send() + } + else{ + res.status(400); + res.send() + } +}); + + module.exports = router; diff --git a/nextcloud_backup/rootfs/opt/nextcloud_backup/tools/hassioApiTools.js b/nextcloud_backup/rootfs/opt/nextcloud_backup/tools/hassioApiTools.js index 42c75ab..aade5f0 100644 --- a/nextcloud_backup/rootfs/opt/nextcloud_backup/tools/hassioApiTools.js +++ b/nextcloud_backup/rootfs/opt/nextcloud_backup/tools/hassioApiTools.js @@ -7,7 +7,7 @@ const stream = require('stream'); const {promisify} = require('util'); const pipeline = promisify(stream.pipeline); const got = require ('got'); - +const FormData = require('form-data'); const create_snap_timeout = 90 * 60 * 1000 @@ -62,154 +62,220 @@ function downloadSnapshot(id) { headers: { 'X-HASSIO-KEY': token }, }; - pipeline( - got.stream.get(`http://hassio/snapshots/${id}/download`, option) - .on('downloadProgress', e => { - let percent = Math.round(e.percent * 100) / 100; - if (status.progress != percent) { - status.progress = percent; - statusTools.setStatus(status); - } - }) - , - stream - ) - .then((res)=>{ - logger.info('Download success !') - status.progress = 1; + pipeline(got.stream.get(`http://hassio/snapshots/${id}/download`, option) + .on('downloadProgress', e => { + let percent = Math.round(e.percent * 100) / 100; + if (status.progress != percent) { + status.progress = percent; statusTools.setStatus(status); - logger.debug("Snapshot dl size : " + (fs.statSync(tmp_file).size / 1024 / 1024)); - resolve(); - }).catch((err)=>{ - fs.unlinkSync(tmp_file); - status.status = "error"; - status.message = "Fail to download Hassio snapshot (" + err.message + ")"; - status.error_code = 7; - statusTools.setStatus(status); - logger.error(status.message); - reject(err.message); - }) - }).catch((err) => { + } + }), stream) + .then((res)=>{ + logger.info('Download success !') + status.progress = 1; + statusTools.setStatus(status); + logger.debug("Snapshot dl size : " + (fs.statSync(tmp_file).size / 1024 / 1024)); + resolve(); + }).catch((err)=>{ + fs.unlinkSync(tmp_file); status.status = "error"; - status.message = "Fail to download Hassio snapshot. Not found ?"; + status.message = "Fail to download Hassio snapshot (" + err.message + ")"; status.error_code = 7; statusTools.setStatus(status); logger.error(status.message); - reject(); - }); - - }); - } - - function dellSnap(id) { - return new Promise((resolve, reject) => { - checkSnap(id).then(() => { - let token = process.env.HASSIO_TOKEN; - - let option = { - headers: { 'X-HASSIO-KEY': token }, - responseType: 'json' - }; - - got.post(`http://hassio/snapshots/${id}/remove`, option) - .then((result) => { - resolve(); - }) - .catch((error) => { - reject(); - }); - }).catch(() => { - reject(); + reject(err.message); }) - }) + }).catch((err) => { + status.status = "error"; + status.message = "Fail to download Hassio snapshot. Not found ?"; + status.error_code = 7; + statusTools.setStatus(status); + logger.error(status.message); + reject(); + }); - } - - function checkSnap(id) { - return new Promise((resolve, reject) => { + }); +} + +function dellSnap(id) { + return new Promise((resolve, reject) => { + checkSnap(id).then(() => { let token = process.env.HASSIO_TOKEN; + let option = { headers: { 'X-HASSIO-KEY': token }, responseType: 'json' }; - got(`http://hassio/snapshots/${id}/info`, option) + got.post(`http://hassio/snapshots/${id}/remove`, option) .then((result) => { - logger.debug(`Snapshot size: ${result.body.data.size}`) resolve(); }) .catch((error) => { reject(); }); + }).catch(() => { + reject(); + }) + }) + +} + +function checkSnap(id) { + return new Promise((resolve, reject) => { + let token = process.env.HASSIO_TOKEN; + let option = { + headers: { 'X-HASSIO-KEY': token }, + responseType: 'json' + }; + + got(`http://hassio/snapshots/${id}/info`, option) + .then((result) => { + logger.debug(`Snapshot size: ${result.body.data.size}`) + resolve(); + }) + .catch((error) => { + reject(); + }); + }); + +} + + +function createNewBackup(name) { + return new Promise((resolve, reject) => { + let status = statusTools.getStatus(); + status.status = "creating"; + status.progress = -1; + statusTools.setStatus(status); + logger.info("Creating new snapshot...") + let token = process.env.HASSIO_TOKEN; + let option = { + headers: { 'X-HASSIO-KEY': token }, + responseType: 'json', + timeout: create_snap_timeout, + json: { name: name } + + }; + + got.post(`http://hassio/snapshots/new/full`, option) + .then((result) => { + logger.info(`Snapshot created with id ${result.body.data.slug}`); + resolve(result.body.data.slug); + }) + .catch((error) => { + status.status = "error"; + status.message = "Can't create new snapshot (" + error.message + ")"; + status.error_code = 5; + statusTools.setStatus(status); + logger.error(status.message); + reject(status.message); }); - } - - - function createNewBackup(name) { - return new Promise((resolve, reject) => { - let status = statusTools.getStatus(); - status.status = "creating"; - status.progress = -1; - statusTools.setStatus(status); - logger.info("Creating new snapshot...") - let token = process.env.HASSIO_TOKEN; - let option = { - headers: { 'X-HASSIO-KEY': token }, - responseType: 'json', - timeout: create_snap_timeout, - json: { name: name } - - }; + + + }); +} + +function clean() { + let limit = settingsTools.getSettings().auto_clean_local_keep; + if (limit == null) + limit = 5; + return new Promise((resolve, reject) => { + getSnapshots().then(async (snaps) => { + if (snaps.length < limit) { + resolve(); + return; + } + snaps.sort((a, b) => { + if (moment(a.date).isBefore(moment(b.date))) + return 1; + else + return -1; + }); + let toDel = snaps.slice(limit); + for (let i in toDel) { + await dellSnap(toDel[i].slug) + } + logger.info('Local clean done.') + resolve(); + }).catch(() => { + reject(); + }); + }) +} + + + +function uploadSnapshot(path) { + return new Promise( (resolve, reject) => { + let status = statusTools.getStatus(); + status.status = "upload-b"; + status.progress = 0; + status.message = null; + status.error_code = null; + statusTools.setStatus(status); + logger.info('Uploading backup...'); + let stream = fs.createReadStream(path); + let token = process.env.HASSIO_TOKEN; + + let form = new FormData(); + form.append('file', stream) + + let options = { + body: form, + username: this.username, + password: this.password, + headers: { 'X-HASSIO-KEY': token }, + } + + got.stream.post(`http://hassio/snapshots/new/upload`, options).on('uploadProgress', e => { + let percent = e.percent; + if (status.progress != percent) { + status.progress = percent; + statusTools.setStatus(status); + } + if (percent >= 1) { + logger.info('Upload done...'); + } + }).on('response', res => { - got.post(`http://hassio/snapshots/new/full`, option) - .then((result) => { - logger.info(`Snapshot created with id ${result.body.data.slug}`); - resolve(result.body.data.slug); - }) - .catch((error) => { + if (res.statusCode != 200) { status.status = "error"; - status.message = "Can't create new snapshot (" + error.message + ")"; - status.error_code = 5; + status.error_code = 4; + status.message = `Fail to upload backup to home assitant (Status code: ${res.statusCode})!`; statusTools.setStatus(status); logger.error(status.message); + fs.unlinkSync(path); reject(status.message); - }); - - - - }); - } - - function clean() { - let limit = settingsTools.getSettings().auto_clean_local_keep; - if (limit == null) - limit = 5; - return new Promise((resolve, reject) => { - getSnapshots().then(async (snaps) => { - if (snaps.length < limit) { - resolve(); - return; - } - snaps.sort((a, b) => { - if (moment(a.date).isBefore(moment(b.date))) - return 1; - else - return -1; - }); - let toDel = snaps.slice(limit); - for (let i in toDel) { - await dellSnap(toDel[i].slug) - } - logger.info('Local clean done.') + } else { + logger.info(`...Upload finish ! (status: ${res.statusCode})`); + status.status = "idle"; + status.progress = -1; + status.message = null; + status.error_code = null; + statusTools.setStatus(status); + fs.unlinkSync(path); resolve(); - }).catch(() => { - reject(); - }); - }) - } - - exports.getSnapshots = getSnapshots; - exports.downloadSnapshot = downloadSnapshot; - exports.createNewBackup = createNewBackup; - exports.clean = clean; \ No newline at end of file + } + }).on('error', err => { + + fs.unlinkSync(path); + status.status = "error"; + status.error_code = 4; + status.message = `Fail to upload backup to home assitant (${err}) !`; + statusTools.setStatus(status); + logger.error(status.message); + reject(status.message); + }); + + }); +} + + + +exports.getSnapshots = getSnapshots; +exports.downloadSnapshot = downloadSnapshot; +exports.createNewBackup = createNewBackup; +exports.uploadSnapshot = uploadSnapshot; +exports.clean = clean; \ No newline at end of file diff --git a/nextcloud_backup/rootfs/opt/nextcloud_backup/tools/webdavTools.js b/nextcloud_backup/rootfs/opt/nextcloud_backup/tools/webdavTools.js index ce838b5..7f30ef9 100644 --- a/nextcloud_backup/rootfs/opt/nextcloud_backup/tools/webdavTools.js +++ b/nextcloud_backup/rootfs/opt/nextcloud_backup/tools/webdavTools.js @@ -13,6 +13,9 @@ const hassioApiTools = require('./hassioApiTools'); const logger = require('../config/winston'); const got = require ('got'); +const stream = require('stream'); +const {promisify} = require('util'); +const pipeline = promisify(stream.pipeline); class WebdavTools { constructor() { @@ -77,9 +80,9 @@ class WebdavTools { logger.debug(`Path ${path} created.`) } catch (error) { if(error.response.status == 405) - logger.debug(`Path ${path} already exist.`) + logger.debug(`Path ${path} already exist.`) else - logger.error(error) + logger.error(error) } } @@ -99,8 +102,8 @@ class WebdavTools { } /** - * Check if theh webdav config is valid, if yes, start init of webdav client - */ + * Check if theh webdav config is valid, if yes, start init of webdav client + */ confIsValid() { return new Promise((resolve, reject) => { let status = statusTools.getStatus(); @@ -133,7 +136,7 @@ class WebdavTools { logger.error(status.message); reject("Nextcloud config invalid !"); } - + if(conf.back_dir == null || conf.back_dir == ''){ logger.info('Backup dir is null, initializing it.'); conf.back_dir = pathTools.default_root; @@ -265,86 +268,157 @@ class WebdavTools { }); } - getFolderContent(path) { + downloadFile(path) { return new Promise((resolve, reject) => { - if(this.client == null){ - reject(); - return; + if (this.client == null) { + this.confIsValid().then(() => { + this._startDownload(path) + .then((path)=> resolve(path)) + .catch(()=> reject()); + }).catch((err) => { + reject(err); + }) } - this.client.getDirectoryContents(path) - .then((contents) => { - resolve(contents); - }).catch((error) => { - reject(error); + else + this._startDownload(path) + .then((path)=> resolve(path)) + .catch(()=> reject()); + }); + } + _startDownload(path) { + return new Promise( (resolve, reject) => { + let status = statusTools.getStatus(); + status.status = "download-b"; + status.progress = 0; + status.message = null; + status.error_code = null; + statusTools.setStatus(status); + + logger.info('Downloading backup...'); + + let tmpFile = `./temp/restore_${moment().format('MMM-DD-YYYY_HH_mm')}.tar` + let stream = fs.createWriteStream(tmpFile); + let conf = this.getConf() + let options = { + username: this.username, + password: this.password, + } + if(conf.ssl === 'true'){ + options["https"] = { rejectUnauthorized: conf.self_signed === "false" } + } + + pipeline( + got.stream.get(this.baseUrl + encodeURI(path), options) + .on('downloadProgress', e => { + let percent = Math.round(e.percent * 100) / 100; + if (status.progress != percent) { + status.progress = percent; + statusTools.setStatus(status); + } + }) + , + stream + ) + .then((res)=>{ + logger.info('Download success !') + status.progress = 1; + statusTools.setStatus(status); + logger.debug("Backup dl size : " + (fs.statSync(tmpFile).size / 1024 / 1024)); + resolve(tmpFile); + }).catch((err)=>{ + fs.unlinkSync(tmpFile); + status.status = "error"; + status.message = "Fail to download Hassio snapshot (" + err.message + ")"; + status.error_code = 7; + statusTools.setStatus(status); + logger.error(status.message); + reject(err.message); + }) + + }); + } + + getFolderContent(path) { + return new Promise((resolve, reject) => { + if(this.client == null){ + reject(); + return; + } + this.client.getDirectoryContents(path) + .then((contents) => { + resolve(contents); + }).catch((error) => { + reject(error); + }) + }); + } + + clean() { + let limit = settingsTools.getSettings().auto_clean_backup_keep; + if (limit == null) + limit = 5; + return new Promise((resolve, reject) => { + this.getFolderContent(this.getConf().back_dir + pathTools.auto).then(async (contents) => { + if (contents.length < limit) { + resolve(); + return; + } + contents.sort((a, b) => { + if (moment(a.lastmod).isBefore(moment(b.lastmod))) + return 1; + else + return -1; + }); + + let toDel = contents.slice(limit); + for (let i in toDel) { + await this.client.deleteFile(toDel[i].filename); + } + logger.info('Cloud clean done.') + resolve(); + + }).catch((error) => { + status.status = "error"; + status.error_code = 6; + status.message = "Fail to clean Nexcloud (" + error + ") !" + statusTools.setStatus(status); + logger.error(status.message); + reject(status.message); + }); }) + + } + + + + + } + + function cleanTempFolder() { + fs.readdir("./temp/", (err, files) => { + if (err) throw err; + + for (const file of files) { + fs.unlink(path.join("./temp/", file), err => { + if (err) throw err; + }); + } }); } - clean() { - let limit = settingsTools.getSettings().auto_clean_backup_keep; - if (limit == null) - limit = 5; - return new Promise((resolve, reject) => { - this.getFolderContent(this.getConf().back_dir + pathTools.auto).then(async (contents) => { - if (contents.length < limit) { - resolve(); - return; - } - contents.sort((a, b) => { - if (moment(a.lastmod).isBefore(moment(b.lastmod))) - return 1; - else - return -1; - }); - - let toDel = contents.slice(limit); - for (let i in toDel) { - await this.client.deleteFile(toDel[i].filename); - } - logger.info('Cloud clean done.') - resolve(); - - }).catch((error) => { - status.status = "error"; - status.error_code = 6; - status.message = "Fail to clean Nexcloud (" + error + ") !" - statusTools.setStatus(status); - logger.error(status.message); - reject(status.message); - }); - }) - - } - - - -} - -function cleanTempFolder() { - fs.readdir("./temp/", (err, files) => { - if (err) throw err; - - for (const file of files) { - fs.unlink(path.join("./temp/", file), err => { - if (err) throw err; - }); + class Singleton { + constructor() { + if (!Singleton.instance) { + Singleton.instance = new WebdavTools(); + } } - }); -} - - -class Singleton { - constructor() { - if (!Singleton.instance) { - Singleton.instance = new WebdavTools(); + + getInstance() { + return Singleton.instance; } } - getInstance() { - return Singleton.instance; - } -} - -module.exports = Singleton; - + module.exports = Singleton; + + \ No newline at end of file diff --git a/nextcloud_backup/rootfs/opt/nextcloud_backup/views/backupSnaps.ejs b/nextcloud_backup/rootfs/opt/nextcloud_backup/views/backupSnaps.ejs index 45ae884..69aafb0 100644 --- a/nextcloud_backup/rootfs/opt/nextcloud_backup/views/backupSnaps.ejs +++ b/nextcloud_backup/rootfs/opt/nextcloud_backup/views/backupSnaps.ejs @@ -40,7 +40,7 @@
- Backup now + Upload to HA Close
diff --git a/nextcloud_backup/rootfs/opt/nextcloud_backup/views/index.ejs b/nextcloud_backup/rootfs/opt/nextcloud_backup/views/index.ejs index 80030c9..defc777 100644 --- a/nextcloud_backup/rootfs/opt/nextcloud_backup/views/index.ejs +++ b/nextcloud_backup/rootfs/opt/nextcloud_backup/views/index.ejs @@ -185,7 +185,7 @@ <%- include('modals/backup-settings-modal') %> - <%- include('modals/restore-modal.ejs') %> +
@@ -261,8 +261,6 @@ var last_manu_back = ""; var last_auto_back = ""; - var to_restore = ""; - var loadingModal = null; document.addEventListener('DOMContentLoaded', function() { updateLocalSnaps(); @@ -299,10 +297,9 @@ manualBackup(id, name); }) $('.restore').click(function(){ - to_restore = this.getAttribute('data-id'); + let to_restore = this.getAttribute('data-id'); console.log(to_restore) - let restore_modal = M.Modal.getInstance(document.querySelector('#modal-restore')); - restore_modal.open(); + restore(to_restore) }) } @@ -368,10 +365,18 @@ printStatusWithBar('Downloading Snapshot', data.progress); $('#btn-backup-now, #btn-clean-now').addClass("disabled"); break; + case "download-b": + printStatusWithBar('Downloading Backup', data.progress); + $('#btn-backup-now, #btn-clean-now').addClass("disabled"); + break; case "upload": printStatusWithBar('Uploading Snapshot', data.progress); $('#btn-backup-now, #btn-clean-now').addClass("disabled"); break; + case "upload-b": + printStatusWithBar('Uploading Snapshot', data.progress); + $('#btn-backup-now, #btn-clean-now').addClass("disabled"); + break; case "creating": printStatusWithBar('Creating Snapshot', data.progress); $('#btn-backup-now, #btn-clean-now').addClass("disabled");