🔨 Add ability to upload backed-up snapshot from Nextcloud to Home assistant

This commit is contained in:
Sebastien Clement 2020-11-04 16:10:25 +01:00
parent 88cd6f2c45
commit 1a1ebc252b
7 changed files with 404 additions and 209 deletions

View File

@ -164,6 +164,11 @@
"lodash": "^4.17.14" "lodash": "^4.17.14"
} }
}, },
"asynckit": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
"integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k="
},
"axios": { "axios": {
"version": "0.19.0", "version": "0.19.0",
"resolved": "https://registry.npmjs.org/axios/-/axios-0.19.0.tgz", "resolved": "https://registry.npmjs.org/axios/-/axios-0.19.0.tgz",
@ -337,6 +342,14 @@
"text-hex": "1.0.x" "text-hex": "1.0.x"
} }
}, },
"combined-stream": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
"integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
"requires": {
"delayed-stream": "~1.0.0"
}
},
"concat-map": { "concat-map": {
"version": "0.0.1", "version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
@ -447,6 +460,11 @@
"object-keys": "^1.0.12" "object-keys": "^1.0.12"
} }
}, },
"delayed-stream": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
"integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk="
},
"depd": { "depd": {
"version": "1.1.2", "version": "1.1.2",
"resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
@ -856,6 +874,16 @@
} }
} }
}, },
"form-data": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.0.tgz",
"integrity": "sha512-CKMFDglpbMi6PyN+brwB9Q/GOw0eAnsrEZDgcsH5Krhz5Od/haKHAX0NmQfha2zPPz0JpWzA7GJHGSnvCRLWsg==",
"requires": {
"asynckit": "^0.4.0",
"combined-stream": "^1.0.8",
"mime-types": "^2.1.12"
}
},
"forwarded": { "forwarded": {
"version": "0.1.2", "version": "0.1.2",
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz",
@ -1986,6 +2014,11 @@
"resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz",
"integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM="
}, },
"uuid": {
"version": "8.3.1",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.1.tgz",
"integrity": "sha512-FOmRr+FmWEIG8uhZv6C2bTgEVXsHk08kE7mPlrBbEe+c3r9pjceVPgupIfNIhc4yx55H69OXANrUaSuu9eInKg=="
},
"v8-compile-cache": { "v8-compile-cache": {
"version": "2.1.0", "version": "2.1.0",
"resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.1.0.tgz", "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.1.0.tgz",

View File

@ -12,10 +12,12 @@
"debug": "~2.6.9", "debug": "~2.6.9",
"ejs": "~2.6.1", "ejs": "~2.6.1",
"express": "~4.16.1", "express": "~4.16.1",
"form-data": "^3.0.0",
"got": "^11.5.2", "got": "^11.5.2",
"http-errors": "~1.6.3", "http-errors": "~1.6.3",
"moment": "^2.24.0", "moment": "^2.24.0",
"morgan": "~1.9.1", "morgan": "~1.9.1",
"uuid": "^8.3.1",
"webdav": "^2.10.0", "webdav": "^2.10.0",
"winston": "^3.2.1" "winston": "^3.2.1"
}, },

View File

@ -190,5 +190,20 @@ router.post('/clean-now', function(req, res, next){
}); });
router.post('/restore', function(req, res, next){
if(req.body['path'] != null){
webdav.downloadFile(req.body['path'] ).then((path)=>{
hassioApiTools.uploadSnapshot(path);
});
res.status(200);
res.send()
}
else{
res.status(400);
res.send()
}
});
module.exports = router; module.exports = router;

View File

@ -7,7 +7,7 @@ const stream = require('stream');
const {promisify} = require('util'); const {promisify} = require('util');
const pipeline = promisify(stream.pipeline); const pipeline = promisify(stream.pipeline);
const got = require ('got'); const got = require ('got');
const FormData = require('form-data');
const create_snap_timeout = 90 * 60 * 1000 const create_snap_timeout = 90 * 60 * 1000
@ -62,154 +62,220 @@ function downloadSnapshot(id) {
headers: { 'X-HASSIO-KEY': token }, headers: { 'X-HASSIO-KEY': token },
}; };
pipeline( pipeline(got.stream.get(`http://hassio/snapshots/${id}/download`, option)
got.stream.get(`http://hassio/snapshots/${id}/download`, option) .on('downloadProgress', e => {
.on('downloadProgress', e => { let percent = Math.round(e.percent * 100) / 100;
let percent = Math.round(e.percent * 100) / 100; if (status.progress != percent) {
if (status.progress != percent) { status.progress = percent;
status.progress = percent;
statusTools.setStatus(status);
}
})
,
stream
)
.then((res)=>{
logger.info('Download success !')
status.progress = 1;
statusTools.setStatus(status); statusTools.setStatus(status);
logger.debug("Snapshot dl size : " + (fs.statSync(tmp_file).size / 1024 / 1024)); }
resolve(); }), stream)
}).catch((err)=>{ .then((res)=>{
fs.unlinkSync(tmp_file); logger.info('Download success !')
status.status = "error"; status.progress = 1;
status.message = "Fail to download Hassio snapshot (" + err.message + ")"; statusTools.setStatus(status);
status.error_code = 7; logger.debug("Snapshot dl size : " + (fs.statSync(tmp_file).size / 1024 / 1024));
statusTools.setStatus(status); resolve();
logger.error(status.message); }).catch((err)=>{
reject(err.message); fs.unlinkSync(tmp_file);
})
}).catch((err) => {
status.status = "error"; status.status = "error";
status.message = "Fail to download Hassio snapshot. Not found ?"; status.message = "Fail to download Hassio snapshot (" + err.message + ")";
status.error_code = 7; status.error_code = 7;
statusTools.setStatus(status); statusTools.setStatus(status);
logger.error(status.message); logger.error(status.message);
reject(); reject(err.message);
});
});
}
function dellSnap(id) {
return new Promise((resolve, reject) => {
checkSnap(id).then(() => {
let token = process.env.HASSIO_TOKEN;
let option = {
headers: { 'X-HASSIO-KEY': token },
responseType: 'json'
};
got.post(`http://hassio/snapshots/${id}/remove`, option)
.then((result) => {
resolve();
})
.catch((error) => {
reject();
});
}).catch(() => {
reject();
}) })
}) }).catch((err) => {
status.status = "error";
status.message = "Fail to download Hassio snapshot. Not found ?";
status.error_code = 7;
statusTools.setStatus(status);
logger.error(status.message);
reject();
});
} });
}
function checkSnap(id) {
return new Promise((resolve, reject) => { function dellSnap(id) {
return new Promise((resolve, reject) => {
checkSnap(id).then(() => {
let token = process.env.HASSIO_TOKEN; let token = process.env.HASSIO_TOKEN;
let option = { let option = {
headers: { 'X-HASSIO-KEY': token }, headers: { 'X-HASSIO-KEY': token },
responseType: 'json' responseType: 'json'
}; };
got(`http://hassio/snapshots/${id}/info`, option) got.post(`http://hassio/snapshots/${id}/remove`, option)
.then((result) => { .then((result) => {
logger.debug(`Snapshot size: ${result.body.data.size}`)
resolve(); resolve();
}) })
.catch((error) => { .catch((error) => {
reject(); reject();
}); });
}).catch(() => {
reject();
})
})
}
function checkSnap(id) {
return new Promise((resolve, reject) => {
let token = process.env.HASSIO_TOKEN;
let option = {
headers: { 'X-HASSIO-KEY': token },
responseType: 'json'
};
got(`http://hassio/snapshots/${id}/info`, option)
.then((result) => {
logger.debug(`Snapshot size: ${result.body.data.size}`)
resolve();
})
.catch((error) => {
reject();
});
});
}
function createNewBackup(name) {
return new Promise((resolve, reject) => {
let status = statusTools.getStatus();
status.status = "creating";
status.progress = -1;
statusTools.setStatus(status);
logger.info("Creating new snapshot...")
let token = process.env.HASSIO_TOKEN;
let option = {
headers: { 'X-HASSIO-KEY': token },
responseType: 'json',
timeout: create_snap_timeout,
json: { name: name }
};
got.post(`http://hassio/snapshots/new/full`, option)
.then((result) => {
logger.info(`Snapshot created with id ${result.body.data.slug}`);
resolve(result.body.data.slug);
})
.catch((error) => {
status.status = "error";
status.message = "Can't create new snapshot (" + error.message + ")";
status.error_code = 5;
statusTools.setStatus(status);
logger.error(status.message);
reject(status.message);
}); });
}
});
function createNewBackup(name) { }
return new Promise((resolve, reject) => {
let status = statusTools.getStatus(); function clean() {
status.status = "creating"; let limit = settingsTools.getSettings().auto_clean_local_keep;
status.progress = -1; if (limit == null)
statusTools.setStatus(status); limit = 5;
logger.info("Creating new snapshot...") return new Promise((resolve, reject) => {
let token = process.env.HASSIO_TOKEN; getSnapshots().then(async (snaps) => {
let option = { if (snaps.length < limit) {
headers: { 'X-HASSIO-KEY': token }, resolve();
responseType: 'json', return;
timeout: create_snap_timeout, }
json: { name: name } snaps.sort((a, b) => {
if (moment(a.date).isBefore(moment(b.date)))
}; return 1;
else
return -1;
});
let toDel = snaps.slice(limit);
for (let i in toDel) {
await dellSnap(toDel[i].slug)
}
logger.info('Local clean done.')
resolve();
}).catch(() => {
reject();
});
})
}
function uploadSnapshot(path) {
return new Promise( (resolve, reject) => {
let status = statusTools.getStatus();
status.status = "upload-b";
status.progress = 0;
status.message = null;
status.error_code = null;
statusTools.setStatus(status);
logger.info('Uploading backup...');
let stream = fs.createReadStream(path);
let token = process.env.HASSIO_TOKEN;
let form = new FormData();
form.append('file', stream)
let options = {
body: form,
username: this.username,
password: this.password,
headers: { 'X-HASSIO-KEY': token },
}
got.stream.post(`http://hassio/snapshots/new/upload`, options).on('uploadProgress', e => {
let percent = e.percent;
if (status.progress != percent) {
status.progress = percent;
statusTools.setStatus(status);
}
if (percent >= 1) {
logger.info('Upload done...');
}
}).on('response', res => {
got.post(`http://hassio/snapshots/new/full`, option) if (res.statusCode != 200) {
.then((result) => {
logger.info(`Snapshot created with id ${result.body.data.slug}`);
resolve(result.body.data.slug);
})
.catch((error) => {
status.status = "error"; status.status = "error";
status.message = "Can't create new snapshot (" + error.message + ")"; status.error_code = 4;
status.error_code = 5; status.message = `Fail to upload backup to home assitant (Status code: ${res.statusCode})!`;
statusTools.setStatus(status); statusTools.setStatus(status);
logger.error(status.message); logger.error(status.message);
fs.unlinkSync(path);
reject(status.message); reject(status.message);
}); } else {
logger.info(`...Upload finish ! (status: ${res.statusCode})`);
status.status = "idle";
status.progress = -1;
}); status.message = null;
} status.error_code = null;
statusTools.setStatus(status);
function clean() { fs.unlinkSync(path);
let limit = settingsTools.getSettings().auto_clean_local_keep;
if (limit == null)
limit = 5;
return new Promise((resolve, reject) => {
getSnapshots().then(async (snaps) => {
if (snaps.length < limit) {
resolve();
return;
}
snaps.sort((a, b) => {
if (moment(a.date).isBefore(moment(b.date)))
return 1;
else
return -1;
});
let toDel = snaps.slice(limit);
for (let i in toDel) {
await dellSnap(toDel[i].slug)
}
logger.info('Local clean done.')
resolve(); resolve();
}).catch(() => { }
reject(); }).on('error', err => {
});
}) fs.unlinkSync(path);
} status.status = "error";
status.error_code = 4;
exports.getSnapshots = getSnapshots; status.message = `Fail to upload backup to home assitant (${err}) !`;
exports.downloadSnapshot = downloadSnapshot; statusTools.setStatus(status);
exports.createNewBackup = createNewBackup; logger.error(status.message);
exports.clean = clean; reject(status.message);
});
});
}
exports.getSnapshots = getSnapshots;
exports.downloadSnapshot = downloadSnapshot;
exports.createNewBackup = createNewBackup;
exports.uploadSnapshot = uploadSnapshot;
exports.clean = clean;

View File

@ -13,6 +13,9 @@ const hassioApiTools = require('./hassioApiTools');
const logger = require('../config/winston'); const logger = require('../config/winston');
const got = require ('got'); const got = require ('got');
const stream = require('stream');
const {promisify} = require('util');
const pipeline = promisify(stream.pipeline);
class WebdavTools { class WebdavTools {
constructor() { constructor() {
@ -77,9 +80,9 @@ class WebdavTools {
logger.debug(`Path ${path} created.`) logger.debug(`Path ${path} created.`)
} catch (error) { } catch (error) {
if(error.response.status == 405) if(error.response.status == 405)
logger.debug(`Path ${path} already exist.`) logger.debug(`Path ${path} already exist.`)
else else
logger.error(error) logger.error(error)
} }
} }
@ -99,8 +102,8 @@ class WebdavTools {
} }
/** /**
* Check if theh webdav config is valid, if yes, start init of webdav client * Check if theh webdav config is valid, if yes, start init of webdav client
*/ */
confIsValid() { confIsValid() {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
let status = statusTools.getStatus(); let status = statusTools.getStatus();
@ -133,7 +136,7 @@ class WebdavTools {
logger.error(status.message); logger.error(status.message);
reject("Nextcloud config invalid !"); reject("Nextcloud config invalid !");
} }
if(conf.back_dir == null || conf.back_dir == ''){ if(conf.back_dir == null || conf.back_dir == ''){
logger.info('Backup dir is null, initializing it.'); logger.info('Backup dir is null, initializing it.');
conf.back_dir = pathTools.default_root; conf.back_dir = pathTools.default_root;
@ -265,86 +268,157 @@ class WebdavTools {
}); });
} }
getFolderContent(path) { downloadFile(path) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
if(this.client == null){ if (this.client == null) {
reject(); this.confIsValid().then(() => {
return; this._startDownload(path)
.then((path)=> resolve(path))
.catch(()=> reject());
}).catch((err) => {
reject(err);
})
} }
this.client.getDirectoryContents(path) else
.then((contents) => { this._startDownload(path)
resolve(contents); .then((path)=> resolve(path))
}).catch((error) => { .catch(()=> reject());
reject(error); });
}
_startDownload(path) {
return new Promise( (resolve, reject) => {
let status = statusTools.getStatus();
status.status = "download-b";
status.progress = 0;
status.message = null;
status.error_code = null;
statusTools.setStatus(status);
logger.info('Downloading backup...');
let tmpFile = `./temp/restore_${moment().format('MMM-DD-YYYY_HH_mm')}.tar`
let stream = fs.createWriteStream(tmpFile);
let conf = this.getConf()
let options = {
username: this.username,
password: this.password,
}
if(conf.ssl === 'true'){
options["https"] = { rejectUnauthorized: conf.self_signed === "false" }
}
pipeline(
got.stream.get(this.baseUrl + encodeURI(path), options)
.on('downloadProgress', e => {
let percent = Math.round(e.percent * 100) / 100;
if (status.progress != percent) {
status.progress = percent;
statusTools.setStatus(status);
}
})
,
stream
)
.then((res)=>{
logger.info('Download success !')
status.progress = 1;
statusTools.setStatus(status);
logger.debug("Backup dl size : " + (fs.statSync(tmpFile).size / 1024 / 1024));
resolve(tmpFile);
}).catch((err)=>{
fs.unlinkSync(tmpFile);
status.status = "error";
status.message = "Fail to download Hassio snapshot (" + err.message + ")";
status.error_code = 7;
statusTools.setStatus(status);
logger.error(status.message);
reject(err.message);
})
});
}
getFolderContent(path) {
return new Promise((resolve, reject) => {
if(this.client == null){
reject();
return;
}
this.client.getDirectoryContents(path)
.then((contents) => {
resolve(contents);
}).catch((error) => {
reject(error);
})
});
}
clean() {
let limit = settingsTools.getSettings().auto_clean_backup_keep;
if (limit == null)
limit = 5;
return new Promise((resolve, reject) => {
this.getFolderContent(this.getConf().back_dir + pathTools.auto).then(async (contents) => {
if (contents.length < limit) {
resolve();
return;
}
contents.sort((a, b) => {
if (moment(a.lastmod).isBefore(moment(b.lastmod)))
return 1;
else
return -1;
});
let toDel = contents.slice(limit);
for (let i in toDel) {
await this.client.deleteFile(toDel[i].filename);
}
logger.info('Cloud clean done.')
resolve();
}).catch((error) => {
status.status = "error";
status.error_code = 6;
status.message = "Fail to clean Nexcloud (" + error + ") !"
statusTools.setStatus(status);
logger.error(status.message);
reject(status.message);
});
}) })
}
}
function cleanTempFolder() {
fs.readdir("./temp/", (err, files) => {
if (err) throw err;
for (const file of files) {
fs.unlink(path.join("./temp/", file), err => {
if (err) throw err;
});
}
}); });
} }
clean() {
let limit = settingsTools.getSettings().auto_clean_backup_keep;
if (limit == null)
limit = 5;
return new Promise((resolve, reject) => {
this.getFolderContent(this.getConf().back_dir + pathTools.auto).then(async (contents) => {
if (contents.length < limit) {
resolve();
return;
}
contents.sort((a, b) => {
if (moment(a.lastmod).isBefore(moment(b.lastmod)))
return 1;
else
return -1;
});
let toDel = contents.slice(limit);
for (let i in toDel) {
await this.client.deleteFile(toDel[i].filename);
}
logger.info('Cloud clean done.')
resolve();
}).catch((error) => {
status.status = "error";
status.error_code = 6;
status.message = "Fail to clean Nexcloud (" + error + ") !"
statusTools.setStatus(status);
logger.error(status.message);
reject(status.message);
});
})
}
class Singleton {
constructor() {
if (!Singleton.instance) {
} Singleton.instance = new WebdavTools();
}
function cleanTempFolder() {
fs.readdir("./temp/", (err, files) => {
if (err) throw err;
for (const file of files) {
fs.unlink(path.join("./temp/", file), err => {
if (err) throw err;
});
} }
});
} getInstance() {
return Singleton.instance;
class Singleton {
constructor() {
if (!Singleton.instance) {
Singleton.instance = new WebdavTools();
} }
} }
getInstance() { module.exports = Singleton;
return Singleton.instance;
}
}
module.exports = Singleton;

View File

@ -40,7 +40,7 @@
</div> </div>
</div> </div>
<div class="modal-footer blue-grey darken-4"> <div class="modal-footer blue-grey darken-4">
<a href="#!" class="waves-effect waves-green btn green restore modal-close" data-id="<%=backups[index].filename%>" data-name='<%= backups[index].basename ? backups[index].basename : backups[index].etag %>'>Backup now</a> <a href="#!" class="waves-effect waves-green btn green restore modal-close" data-id="<%=backups[index].filename%>" data-name='<%= backups[index].basename ? backups[index].basename : backups[index].etag %>'>Upload to HA</a>
<a href="#!" class="modal-close waves-effect waves-green btn red">Close</a> <a href="#!" class="modal-close waves-effect waves-green btn red">Close</a>
</div> </div>
</div> </div>

View File

@ -185,7 +185,7 @@
<%- include('modals/backup-settings-modal') %> <%- include('modals/backup-settings-modal') %>
<%- include('modals/restore-modal.ejs') %> <!-- <%- include('modals/restore-modal.ejs') %> -->
<div id="modal-loading" class="modal blue-grey darken-4 white-text"> <div id="modal-loading" class="modal blue-grey darken-4 white-text">
<div class="modal-content "> <div class="modal-content ">
@ -261,8 +261,6 @@
var last_manu_back = ""; var last_manu_back = "";
var last_auto_back = ""; var last_auto_back = "";
var to_restore = "";
var loadingModal = null; var loadingModal = null;
document.addEventListener('DOMContentLoaded', function() { document.addEventListener('DOMContentLoaded', function() {
updateLocalSnaps(); updateLocalSnaps();
@ -299,10 +297,9 @@
manualBackup(id, name); manualBackup(id, name);
}) })
$('.restore').click(function(){ $('.restore').click(function(){
to_restore = this.getAttribute('data-id'); let to_restore = this.getAttribute('data-id');
console.log(to_restore) console.log(to_restore)
let restore_modal = M.Modal.getInstance(document.querySelector('#modal-restore')); restore(to_restore)
restore_modal.open();
}) })
} }
@ -368,10 +365,18 @@
printStatusWithBar('Downloading Snapshot', data.progress); printStatusWithBar('Downloading Snapshot', data.progress);
$('#btn-backup-now, #btn-clean-now').addClass("disabled"); $('#btn-backup-now, #btn-clean-now').addClass("disabled");
break; break;
case "download-b":
printStatusWithBar('Downloading Backup', data.progress);
$('#btn-backup-now, #btn-clean-now').addClass("disabled");
break;
case "upload": case "upload":
printStatusWithBar('Uploading Snapshot', data.progress); printStatusWithBar('Uploading Snapshot', data.progress);
$('#btn-backup-now, #btn-clean-now').addClass("disabled"); $('#btn-backup-now, #btn-clean-now').addClass("disabled");
break; break;
case "upload-b":
printStatusWithBar('Uploading Snapshot', data.progress);
$('#btn-backup-now, #btn-clean-now').addClass("disabled");
break;
case "creating": case "creating":
printStatusWithBar('Creating Snapshot', data.progress); printStatusWithBar('Creating Snapshot', data.progress);
$('#btn-backup-now, #btn-clean-now').addClass("disabled"); $('#btn-backup-now, #btn-clean-now').addClass("disabled");