mirror of
https://github.com/Sebclem/hassio-nextcloud-backup.git
synced 2024-11-12 20:42:58 +01:00
Add chunked upload progress
This commit is contained in:
parent
4f88ca4ed2
commit
ca61d6015d
@ -29,6 +29,7 @@
|
|||||||
"kleur": "^4.1.5",
|
"kleur": "^4.1.5",
|
||||||
"luxon": "3.5.0",
|
"luxon": "3.5.0",
|
||||||
"morgan": "1.10.0",
|
"morgan": "1.10.0",
|
||||||
|
"url-join": "^5.0.0",
|
||||||
"webdav": "5.7.1",
|
"webdav": "5.7.1",
|
||||||
"winston": "3.14.1"
|
"winston": "3.14.1"
|
||||||
},
|
},
|
||||||
|
@ -59,6 +59,9 @@ importers:
|
|||||||
morgan:
|
morgan:
|
||||||
specifier: 1.10.0
|
specifier: 1.10.0
|
||||||
version: 1.10.0
|
version: 1.10.0
|
||||||
|
url-join:
|
||||||
|
specifier: ^5.0.0
|
||||||
|
version: 5.0.0
|
||||||
webdav:
|
webdav:
|
||||||
specifier: 5.7.1
|
specifier: 5.7.1
|
||||||
version: 5.7.1
|
version: 5.7.1
|
||||||
|
@ -10,6 +10,7 @@ import got, {
|
|||||||
RequestError,
|
RequestError,
|
||||||
type Method,
|
type Method,
|
||||||
type PlainResponse,
|
type PlainResponse,
|
||||||
|
type Progress,
|
||||||
} from "got";
|
} from "got";
|
||||||
import { DateTime } from "luxon";
|
import { DateTime } from "luxon";
|
||||||
import logger from "../config/winston.js";
|
import logger from "../config/winston.js";
|
||||||
@ -25,6 +26,7 @@ import { pipeline } from "stream/promises";
|
|||||||
import { humanFileSize } from "../tools/toolbox.js";
|
import { humanFileSize } from "../tools/toolbox.js";
|
||||||
import type { BackupConfig } from "../types/services/backupConfig.js";
|
import type { BackupConfig } from "../types/services/backupConfig.js";
|
||||||
import path from "path";
|
import path from "path";
|
||||||
|
import urlJoin from "url-join";
|
||||||
|
|
||||||
const CHUNK_SIZE = 5 * 1024 * 1024; // 5MiB Same as desktop client
|
const CHUNK_SIZE = 5 * 1024 * 1024; // 5MiB Same as desktop client
|
||||||
const CHUNK_NUMBER_SIZE = 5; // To add landing "0"
|
const CHUNK_NUMBER_SIZE = 5; // To add landing "0"
|
||||||
@ -130,7 +132,7 @@ export async function createBackupFolder(conf: WebdavConfig) {
|
|||||||
|
|
||||||
function createDirectory(pathToCreate: string, config: WebdavConfig) {
|
function createDirectory(pathToCreate: string, config: WebdavConfig) {
|
||||||
const endpoint = getEndpoint(config);
|
const endpoint = getEndpoint(config);
|
||||||
return got(path.join(config.url, endpoint, pathToCreate), {
|
return got(urlJoin(config.url, endpoint, pathToCreate), {
|
||||||
method: "MKCOL" as Method,
|
method: "MKCOL" as Method,
|
||||||
headers: {
|
headers: {
|
||||||
authorization:
|
authorization:
|
||||||
@ -150,7 +152,7 @@ export function getBackups(
|
|||||||
return Promise.reject(new Error("Not logged in"));
|
return Promise.reject(new Error("Not logged in"));
|
||||||
}
|
}
|
||||||
const endpoint = getEndpoint(config);
|
const endpoint = getEndpoint(config);
|
||||||
return got(path.join(config.url, endpoint, config.backupDir, folder), {
|
return got(urlJoin(config.url, endpoint, config.backupDir, folder), {
|
||||||
method: "PROPFIND" as Method,
|
method: "PROPFIND" as Method,
|
||||||
headers: {
|
headers: {
|
||||||
authorization:
|
authorization:
|
||||||
@ -206,7 +208,7 @@ export function deleteBackup(pathToDelete: string, config: WebdavConfig) {
|
|||||||
logger.debug(`Deleting Cloud backup ${pathToDelete}`);
|
logger.debug(`Deleting Cloud backup ${pathToDelete}`);
|
||||||
const endpoint = getEndpoint(config);
|
const endpoint = getEndpoint(config);
|
||||||
return got
|
return got
|
||||||
.delete(path.join(config.url, endpoint, pathToDelete), {
|
.delete(urlJoin(config.url, endpoint, pathToDelete), {
|
||||||
headers: {
|
headers: {
|
||||||
authorization:
|
authorization:
|
||||||
"Basic " +
|
"Basic " +
|
||||||
@ -280,7 +282,7 @@ export function webdavUploadFile(
|
|||||||
},
|
},
|
||||||
https: { rejectUnauthorized: !config.allowSelfSignedCerts },
|
https: { rejectUnauthorized: !config.allowSelfSignedCerts },
|
||||||
};
|
};
|
||||||
const url = path.join(config.url, getEndpoint(config), webdavPath);
|
const url = urlJoin(config.url, getEndpoint(config), webdavPath);
|
||||||
|
|
||||||
logger.debug(`...URI: ${encodeURI(url)}`);
|
logger.debug(`...URI: ${encodeURI(url)}`);
|
||||||
logger.debug(`...rejectUnauthorized: ${options.https?.rejectUnauthorized}`);
|
logger.debug(`...rejectUnauthorized: ${options.https?.rejectUnauthorized}`);
|
||||||
@ -290,7 +292,7 @@ export function webdavUploadFile(
|
|||||||
statusTools.setStatus(status);
|
statusTools.setStatus(status);
|
||||||
got.stream
|
got.stream
|
||||||
.put(encodeURI(url), options)
|
.put(encodeURI(url), options)
|
||||||
.on("uploadProgress", (e) => {
|
.on("uploadProgress", (e: Progress) => {
|
||||||
const percent = e.percent;
|
const percent = e.percent;
|
||||||
if (status.progress !== percent) {
|
if (status.progress !== percent) {
|
||||||
status.progress = percent;
|
status.progress = percent;
|
||||||
@ -344,12 +346,8 @@ export async function chunkedUpload(
|
|||||||
const fileSize = fs.statSync(localPath).size;
|
const fileSize = fs.statSync(localPath).size;
|
||||||
|
|
||||||
const chunkEndpoint = getChunkEndpoint(config);
|
const chunkEndpoint = getChunkEndpoint(config);
|
||||||
const chunkedUrl = path.join(config.url, chunkEndpoint, uuid);
|
const chunkedUrl = urlJoin(config.url, chunkEndpoint, uuid);
|
||||||
const finalDestination = path.join(
|
const finalDestination = urlJoin(config.url, getEndpoint(config), webdavPath);
|
||||||
config.url,
|
|
||||||
getEndpoint(config),
|
|
||||||
webdavPath
|
|
||||||
);
|
|
||||||
const status = statusTools.getStatus();
|
const status = statusTools.getStatus();
|
||||||
status.status = States.BKUP_UPLOAD_CLOUD;
|
status.status = States.BKUP_UPLOAD_CLOUD;
|
||||||
status.progress = -1;
|
status.progress = -1;
|
||||||
@ -385,7 +383,10 @@ export async function chunkedUpload(
|
|||||||
let end = Math.min(CHUNK_SIZE - 1, fileSize - 1);
|
let end = Math.min(CHUNK_SIZE - 1, fileSize - 1);
|
||||||
|
|
||||||
let current_size = end + 1;
|
let current_size = end + 1;
|
||||||
// const uploadedBytes = 0;
|
const progress = {
|
||||||
|
before_bytes: 0,
|
||||||
|
current_bytes: 0,
|
||||||
|
};
|
||||||
|
|
||||||
let i = 1;
|
let i = 1;
|
||||||
while (start < fileSize - 1) {
|
while (start < fileSize - 1) {
|
||||||
@ -393,13 +394,16 @@ export async function chunkedUpload(
|
|||||||
try {
|
try {
|
||||||
const chunckNumber = i.toString().padStart(CHUNK_NUMBER_SIZE, "0");
|
const chunckNumber = i.toString().padStart(CHUNK_NUMBER_SIZE, "0");
|
||||||
await uploadChunk(
|
await uploadChunk(
|
||||||
path.join(chunkedUrl, chunckNumber),
|
urlJoin(chunkedUrl, chunckNumber),
|
||||||
finalDestination,
|
finalDestination,
|
||||||
chunk,
|
chunk,
|
||||||
current_size,
|
current_size,
|
||||||
fileSize,
|
fileSize,
|
||||||
config
|
config,
|
||||||
|
progress
|
||||||
);
|
);
|
||||||
|
progress.before_bytes = progress.before_bytes + current_size;
|
||||||
|
progress.current_bytes = 0;
|
||||||
start = end + 1;
|
start = end + 1;
|
||||||
end = Math.min(start + CHUNK_SIZE - 1, fileSize - 1);
|
end = Math.min(start + CHUNK_SIZE - 1, fileSize - 1);
|
||||||
current_size = end - start + 1;
|
current_size = end - start + 1;
|
||||||
@ -467,7 +471,8 @@ function uploadChunk(
|
|||||||
body: fs.ReadStream,
|
body: fs.ReadStream,
|
||||||
contentLength: number,
|
contentLength: number,
|
||||||
totalLength: number,
|
totalLength: number,
|
||||||
config: WebdavConfig
|
config: WebdavConfig,
|
||||||
|
progress: { before_bytes: number; current_bytes: number }
|
||||||
) {
|
) {
|
||||||
return new Promise<PlainResponse>((resolve, reject) => {
|
return new Promise<PlainResponse>((resolve, reject) => {
|
||||||
logger.debug(`Uploading chunck.`);
|
logger.debug(`Uploading chunck.`);
|
||||||
@ -490,6 +495,10 @@ function uploadChunk(
|
|||||||
https: { rejectUnauthorized: !config.allowSelfSignedCerts },
|
https: { rejectUnauthorized: !config.allowSelfSignedCerts },
|
||||||
body: body,
|
body: body,
|
||||||
})
|
})
|
||||||
|
.on("uploadProgress", (e: Progress) => {
|
||||||
|
progress.current_bytes = e.transferred;
|
||||||
|
chunckedUploadProgress(progress, totalLength);
|
||||||
|
})
|
||||||
.on("response", (res: PlainResponse) => {
|
.on("response", (res: PlainResponse) => {
|
||||||
if (res.ok) {
|
if (res.ok) {
|
||||||
logger.debug("Chunk upload done.");
|
logger.debug("Chunk upload done.");
|
||||||
@ -506,6 +515,16 @@ function uploadChunk(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function chunckedUploadProgress(
|
||||||
|
progress: { before_bytes: number; current_bytes: number },
|
||||||
|
totalLength: number
|
||||||
|
) {
|
||||||
|
const current = progress.before_bytes + progress.current_bytes;
|
||||||
|
const status = statusTools.getStatus();
|
||||||
|
status.progress = current / totalLength;
|
||||||
|
statusTools.setStatus(status);
|
||||||
|
}
|
||||||
|
|
||||||
function initChunkedUpload(
|
function initChunkedUpload(
|
||||||
url: string,
|
url: string,
|
||||||
finalDestination: string,
|
finalDestination: string,
|
||||||
@ -532,7 +551,7 @@ function assembleChunkedUpload(
|
|||||||
totalLength: number,
|
totalLength: number,
|
||||||
config: WebdavConfig
|
config: WebdavConfig
|
||||||
) {
|
) {
|
||||||
const chunckFile = path.join(url, ".file");
|
const chunckFile = urlJoin(url, ".file");
|
||||||
logger.info(`Assemble chuncked upload.`);
|
logger.info(`Assemble chuncked upload.`);
|
||||||
logger.debug(`...URI: ${encodeURI(chunckFile)}`);
|
logger.debug(`...URI: ${encodeURI(chunckFile)}`);
|
||||||
logger.debug(`...Final destination: ${encodeURI(finalDestination)}`);
|
logger.debug(`...Final destination: ${encodeURI(finalDestination)}`);
|
||||||
@ -568,7 +587,7 @@ export function downloadFile(
|
|||||||
},
|
},
|
||||||
https: { rejectUnauthorized: !config.allowSelfSignedCerts },
|
https: { rejectUnauthorized: !config.allowSelfSignedCerts },
|
||||||
};
|
};
|
||||||
const url = path.join(config.url, getEndpoint(config), webdavPath);
|
const url = urlJoin(config.url, getEndpoint(config), webdavPath);
|
||||||
logger.debug(`...URI: ${encodeURI(url)}`);
|
logger.debug(`...URI: ${encodeURI(url)}`);
|
||||||
logger.debug(`...rejectUnauthorized: ${options.https?.rejectUnauthorized}`);
|
logger.debug(`...rejectUnauthorized: ${options.https?.rejectUnauthorized}`);
|
||||||
const status = statusTools.getStatus();
|
const status = statusTools.getStatus();
|
||||||
|
Loading…
Reference in New Issue
Block a user