From ca61d6015d07c70954a3102e92de95fbf60c9fb5 Mon Sep 17 00:00:00 2001 From: SebClem Date: Wed, 14 Aug 2024 16:09:39 +0200 Subject: [PATCH] Add chunked upload progress --- nextcloud_backup/backend/package.json | 1 + nextcloud_backup/backend/pnpm-lock.yaml | 3 ++ .../backend/src/services/webdavService.ts | 53 +++++++++++++------ 3 files changed, 40 insertions(+), 17 deletions(-) diff --git a/nextcloud_backup/backend/package.json b/nextcloud_backup/backend/package.json index f34ab64..e7bfa4c 100644 --- a/nextcloud_backup/backend/package.json +++ b/nextcloud_backup/backend/package.json @@ -29,6 +29,7 @@ "kleur": "^4.1.5", "luxon": "3.5.0", "morgan": "1.10.0", + "url-join": "^5.0.0", "webdav": "5.7.1", "winston": "3.14.1" }, diff --git a/nextcloud_backup/backend/pnpm-lock.yaml b/nextcloud_backup/backend/pnpm-lock.yaml index 2d57deb..d85cbe1 100644 --- a/nextcloud_backup/backend/pnpm-lock.yaml +++ b/nextcloud_backup/backend/pnpm-lock.yaml @@ -59,6 +59,9 @@ importers: morgan: specifier: 1.10.0 version: 1.10.0 + url-join: + specifier: ^5.0.0 + version: 5.0.0 webdav: specifier: 5.7.1 version: 5.7.1 diff --git a/nextcloud_backup/backend/src/services/webdavService.ts b/nextcloud_backup/backend/src/services/webdavService.ts index c7d5174..8e7a193 100644 --- a/nextcloud_backup/backend/src/services/webdavService.ts +++ b/nextcloud_backup/backend/src/services/webdavService.ts @@ -10,6 +10,7 @@ import got, { RequestError, type Method, type PlainResponse, + type Progress, } from "got"; import { DateTime } from "luxon"; import logger from "../config/winston.js"; @@ -25,6 +26,7 @@ import { pipeline } from "stream/promises"; import { humanFileSize } from "../tools/toolbox.js"; import type { BackupConfig } from "../types/services/backupConfig.js"; import path from "path"; +import urlJoin from "url-join"; const CHUNK_SIZE = 5 * 1024 * 1024; // 5MiB Same as desktop client const CHUNK_NUMBER_SIZE = 5; // To add landing "0" @@ -130,7 +132,7 @@ export async function createBackupFolder(conf: WebdavConfig) { function createDirectory(pathToCreate: string, config: WebdavConfig) { const endpoint = getEndpoint(config); - return got(path.join(config.url, endpoint, pathToCreate), { + return got(urlJoin(config.url, endpoint, pathToCreate), { method: "MKCOL" as Method, headers: { authorization: @@ -150,7 +152,7 @@ export function getBackups( return Promise.reject(new Error("Not logged in")); } const endpoint = getEndpoint(config); - return got(path.join(config.url, endpoint, config.backupDir, folder), { + return got(urlJoin(config.url, endpoint, config.backupDir, folder), { method: "PROPFIND" as Method, headers: { authorization: @@ -206,7 +208,7 @@ export function deleteBackup(pathToDelete: string, config: WebdavConfig) { logger.debug(`Deleting Cloud backup ${pathToDelete}`); const endpoint = getEndpoint(config); return got - .delete(path.join(config.url, endpoint, pathToDelete), { + .delete(urlJoin(config.url, endpoint, pathToDelete), { headers: { authorization: "Basic " + @@ -280,7 +282,7 @@ export function webdavUploadFile( }, https: { rejectUnauthorized: !config.allowSelfSignedCerts }, }; - const url = path.join(config.url, getEndpoint(config), webdavPath); + const url = urlJoin(config.url, getEndpoint(config), webdavPath); logger.debug(`...URI: ${encodeURI(url)}`); logger.debug(`...rejectUnauthorized: ${options.https?.rejectUnauthorized}`); @@ -290,7 +292,7 @@ export function webdavUploadFile( statusTools.setStatus(status); got.stream .put(encodeURI(url), options) - .on("uploadProgress", (e) => { + .on("uploadProgress", (e: Progress) => { const percent = e.percent; if (status.progress !== percent) { status.progress = percent; @@ -344,12 +346,8 @@ export async function chunkedUpload( const fileSize = fs.statSync(localPath).size; const chunkEndpoint = getChunkEndpoint(config); - const chunkedUrl = path.join(config.url, chunkEndpoint, uuid); - const finalDestination = path.join( - config.url, - getEndpoint(config), - webdavPath - ); + const chunkedUrl = urlJoin(config.url, chunkEndpoint, uuid); + const finalDestination = urlJoin(config.url, getEndpoint(config), webdavPath); const status = statusTools.getStatus(); status.status = States.BKUP_UPLOAD_CLOUD; status.progress = -1; @@ -385,7 +383,10 @@ export async function chunkedUpload( let end = Math.min(CHUNK_SIZE - 1, fileSize - 1); let current_size = end + 1; - // const uploadedBytes = 0; + const progress = { + before_bytes: 0, + current_bytes: 0, + }; let i = 1; while (start < fileSize - 1) { @@ -393,13 +394,16 @@ export async function chunkedUpload( try { const chunckNumber = i.toString().padStart(CHUNK_NUMBER_SIZE, "0"); await uploadChunk( - path.join(chunkedUrl, chunckNumber), + urlJoin(chunkedUrl, chunckNumber), finalDestination, chunk, current_size, fileSize, - config + config, + progress ); + progress.before_bytes = progress.before_bytes + current_size; + progress.current_bytes = 0; start = end + 1; end = Math.min(start + CHUNK_SIZE - 1, fileSize - 1); current_size = end - start + 1; @@ -467,7 +471,8 @@ function uploadChunk( body: fs.ReadStream, contentLength: number, totalLength: number, - config: WebdavConfig + config: WebdavConfig, + progress: { before_bytes: number; current_bytes: number } ) { return new Promise((resolve, reject) => { logger.debug(`Uploading chunck.`); @@ -490,6 +495,10 @@ function uploadChunk( https: { rejectUnauthorized: !config.allowSelfSignedCerts }, body: body, }) + .on("uploadProgress", (e: Progress) => { + progress.current_bytes = e.transferred; + chunckedUploadProgress(progress, totalLength); + }) .on("response", (res: PlainResponse) => { if (res.ok) { logger.debug("Chunk upload done."); @@ -506,6 +515,16 @@ function uploadChunk( }); } +function chunckedUploadProgress( + progress: { before_bytes: number; current_bytes: number }, + totalLength: number +) { + const current = progress.before_bytes + progress.current_bytes; + const status = statusTools.getStatus(); + status.progress = current / totalLength; + statusTools.setStatus(status); +} + function initChunkedUpload( url: string, finalDestination: string, @@ -532,7 +551,7 @@ function assembleChunkedUpload( totalLength: number, config: WebdavConfig ) { - const chunckFile = path.join(url, ".file"); + const chunckFile = urlJoin(url, ".file"); logger.info(`Assemble chuncked upload.`); logger.debug(`...URI: ${encodeURI(chunckFile)}`); logger.debug(`...Final destination: ${encodeURI(finalDestination)}`); @@ -568,7 +587,7 @@ export function downloadFile( }, https: { rejectUnauthorized: !config.allowSelfSignedCerts }, }; - const url = path.join(config.url, getEndpoint(config), webdavPath); + const url = urlJoin(config.url, getEndpoint(config), webdavPath); logger.debug(`...URI: ${encodeURI(url)}`); logger.debug(`...rejectUnauthorized: ${options.https?.rejectUnauthorized}`); const status = statusTools.getStatus();