From 0ddf7bdb068760e20df7bb88b234fff77fe61253 Mon Sep 17 00:00:00 2001 From: SebClem Date: Wed, 10 Jul 2024 17:41:00 +0200 Subject: [PATCH] First version of functional chunked upload --- .../backend/src/services/orchestrator.ts | 7 +- .../src/services/webdavConfigService.ts | 25 +++ .../backend/src/services/webdavService.ts | 156 +++++++++++++++++- .../src/types/services/webdavConfig.ts | 14 +- .../types/services/webdavConfigValidation.ts | 5 + 5 files changed, 192 insertions(+), 15 deletions(-) diff --git a/nextcloud_backup/backend/src/services/orchestrator.ts b/nextcloud_backup/backend/src/services/orchestrator.ts index df550a9..fa3a82e 100644 --- a/nextcloud_backup/backend/src/services/orchestrator.ts +++ b/nextcloud_backup/backend/src/services/orchestrator.ts @@ -71,11 +71,16 @@ export function doBackupWorkflow(type: WorkflowType) { }) .then((tmpFile) => { tmpBackupFile = tmpFile; - return webDavService.webdavUploadFile( + return webDavService.chunkedUpload( tmpFile, getBackupFolder(type, webdavConfig) + name, webdavConfig ); + // return webDavService.webdavUploadFile( + // tmpFile, + // getBackupFolder(type, webdavConfig) + name, + // webdavConfig + // ); }) .then(() => { logger.info("Backup workflow finished successfully !"); diff --git a/nextcloud_backup/backend/src/services/webdavConfigService.ts b/nextcloud_backup/backend/src/services/webdavConfigService.ts index 44ccbec..fca98ed 100644 --- a/nextcloud_backup/backend/src/services/webdavConfigService.ts +++ b/nextcloud_backup/backend/src/services/webdavConfigService.ts @@ -14,6 +14,7 @@ import e from "express"; const webdavConfigPath = "/data/webdavConfigV2.json"; const NEXTCLOUD_ENDPOINT = "/remote.php/dav/files/$username"; +const NEXTCLOUD_CHUNK_ENDPOINT = "/remote.php/dav/uploads/$username"; export function validateWebdavConfig(config: WebdavConfig) { const validator = Joi.object(WebdavConfigValidation); @@ -61,6 +62,30 @@ export function getEndpoint(config: WebdavConfig) { return endpoint; } +export function getChunkEndpoint(config: WebdavConfig) { + let endpoint: string; + + if (config.webdavEndpoint.type == WebdavEndpointType.NEXTCLOUD) { + endpoint = NEXTCLOUD_CHUNK_ENDPOINT.replace("$username", config.username); + } else if (config.webdavEndpoint.customChunkEndpoint) { + endpoint = config.webdavEndpoint.customChunkEndpoint.replace( + "$username", + config.username + ); + } else { + return ""; + } + if (!endpoint.startsWith("/")) { + endpoint = "/" + endpoint; + } + + if (!endpoint.endsWith("/")) { + return endpoint + "/"; + } + + return endpoint; +} + export function getBackupFolder(type: WorkflowType, config: WebdavConfig) { const end = type == WorkflowType.AUTO ? pathTools.auto : pathTools.manual; return config.backupDir.endsWith("/") diff --git a/nextcloud_backup/backend/src/services/webdavService.ts b/nextcloud_backup/backend/src/services/webdavService.ts index c3c7340..cad7f85 100644 --- a/nextcloud_backup/backend/src/services/webdavService.ts +++ b/nextcloud_backup/backend/src/services/webdavService.ts @@ -1,9 +1,10 @@ import { XMLParser } from "fast-xml-parser"; -import { createReadStream, stat, statSync, unlinkSync } from "fs"; +import fs from "fs"; import got, { HTTPError, RequestError, type Method, + type OptionsInit, type PlainResponse, } from "got"; import { DateTime } from "luxon"; @@ -14,9 +15,14 @@ import * as statusTools from "../tools/status.js"; import type { WebdavBackup } from "../types/services/webdav.js"; import type { WebdavConfig } from "../types/services/webdavConfig.js"; import { templateToRegexp } from "./backupConfigService.js"; -import { getEndpoint } from "./webdavConfigService.js"; +import { getChunkEndpoint, getEndpoint } from "./webdavConfigService.js"; import { States } from "../types/status.js"; +import { randomUUID } from "crypto"; +import e, { response } from "express"; +import { NONAME } from "dns"; +const CHUNK_SIZE = 5 * 1024 * 1024; // 5MiB Same as desktop client +const CHUNK_NUMBER_SIZE = 5; // To add landing "0" const PROPFIND_BODY = '\ \ @@ -41,6 +47,7 @@ export function checkWebdavLogin( "Basic " + Buffer.from(config.username + ":" + config.password).toString("base64"), }, + https: { rejectUnauthorized: !config.allowSelfSignedCerts }, }).then( (response) => { const status = statusTools.getStatus(); @@ -146,6 +153,7 @@ export function getBackups( Buffer.from(config.username + ":" + config.password).toString("base64"), Depth: "1", }, + https: { rejectUnauthorized: !config.allowSelfSignedCerts }, body: PROPFIND_BODY, }).then( (value) => { @@ -200,6 +208,7 @@ export function deleteBackup(path: string, config: WebdavConfig) { "base64" ), }, + https: { rejectUnauthorized: !config.allowSelfSignedCerts }, }) .then( (response) => { @@ -254,8 +263,8 @@ export function webdavUploadFile( return new Promise((resolve, reject) => { logger.info(`Uploading ${localPath} to webdav...`); - const stats = statSync(localPath); - const stream = createReadStream(localPath); + const stats = fs.statSync(localPath); + const stream = fs.createReadStream(localPath); const options = { body: stream, headers: { @@ -301,11 +310,11 @@ export function webdavUploadFile( logger.error(`Fail to upload file to Cloud`); logger.error(`Code: ${res.statusCode}`); logger.error(`Body: ${res.body}`); - unlinkSync(localPath); + fs.unlinkSync(localPath); reject(res); } else { logger.info(`...Upload finish ! (status: ${res.statusCode})`); - unlinkSync(localPath); + fs.unlinkSync(localPath); resolve(undefined); } }) @@ -317,12 +326,145 @@ export function webdavUploadFile( messageManager.error("Fail to upload backup to Cloud", err.message); logger.error("Fail to upload backup to Cloud"); logger.error(err); - unlinkSync(localPath); + fs.unlinkSync(localPath); reject(err); }); }); } +export function chunkedUpload( + localPath: string, + webdavPath: string, + config: WebdavConfig +) { + return new Promise(async (resolve, reject) => { + const uuid = randomUUID(); + const fileSize = fs.statSync(localPath).size; + + const chunkEndpoint = getChunkEndpoint(config); + const chunkedUrl = config.url + chunkEndpoint + uuid; + const finalDestination = config.url + getEndpoint(config) + webdavPath; + await initChunkedUpload(chunkedUrl, finalDestination, config); + + let start = 0; + let end = fileSize > CHUNK_SIZE ? CHUNK_SIZE : fileSize; + let current_size = end; + let uploadedBytes = 0; + + let i = 0; + while (start < fileSize) { + const chunk = fs.createReadStream(localPath, { start, end }); + try { + const chunckNumber = i.toString().padStart(CHUNK_NUMBER_SIZE, "0"); + await uploadChunk( + chunkedUrl + `/${chunckNumber}`, + finalDestination, + chunk, + current_size, + fileSize, + config + ); + start = end; + end = Math.min(start + CHUNK_SIZE, fileSize - 1); + current_size = end - start; + i++; + } catch (error) { + reject(); + return; + } + } + logger.debug("Chunked upload funished, assembling chunks."); + assembleChunkedUpload(chunkedUrl, finalDestination, fileSize, config); + resolve(); + }); +} + +export function uploadChunk( + url: string, + finalDestination: string, + body: fs.ReadStream, + contentLength: number, + totalLength: number, + config: WebdavConfig +) { + return new Promise((resolve, reject) => { + logger.debug(`Uploading chunck.`); + logger.debug(`...URI: ${encodeURI(url)}`); + logger.debug(`...Final destination: ${encodeURI(finalDestination)}`); + logger.debug(`...Chunk size: ${contentLength}`); + got.stream + .put(url, { + headers: { + authorization: + "Basic " + + Buffer.from(config.username + ":" + config.password).toString( + "base64" + ), + Destination: encodeURI(finalDestination), + "OC-Total-Length": totalLength.toString(), + "content-length": contentLength.toString(), + }, + https: { rejectUnauthorized: !config.allowSelfSignedCerts }, + body: body, + }) + .on("response", (res: PlainResponse) => { + if (res.ok) { + logger.debug("Chunk upload done."); + resolve(res); + } else { + logger.error(`Fail to upload chunk: ${res.statusCode}`); + reject(res); + } + }) + .on("error", (err) => { + logger.error(`Fail to upload chunk: ${err.message}`); + reject(err); + }); + }); +} + +export function initChunkedUpload( + url: string, + finalDestination: string, + config: WebdavConfig +) { + logger.debug(`Init chuncked upload.`); + logger.debug(`...URI: ${encodeURI(url)}`); + logger.debug(`...Final destination: ${encodeURI(finalDestination)}`); + return got(encodeURI(url), { + method: "MKCOL" as Method, + headers: { + authorization: + "Basic " + + Buffer.from(config.username + ":" + config.password).toString("base64"), + Destination: encodeURI(finalDestination), + }, + https: { rejectUnauthorized: !config.allowSelfSignedCerts }, + }); +} + +export function assembleChunkedUpload( + url: string, + finalDestination: string, + totalLength: number, + config: WebdavConfig +) { + let chunckFile = `${url}/.file`; + logger.debug(`Assemble chuncked upload.`); + logger.debug(`...URI: ${encodeURI(chunckFile)}`); + logger.debug(`...Final destination: ${encodeURI(finalDestination)}`); + return got(encodeURI(chunckFile), { + method: "MOVE" as Method, + headers: { + authorization: + "Basic " + + Buffer.from(config.username + ":" + config.password).toString("base64"), + Destination: encodeURI(finalDestination), + "OC-Total-Length": totalLength.toString(), + }, + https: { rejectUnauthorized: !config.allowSelfSignedCerts }, + }); +} // import fs from "fs"; // import got from "got"; // import https from "https"; diff --git a/nextcloud_backup/backend/src/types/services/webdavConfig.ts b/nextcloud_backup/backend/src/types/services/webdavConfig.ts index 18d1d1e..71adc0f 100644 --- a/nextcloud_backup/backend/src/types/services/webdavConfig.ts +++ b/nextcloud_backup/backend/src/types/services/webdavConfig.ts @@ -1,18 +1,18 @@ export enum WebdavEndpointType { NEXTCLOUD = "NEXTCLOUD", - CUSTOM = "CUSTOM" + CUSTOM = "CUSTOM", } - export interface WebdavConfig { url: string; username: string; password: string; backupDir: string; - allowSelfSignedCerts: boolean - chunckedUpload: boolean + allowSelfSignedCerts: boolean; + chunckedUpload: boolean; webdavEndpoint: { type: WebdavEndpointType; - customEndpoint?: string; - } -} \ No newline at end of file + customEndpoint?: string; + customChunkEndpoint?: string; + }; +} diff --git a/nextcloud_backup/backend/src/types/services/webdavConfigValidation.ts b/nextcloud_backup/backend/src/types/services/webdavConfigValidation.ts index 36cb314..945a43a 100644 --- a/nextcloud_backup/backend/src/types/services/webdavConfigValidation.ts +++ b/nextcloud_backup/backend/src/types/services/webdavConfigValidation.ts @@ -15,6 +15,11 @@ const WebdavConfigValidation = { is: WebdavEndpointType.CUSTOM, then: Joi.string().not().empty().required, otherwise: Joi.disallow() + }), + customChunkEndpoint: Joi.alternatives().conditional("type", { + is: WebdavEndpointType.CUSTOM, + then: Joi.string().not().empty().required, + otherwise: Joi.disallow() }) }).required().label("Webdav endpoint"), }