From 5705c3909d3c764055dece809b2a7a22aafd4c75 Mon Sep 17 00:00:00 2001 From: aza547 Date: Fri, 14 Jun 2024 11:46:39 +0100 Subject: [PATCH] fixes #485 --- CHANGELOG.md | 2 + src/main/VideoProcessQueue.ts | 15 +++++-- src/main/configSchema.ts | 13 ++++++ src/renderer/CloudSettings.tsx | 76 ++++++++++++++++++++++++++++++++++ src/renderer/SettingsPage.tsx | 1 + src/renderer/useSettings.ts | 2 + src/storage/CloudClient.ts | 30 ++++++++++++-- 7 files changed, 133 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 505a5957..083e05b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased ### Changed ### Added +- [Issue 485](https://github.com/aza547/wow-recorder/issues/485) - Added upload rate limit setting. + ### Fixed ## [5.5.0] - 2024-06-12 diff --git a/src/main/VideoProcessQueue.ts b/src/main/VideoProcessQueue.ts index a3e0f1db..ecf01882 100644 --- a/src/main/VideoProcessQueue.ts +++ b/src/main/VideoProcessQueue.ts @@ -251,6 +251,11 @@ export default class VideoProcessQueue { ): Promise { let lastProgress = 0; + // Decide if we need to use a rate limit or not. Setting to -1 is unlimited. + const rateLimit = this.cfg.get('cloudUploadRateLimit') + ? this.cfg.get('cloudUploadRateLimitMbps') + : -1; + const progressCallback = (progress: number) => { if (progress === lastProgress) { return; @@ -262,13 +267,17 @@ export default class VideoProcessQueue { try { assert(this.cloudClient); - const thumbNailPath = getThumbnailFileNameForVideo(item.path); // Upload the video first, this can take a bit of time, and don't want // to confuse the frontend by having metadata without video. - await this.cloudClient.putFile(item.path, progressCallback); + await this.cloudClient.putFile(item.path, rateLimit, progressCallback); progressCallback(100); - await this.cloudClient.putFile(thumbNailPath); + + // Upload the thumbnail. + const thumbNailPath = getThumbnailFileNameForVideo(item.path); + await this.cloudClient.putFile(thumbNailPath, rateLimit); + + // Now add the metadata. const metadata = await getMetadataForVideo(item.path); const cloudMetadata: CloudMetadata = { diff --git a/src/main/configSchema.ts b/src/main/configSchema.ts index b8da7af1..c5030bdf 100644 --- a/src/main/configSchema.ts +++ b/src/main/configSchema.ts @@ -57,6 +57,8 @@ export type ConfigurationSchema = { dungeonOverrun: number; cloudStorage: boolean; cloudUpload: boolean; + cloudUploadRateLimit: boolean; + cloudUploadRateLimitMbps: number; cloudAccountName: string; cloudAccountPassword: string; cloudGuildName: string; @@ -402,6 +404,17 @@ export const configSchema = { type: 'boolean', default: false, }, + cloudUploadRateLimit: { + description: + 'If upload to the cloud should be rate limited. Useful if you are finding uploading is causing you to lag.', + type: 'boolean', + default: false, + }, + cloudUploadRateLimitMbps: { + description: 'The upload rate limit in MB/s ', + type: 'integer', + default: 100, + }, cloudAccountName: { description: 'Your Warcraft Recorder account username.', type: 'string', diff --git a/src/renderer/CloudSettings.tsx b/src/renderer/CloudSettings.tsx index 55cd217d..36f7d59f 100644 --- a/src/renderer/CloudSettings.tsx +++ b/src/renderer/CloudSettings.tsx @@ -76,6 +76,8 @@ const CloudSettings = (props: IProps) => { cloudAccountPassword: config.cloudAccountPassword, cloudGuildName: config.cloudGuildName, cloudUpload: config.cloudUpload, + cloudUploadRateLimit: config.cloudUploadRateLimit, + cloudUploadRateLimitMbps: config.cloudUploadRateLimitMbps, cloudUpload2v2: config.cloudUpload2v2, cloudUpload3v3: config.cloudUpload3v3, cloudUpload5v5: config.cloudUpload5v5, @@ -99,6 +101,8 @@ const CloudSettings = (props: IProps) => { config.cloudAccountPassword, config.cloudGuildName, config.cloudUpload, + config.cloudUploadRateLimit, + config.cloudUploadRateLimitMbps, config.cloudUpload2v2, config.cloudUpload3v3, config.cloudUpload5v5, @@ -257,6 +261,10 @@ const CloudSettings = (props: IProps) => { }; const setMinKeystoneLevel = (event: React.ChangeEvent) => { + if (!event.target.value) { + return; + } + setConfig((prevState) => { return { ...prevState, @@ -350,6 +358,35 @@ const CloudSettings = (props: IProps) => { ); }; + const setCloudUploadRateLimit = ( + event: React.ChangeEvent + ) => { + setConfig((prevState) => { + return { + ...prevState, + cloudUploadRateLimit: event.target.checked, + }; + }); + }; + + const getCloudUploadRateLimitSwitch = () => { + if (isComponentDisabled() || !config.cloudUpload) { + return <>; + } + + return ( + + + + ); + }; + const setCloudAccountName = async ( event: React.ChangeEvent ) => { @@ -560,6 +597,43 @@ const CloudSettings = (props: IProps) => { ); }; + const setUploadRateLimit = (event: React.ChangeEvent) => { + if (!event.target.value) { + return; + } + + setConfig((prevState) => { + return { + ...prevState, + cloudUploadRateLimitMbps: parseInt(event.target.value, 10), + }; + }); + }; + + const getRateLimitField = () => { + if (!config.cloudUploadRateLimit) { + return <>; + } + + const helperText = + config.cloudUploadRateLimitMbps < 1 ? 'Must be 1 or greater' : ''; + + return ( + + ); + }; + return ( {getDisabledText()} @@ -567,6 +641,8 @@ const CloudSettings = (props: IProps) => { {getCloudSwitch()} {getCloudUploadSwitch()} + {getCloudUploadRateLimitSwitch()} + {getRateLimitField()} diff --git a/src/renderer/SettingsPage.tsx b/src/renderer/SettingsPage.tsx index 490a40f1..fd7a7452 100644 --- a/src/renderer/SettingsPage.tsx +++ b/src/renderer/SettingsPage.tsx @@ -98,6 +98,7 @@ const getCloudSettingsInfoIcon = () => { /* eslint-disable prettier/prettier */ ['Cloud Playback', configSchema.cloudStorage.description].join('\n'), ['Cloud Upload', configSchema.cloudUpload.description].join('\n'), + ['Upload Rate Limit', configSchema.cloudUploadRateLimit.description].join('\n'), ['Account Name', configSchema.cloudAccountName.description].join('\n'), ['Account Password', configSchema.cloudAccountPassword.description].join('\n'), ['Guild Name', configSchema.cloudGuildName.description].join('\n'), diff --git a/src/renderer/useSettings.ts b/src/renderer/useSettings.ts index d6b5fee0..c0acae3f 100644 --- a/src/renderer/useSettings.ts +++ b/src/renderer/useSettings.ts @@ -77,6 +77,8 @@ export const getSettings = (): ConfigurationSchema => { dungeonOverrun: getConfigValue('dungeonOverrun'), cloudStorage: getConfigValue('cloudStorage'), cloudUpload: getConfigValue('cloudUpload'), + cloudUploadRateLimit: getConfigValue('cloudUploadRateLimit'), + cloudUploadRateLimitMbps: getConfigValue('cloudUploadRateLimitMbps'), cloudAccountName: getConfigValue('cloudAccountName'), cloudAccountPassword: getConfigValue('cloudAccountPassword'), cloudGuildName: getConfigValue('cloudGuildName'), diff --git a/src/storage/CloudClient.ts b/src/storage/CloudClient.ts index 87c107c8..cf83bc38 100644 --- a/src/storage/CloudClient.ts +++ b/src/storage/CloudClient.ts @@ -380,20 +380,33 @@ export default class CloudClient extends EventEmitter { /** * Write a file into R2. + * + * @param file file path to upload + * @param rate rate of upload in MB/s, or -1 to signify no limit */ public async putFile( file: string, + rate = -1, // eslint-disable-next-line @typescript-eslint/no-unused-vars progressCallback = (_progress: number) => {} ) { const key = path.basename(file); - console.info('[CloudClient] Uploading', file, 'to', key); + + console.info( + '[CloudClient] Uploading', + file, + 'to', + key, + 'with rate limit', + rate + ); + const stats = await fs.promises.stat(file); if (stats.size < this.multiPartSizeBytes) { - await this.doSinglePartUpload(file, progressCallback); + await this.doSinglePartUpload(file, rate, progressCallback); } else { - await this.doMultiPartUpload(file, progressCallback); + await this.doMultiPartUpload(file, rate, progressCallback); } await this.updateLastMod(); @@ -620,6 +633,7 @@ export default class CloudClient extends EventEmitter { */ private async doSinglePartUpload( file: string, + rate: number, // eslint-disable-next-line @typescript-eslint/no-unused-vars progressCallback = (_progress: number) => {} ) { @@ -636,6 +650,10 @@ export default class CloudClient extends EventEmitter { // into memory which is just a disaster. This makes me want to pick // a different HTTP library. https://github.com/axios/axios/issues/1045. maxRedirects: 0, + + // Apply the rate limit here if it's in-play. + // Convert units from MB/s to bytes per sec. + maxRate: rate > 0 ? rate * 1024 ** 2 : undefined, }; const signedUrl = await this.signPutUrl(key, stats.size); @@ -690,6 +708,7 @@ export default class CloudClient extends EventEmitter { */ private async doMultiPartUpload( file: string, + rate: number, // eslint-disable-next-line @typescript-eslint/no-unused-vars progressCallback = (_progress: number) => {} ) { @@ -737,10 +756,15 @@ export default class CloudClient extends EventEmitter { const actual = Math.round(previous + normalized); progressCallback(actual); }, + // Without this, we buffer the whole file (which can be several GB) // into memory which is just a disaster. This makes me want to pick // a different HTTP library. https://github.com/axios/axios/issues/1045. maxRedirects: 0, + + // Apply the rate limit here if it's in-play. + // Convert units from MB/s to bytes per sec. + maxRate: rate > 0 ? rate * 1024 ** 2 : undefined, }; // Retry each part upload a few times on failure to allow for