feat: v3.10.0

CHANGELOG.md

@@ -2,6 +2,13 @@
 
 All notable changes to this project will be documented in this file.
 
+### 3.10.0 (2022-05-15)
+
+**Feature**
+
+- AliYun OSS to AWS S3
+- Remove STSToken API for cloud storage
+
 ### 3.9.4 (2022-04-12)
 
 **Feature**

package.json

@@ -1,6 +1,6 @@
 {
   "name": "nodepress",
-  "version": "3.9.4",
+  "version": "3.10.0",
   "description": "RESTful API service for Surmon.me blog",
   "author": {
     "name": "Surmon",
@@ -35,6 +35,7 @@
     "release": ". ./scripts/release.sh"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.88.0",
     "@nestjs/axios": "^0.0.3",
     "@nestjs/common": "^8.2.4",
     "@nestjs/core": "^8.2.4",
@@ -46,7 +47,6 @@
     "@typegoose/auto-increment": "^1.2.0",
     "@typegoose/typegoose": "^9.7.0",
     "akismet-api": "^5.3.0",
-    "ali-oss": "^6.16.0",
     "axios": "^0.24.0",
     "body-parser": "^1.19.1",
     "cache-manager": "^3.6.0",
@@ -81,7 +81,6 @@
     "@nestjs/cli": "^8.1.5",
     "@nestjs/schematics": "^8.0.5",
     "@nestjs/testing": "^8.2.4",
-    "@types/ali-oss": "^6.16.2",
     "@types/cache-manager": "^3.4.2",
     "@types/cookie-parser": "^1.4.2",
     "@types/express": "^4.17.13",

src/app.config.ts

@@ -20,6 +20,7 @@ export const APP = {
   ADMIN_EMAIL: (argv.admin_email as string) || 'admin email, e.g. [email protected]',
   FE_NAME: 'Surmon.me',
   FE_URL: 'https://surmon.me',
+  STATIC_URL: 'https://static.surmon.me',
 }
 
 export const PROJECT = {
@@ -87,13 +88,15 @@ export const GOOGLE = {
   serverAccountFilePath: path.resolve(ROOT_PATH, 'classified', 'google_service_account.json'),
 }
 
-export const ALIYUN_CLOUD_STORAGE = {
-  accessKey: (argv.cs_access_key as string) || 'cloudstorage access key for cloud storage',
-  secretKey: (argv.cs_secret_key as string) || 'cloudstorage secret key for cloud storage',
-  aliyunAcsARN: (argv.cs_aliyun_acs as string) || 'aliyun Acs ARN, e.g. acs:ram::xxx:role/xxx',
+export const AWS = {
+  accessKeyId: argv.aws_access_key_id as string,
+  secretAccessKey: argv.aws_secret_access_key as string,
+  s3StaticRegion: argv.aws_s3_static_region as string,
+  s3StaticBucket: argv.aws_s3_static_bucket as string,
 }
 
 export const DB_BACKUP = {
-  bucket: (argv.db_backup_bucket as string) || 'cloudstorage bucket name for dbbackup',
-  region: (argv.db_backup_region as string) || 'cloudstorage region for dbbackup, e.g. oss-cn-hangzhou',
+  s3Region: argv.db_backup_s3_region as string,
+  s3Bucket: argv.db_backup_s3_bucket as string,
+  password: argv.db_backup_file_password as string,
 }

src/modules/article/article.service.ts

@@ -11,6 +11,7 @@ import { InjectModel } from '@app/transformers/model.transformer'
 import { getArticleUrl } from '@app/transformers/urlmap.transformer'
 import { SeoService } from '@app/processors/helper/helper.service.seo'
 import { CacheService, CacheIntervalResult } from '@app/processors/cache/cache.service'
+import { increaseTodayViewsCount } from '@app/modules/expansion/expansion.helper'
 import { ArchiveService } from '@app/modules/archive/archive.service'
 import { TagService } from '@app/modules/tag/tag.service'
 import { PublishState } from '@app/constants/biz.constant'
@@ -149,9 +150,7 @@ export class ArticleService {
     article.save({ timestamps: false })
 
     // global today views
-    this.cacheService.get<number>(CACHE_KEY.TODAY_VIEWS).then((views) => {
-      this.cacheService.set(CACHE_KEY.TODAY_VIEWS, (views || 0) + 1)
-    })
+    increaseTodayViewsCount(this.cacheService)
 
     return article.toObject()
   }

src/modules/expansion/expansion.controller.ts

@@ -5,22 +5,24 @@
  */
 
 import { Credentials } from 'google-auth-library'
-import { Controller, Get, Patch, UseGuards } from '@nestjs/common'
+import { Controller, Get, Post, Patch, UploadedFile, Body, UseGuards, UseInterceptors } from '@nestjs/common'
+import { FileInterceptor } from '@nestjs/platform-express'
 import { AdminOnlyGuard } from '@app/guards/admin-only.guard'
 import { AdminMaybeGuard } from '@app/guards/admin-maybe.guard'
 import { Responsor } from '@app/decorators/responsor.decorator'
 import { QueryParams, QueryParamsResult } from '@app/decorators/queryparams.decorator'
-import { CloudStorageService, UploadToken } from '@app/processors/helper/helper.service.cloud-storage'
+import { AWSService } from '@app/processors/helper/helper.service.aws'
 import { GoogleService } from '@app/processors/helper/helper.service.google'
 import { StatisticService, Statistic } from './expansion.service.statistic'
 import { DBBackupService } from './expansion.service.dbbackup'
+import * as APP_CONFIG from '@app/app.config'
 
 @Controller('expansion')
 export class ExpansionController {
   constructor(
+    private readonly awsService: AWSService,
     private readonly googleService: GoogleService,
     private readonly dbBackupService: DBBackupService,
-    private readonly cloudStorageService: CloudStorageService,
     private readonly statisticService: StatisticService
   ) {}
 
@@ -31,13 +33,6 @@ export class ExpansionController {
     return this.statisticService.getStatistic(isUnauthenticated)
   }
 
-  @Get('uptoken')
-  @UseGuards(AdminOnlyGuard)
-  @Responsor.handle('Get cloud storage upload token')
-  getCloudStorageUploadToken(): Promise<UploadToken> {
-    return this.cloudStorageService.getToken()
-  }
-
   @Get('google-token')
   @UseGuards(AdminOnlyGuard)
   @Responsor.handle('Get Google credentials')
@@ -51,4 +46,23 @@ export class ExpansionController {
   updateDatabaseBackup() {
     return this.dbBackupService.backup()
   }
+
+  @Post('upload')
+  @UseGuards(AdminOnlyGuard)
+  @UseInterceptors(FileInterceptor('file'))
+  @Responsor.handle('Upload file to cloud storage')
+  uploadStatic(@UploadedFile() file: Express.Multer.File, @Body() body) {
+    return this.awsService
+      .uploadFile({
+        name: body.name,
+        file: file.buffer,
+        fileContentType: file.mimetype,
+        region: APP_CONFIG.AWS.s3StaticRegion,
+        bucket: APP_CONFIG.AWS.s3StaticBucket,
+      })
+      .then((result) => ({
+        ...result,
+        url: `${APP_CONFIG.APP.STATIC_URL}/${result.key}`,
+      }))
+  }
 }

src/modules/expansion/expansion.helper.ts

@@ -0,0 +1,17 @@
+import { CacheService } from '@app/processors/cache/cache.service'
+import * as CACHE_KEY from '@app/constants/cache.constant'
+
+export const getTodayViewsCount = async (cache: CacheService) => {
+  const count = await cache.get<number>(CACHE_KEY.TODAY_VIEWS)
+  return count ? Number(count) : 0
+}
+
+export const increaseTodayViewsCount = async (cache: CacheService) => {
+  const views = await getTodayViewsCount(cache)
+  await cache.set(CACHE_KEY.TODAY_VIEWS, views + 1)
+  return views + 1
+}
+
+export const resetTodayViewsCount = (cache: CacheService) => {
+  return cache.set(CACHE_KEY.TODAY_VIEWS, 0)
+}

src/modules/expansion/expansion.service.dbbackup.ts

@@ -4,27 +4,25 @@
  * @author Surmon <https://github.com/surmon-china>
  */
 
+import fs from 'fs'
 import path from 'path'
 import shell from 'shelljs'
 import moment from 'moment'
 import schedule from 'node-schedule'
 import { Injectable } from '@nestjs/common'
 import { EmailService } from '@app/processors/helper/helper.service.email'
-import { CloudStorageService } from '@app/processors/helper/helper.service.cloud-storage'
+import { AWSService } from '@app/processors/helper/helper.service.aws'
 import { APP, MONGO_DB, DB_BACKUP } from '@app/app.config'
 import logger from '@app/utils/logger'
 
 const UP_FAILED_TIMEOUT = 1000 * 60 * 5
 const UPLOAD_INTERVAL = '0 0 3 * * *'
-const BACKUP_FILE_NAME = 'nodepress.tar.gz'
+const BACKUP_FILE_NAME = 'nodepress.zip'
 const BACKUP_DIR_PATH = path.join(APP.ROOT_PATH, 'dbbackup')
 
 @Injectable()
 export class DBBackupService {
-  constructor(
-    private readonly emailService: EmailService,
-    private readonly cloudStorageService: CloudStorageService
-  ) {
+  constructor(private readonly emailService: EmailService, private readonly awsService: AWSService) {
     logger.info('[expansion]', 'DB Backup 开始执行定时数据备份任务！')
     schedule.scheduleJob(UPLOAD_INTERVAL, () => {
       this.backup().catch(() => {
@@ -33,11 +31,11 @@ export class DBBackupService {
     })
   }
 
-  public async backup(): Promise<string> {
+  public async backup() {
     try {
       const result = await this.doBackup()
       this.mailToAdmin('Database backup succeed', JSON.stringify(result, null, 2))
-      return result.name
+      return result
     } catch (error) {
       this.mailToAdmin('Database backup failed!', String(error))
       throw error
@@ -55,7 +53,7 @@ export class DBBackupService {
   }
 
   private doBackup() {
-    return new Promise<{ url: string; name: string }>((resolve, reject) => {
+    return new Promise<{ url: string; key: string }>((resolve, reject) => {
       if (!shell.which('mongodump')) {
         return reject('DB Backup script requires [mongodump]')
       }
@@ -72,24 +70,33 @@ export class DBBackupService {
           return reject(out)
         }
 
-        shell.exec(`tar -czf ${BACKUP_FILE_NAME} ./backup`)
+        if (!shell.which('zip')) {
+          return reject('DB Backup script requires [zip]')
+        }
+
+        // tar -czf - backup | openssl des3 -salt -k <password> -out target.tar.gz
+        // shell.exec(`tar -czf ${BACKUP_FILE_NAME} ./backup`)
+        shell.exec(`zip -r -P ${DB_BACKUP.password} ${BACKUP_FILE_NAME} ./backup`)
         const fileDate = moment(new Date()).format('YYYY-MM-DD-HH:mm')
-        const fileName = `nodepress-mongodb/backup-${fileDate}.tar.gz`
+        const fileName = `nodepress-mongodb/backup-${fileDate}.zip`
         const filePath = path.join(BACKUP_DIR_PATH, BACKUP_FILE_NAME)
         logger.info('[expansion]', 'DB Backup 上传文件: ' + fileName)
         logger.info('[expansion]', 'DB Backup 文件源位置: ' + filePath)
 
-        // 上传文件
-        this.cloudStorageService
-          .uploadFile(fileName, filePath, DB_BACKUP.region, DB_BACKUP.bucket)
+        // upload to cloud storage
+        this.awsService
+          .uploadFile({
+            name: fileName,
+            file: fs.createReadStream(filePath),
+            fileContentType: 'application/zip',
+            region: DB_BACKUP.s3Region,
+            bucket: DB_BACKUP.s3Bucket,
+            classType: 'GLACIER',
+            encryption: 'AES256',
+          })
           .then((result) => {
-            const data = {
-              name: result.name,
-              url: result.url,
-              data: result.data,
-            }
-            logger.info('[expansion]', 'DB Backup succeed!', data)
-            resolve(data)
+            logger.info('[expansion]', 'DB Backup succeed!', result.url)
+            resolve(result)
           })
           .catch((error) => {
             logger.warn('[expansion]', 'DB Backup failed!', error)

src/modules/expansion/expansion.service.statistic.ts

@@ -11,8 +11,8 @@ import { ArticleService } from '@app/modules/article/article.service'
 import { CommentService } from '@app/modules/comment/comment.service'
 import { FeedbackService } from '@app/modules/feedback/feedback.service'
 import { TagService } from '@app/modules/tag/tag.service'
-import * as CACHE_KEY from '@app/constants/cache.constant'
 import logger from '@app/utils/logger'
+import { getTodayViewsCount, resetTodayViewsCount } from './expansion.helper'
 
 const DEFAULT_STATISTIC = Object.freeze({
   tags: null,
@@ -28,8 +28,6 @@ export type Statistic = Record<keyof typeof DEFAULT_STATISTIC, number | null>
 
 @Injectable()
 export class StatisticService {
-  private resultData: Statistic = { ...DEFAULT_STATISTIC }
-
   constructor(
     private readonly cacheService: CacheService,
     private readonly articleService: ArticleService,
@@ -39,49 +37,36 @@ export class StatisticService {
   ) {
     // clear date when everyday 00:00
     schedule.scheduleJob('1 0 0 * * *', () => {
-      this.cacheService.set(CACHE_KEY.TODAY_VIEWS, 0).catch((error) => {
+      resetTodayViewsCount(this.cacheService).catch((error) => {
         logger.warn('[expansion]', 'statistic set TODAY_VIEWS Error:', error)
       })
     })
   }
 
-  private async getTodayViewsCount() {
-    const views = await this.cacheService.get<number>(CACHE_KEY.TODAY_VIEWS)
-    this.resultData.todayViews = views || 0
-  }
-
-  private async getArticlesStatistic() {
-    const meta = await this.articleService.getMetaStatistic()
-    this.resultData.totalViews = meta.totalViews
-    this.resultData.totalLikes = meta.totalLikes
-  }
-
-  private async getArticlesCount(publicOnly: boolean) {
-    this.resultData.articles = await this.articleService.getTotalCount(publicOnly)
-  }
-
-  private async getTagsCount() {
-    this.resultData.tags = await this.tagService.getTotalCount()
-  }
-
-  private async getCommentsCount(publicOnly: boolean) {
-    this.resultData.comments = await this.commentService.getTotalCount(publicOnly)
-  }
-
-  private async getAverageEmotion() {
-    this.resultData.averageEmotion = await this.feedbackService.getRootFeedbackAverageEmotion()
-  }
-
   public getStatistic(publicOnly: boolean) {
+    const resultData: Statistic = { ...DEFAULT_STATISTIC }
     return Promise.all([
-      this.getTagsCount(),
-      this.getArticlesCount(publicOnly),
-      this.getCommentsCount(publicOnly),
-      this.getAverageEmotion(),
-      this.getArticlesStatistic(),
-      this.getTodayViewsCount(),
+      this.tagService.getTotalCount().then((value) => {
+        resultData.tags = value
+      }),
+      this.articleService.getTotalCount(publicOnly).then((value) => {
+        resultData.articles = value
+      }),
+      this.commentService.getTotalCount(publicOnly).then((value) => {
+        resultData.comments = value
+      }),
+      this.articleService.getMetaStatistic().then((value) => {
+        resultData.totalViews = value.totalViews
+        resultData.totalLikes = value.totalLikes
+      }),
+      getTodayViewsCount(this.cacheService).then((value) => {
+        resultData.todayViews = value
+      }),
+      this.feedbackService.getRootFeedbackAverageEmotion().then((value) => {
+        resultData.averageEmotion = value
+      }),
     ])
-      .then(() => Promise.resolve(this.resultData))
-      .catch(() => Promise.resolve(this.resultData))
+      .then(() => Promise.resolve(resultData))
+      .catch(() => Promise.resolve(resultData))
   }
 }

src/processors/helper/helper.module.ts

@@ -8,12 +8,12 @@ import { Module, Global } from '@nestjs/common'
 import { HttpModule } from '@nestjs/axios'
 import { GoogleService } from './helper.service.google'
 import { AkismetService } from './helper.service.akismet'
-import { CloudStorageService } from './helper.service.cloud-storage'
+import { AWSService } from './helper.service.aws'
 import { EmailService } from './helper.service.email'
 import { SeoService } from './helper.service.seo'
 import { IPService } from './helper.service.ip'
 
-const services = [GoogleService, AkismetService, CloudStorageService, EmailService, SeoService, IPService]
+const services = [GoogleService, AkismetService, AWSService, EmailService, SeoService, IPService]
 
 @Global()
 @Module({