diff --git a/apps/triage/src/app/app.service.ts b/apps/triage/src/app/app.service.ts index 4538d508..19852a78 100644 --- a/apps/triage/src/app/app.service.ts +++ b/apps/triage/src/app/app.service.ts @@ -44,7 +44,7 @@ export class AppService { startTime = lastMeedanReport[0].node?.tasks?.edges?.find(task => task.node?.label === this.config.originalPostTimeField).node?.first_response_value } } - const postsCount = await this.unitedwaveClient.getPostsCount().toPromise(); + const postsCount = await this.unitedwaveClient.getPostsCount(startTime).toPromise(); this.logger.log('Latest unitedwaveClient count', postsCount) const postsPerPage = 50; const pageCount = Math.ceil(Number(postsCount) / postsPerPage); @@ -55,10 +55,13 @@ export class AppService { const list = await this.unitedwaveClient.getPosts(page,startTime).toPromise(); for (let i = list.length - 1; i >= 0; i--) { const post = list[i]; - //check duplication + const title = [post?.clip_name, post?.keywords, post?.date_reported] + .filter(Boolean) + .join(' - '); + // check duplication const isDuplicate = this.isDuplicatePost(post, createdPosts); if (!isDuplicate) { - const item = await this.checkClient.createItemFromRadio(post?.clip_url, post?.clip_name, post?.source_text, post?.date_reported).toPromise(); + const item = await this.checkClient.createItemFromRadio(post?.clip_url, title, post?.source_text, post?.date_reported).toPromise(); if(!item.error) { createdItems = [...createdItems, item]; createdPosts = [...createdPosts, post]; diff --git a/libs/unitedwave-client/src/lib/unitedwave-client.service.ts b/libs/unitedwave-client/src/lib/unitedwave-client.service.ts index 02eb1612..b86dba13 100644 --- a/libs/unitedwave-client/src/lib/unitedwave-client.service.ts +++ b/libs/unitedwave-client/src/lib/unitedwave-client.service.ts @@ -24,11 +24,14 @@ export class UnitedwaveClientService{ return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}`; } - getPostsCount(filter?:string) { + getPostsCount(startDate?: string,filter?:string) { let query = this.config.endpoints.count + `?user[name]=${this.config.username}&user[secret]=${this.config.password}`; if(filter) { query += `&clip[filters]=${filter}` } + if (startDate) { + query += `&clip[from_date]=${startDate}` + } return this.http.post(query).pipe( map(res => res.data), retry(3),