Skip to content

Commit

Permalink
Fix [sc-8784]: Fix sitemap connector (#12)
Browse files Browse the repository at this point in the history
* Fix [sc-8784]: Fix sitemap connector

Fix `sync.entity` `getLastModified` function to call the method to get files from the connector only if the folders to sync is not empty. This is for preventing syncs with static folders like sitemap to upload their content several times on a same sync, and to prevent uploading the files on each synchronization.

* Formatting
  • Loading branch information
mpellerin42 authored Feb 9, 2024
1 parent 5fecf51 commit d5f7344
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 14 deletions.
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { from, map, Observable, of, switchMap } from 'rxjs';
import { ConnectorParameters, FileStatus, IConnector, Link, SearchResults, SyncItem } from '../../domain/connector';
import { SourceConnectorDefinition } from '../factory';
import cheerio from 'cheerio';
import * as cheerio from 'cheerio';

interface SiteMapModel {
loc: string;
Expand All @@ -10,7 +10,7 @@ interface SiteMapModel {

async function fetchSitemap(url: string): Promise<string> {
const response = await fetch(url);
// todo: control whether it is zipped or plain
// TODO: control whether it is zipped or plain
return response.text();
}

Expand Down Expand Up @@ -53,11 +53,8 @@ class SitemapImpl implements IConnector {
this.params = params;
}

areParametersValid(params: ConnectorParameters) {
if (!params?.sitemap) {
return false;
}
return true;
areParametersValid(params: ConnectorParameters): boolean {
return !!params?.sitemap;
}

getParameters(): ConnectorParameters {
Expand Down
17 changes: 10 additions & 7 deletions server/src/logic/sync/domain/sync.entity.ts
Original file line number Diff line number Diff line change
Expand Up @@ -105,14 +105,17 @@ export class SyncEntity {
const foldersToSyncUpdated: SyncItem[] = (this.foldersToSync ?? []).filter(
(folder) => folder.status === FileStatus.UPLOADED,
);
const getFilesFoldersUpdated = this.sourceConnector!.getLastModified(
this.lastSyncGMT || '2000-01-01T00:00:00.000Z',
foldersToSyncUpdated,
);
const getFilesFolderPending = this.sourceConnector!.getFilesFromFolders(foldersToSyncPending);
const getFilesFoldersUpdated =
foldersToSyncUpdated.length > 0
? this.sourceConnector!.getLastModified(this.lastSyncGMT || '2000-01-01T00:00:00.000Z', foldersToSyncUpdated)
: of({ items: [] });

const getFilesFolderPending =
foldersToSyncPending.length > 0
? this.sourceConnector!.getFilesFromFolders(foldersToSyncPending)
: of({ items: [] });
return forkJoin([getFilesFoldersUpdated, getFilesFolderPending]).pipe(
map((results) => {
const [updated, pending] = results;
map(([updated, pending]) => {
return { success: true, results: [...updated.items, ...pending.items] };
}),
catchError((err) => {
Expand Down

0 comments on commit d5f7344

Please sign in to comment.