-
Notifications
You must be signed in to change notification settings - Fork 0
/
Crawler.ts
29 lines (28 loc) · 1015 Bytes
/
Crawler.ts
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
import {logger, sleep, urlMap} from './settings'
import GetPostList from './GetPostList'
import GetImageList from './GetImageList'
import {save} from './DataBaseOperator'
import {PostInfo, PostInfoWithImages, SingleImage} from "./Interfaces"
export default async () => {
for (let forum in urlMap) {
let urls = urlMap[forum]
for (let url of urls) {
const postList: PostInfo[] = await GetPostList(url, forum)
for (let post of postList) {
await sleep(1000)
const imageList: SingleImage[] = await GetImageList(post)
if (imageList.length) {
let postInfo = <PostInfoWithImages>{
...post,
images: imageList,
forum
}
await save(postInfo)
} else {
logger.warn(`post ${post.postUrl} get no images`)
}
}
}
}
return true
}