-
Notifications
You must be signed in to change notification settings - Fork 0
/
crawler.rb
36 lines (30 loc) · 867 Bytes
/
crawler.rb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
require "open-uri"
require "rexml/document"
class Crawler
include REXML
BASE_URL = "https://finshop.belgium.be"
attr_reader :data
def self.search(keyword)
require_relative "./results_crawler"
require_relative "./detail_crawler"
results_crawler = ResultsCrawler.new(keyword)
results_crawler.fetch_and_parse
results_data = results_crawler.data
results_data.inject([]) do |memo, result_data|
detail_crawler = DetailCrawler.new(result_data[:url])
detail_crawler.fetch_and_parse
detail_data = detail_crawler.data
memo << {
title: result_data[:title],
url: result_data[:url],
price: detail_data[:price],
available: detail_data[:available]
}
end
end
def fetch_and_parse
open(@url).each_line do |line|
parse_line(line.strip)
end
end
end