-
Notifications
You must be signed in to change notification settings - Fork 183
/
example.config
209 lines (159 loc) · 9.16 KB
/
example.config
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
[DATABASE]
# ------------------------------------------------------------------------
# The database is a PostgreSQL database, and these are the standard
# connection parameters.
# ------------------------------------------------------------------------
# The host is the machine name for the connection
# ------------------------------------------------------------------------
db_host =
# db_host = localhost
# ------------------------------------------------------------------------
# The port number (PostgreSQL default port is 5432)
# ------------------------------------------------------------------------
db_port =
# ------------------------------------------------------------------------
# A PostgreSQL database server may host multiple databases.
# Specify the database you are using here.
# ------------------------------------------------------------------------
db_name =
# ------------------------------------------------------------------------
# Database user, which must have sufficient permissions for
# whatever operations you are carrying out
# ------------------------------------------------------------------------
db_user =
# ------------------------------------------------------------------------
# password
# ------------------------------------------------------------------------
db_password =
# If you are using a set of proxies, supply a comma-separated list of
# host:port pairs here. You can split the list over multiple lines
# and leave whitespace at the beginning of the line, like this:
#
# proxy_list=n.nn.nn.nn:12345,
# n.nn.nn.mm:12345
#
proxy_list = host1:port1,host2:port2
[NETWORK]
# ------------------------------------------------------------------------
# Network parameters control the behaviour of the script when accessing
# the Airbnb website.
# ------------------------------------------------------------------------
# If you are using a set of proxies, supply a comma-separated list of
# host:port pairs here. You can split the list over multiple lines if you
# wish.
# ------------------------------------------------------------------------
proxy_list =
# ------------------------------------------------------------------------
# A user agent string is used to identify the program making the request
# As the user agent string can contain commas, the separator is a double comma: ",,"
# Each user agent string should also be enclosed in double quotes.
# ------------------------------------------------------------------------
user_agent_list = "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko",,
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A",,
"Mozilla/5.0 (compatible; MSIE 10.0; Macintosh; Intel Mac OS X 10_7_3; Trident/6.0)",,
"Mozilla/5.0 (compatible, MSIE 11, Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko",,
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36",,
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36",,
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2309.372 Safari/537.36",,
"Mozilla/5.0 (compatible; MSIE 10.6; Windows NT 6.1; Trident/5.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727) 3gpp-gba UNTRUSTED/1.0",,
"Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25",,
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.517 Safari/537.36"
# "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36",,
# "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/534.55.3 (KHTML, like Gecko) Version/5.1.3 Safari/534.53.10",,
# ------------------------------------------------------------------------
# How many times to try connecting to the Airbnb site before failing
# ------------------------------------------------------------------------
max_connection_attempts = 15
# ------------------------------------------------------------------------
# Be nice: pause between requests. This is a number of seconds,
# but the pause is a random number in the interval [0, request_sleep]
# ------------------------------------------------------------------------
request_sleep = 0.0
# ------------------------------------------------------------------------
# how long to wait before failing on an individual request
# ------------------------------------------------------------------------
http_timeout = 10.0
# ------------------------------------------------------------------------
# The root API for searches
# ------------------------------------------------------------------------
url_api_search_root = https://www.airbnb.com/api/v2/explore_tabs
# ------------------------------------------------------------------------
# API Key
# To collect an API key using Chrome as your browser:
# - In your browser, go to the Airbnb web site
# - From the menu button at the top right, open More Tools > Developer Tools
# - Click the Network tab
# - In the Developer Tools, click the stop sign to clear the list of requests
# - Click XHR (this is the kind of event we want)
# - Move the map to generate a new search
# - Find a request with explore_tabs in the name, right click it to show
# the URL in a new pane.
# - Find the key= part of the URL and add it here
# ------------------------------------------------------------------------
api_key = <32-character-string>
# ------------------------------------------------------------------------
# Client Session ID
# From what I can see, this can be left as empty
# ------------------------------------------------------------------------
client_session_id =
[SURVEY]
# ------------------------------------------------------------------------
# These don't often need to be changed
# ------------------------------------------------------------------------
fill_max_room_count = 50000
# ------------------------------------------------------------------------
# For the special case of doing a global sample of Airbnb listings, room
# values are chosen at random for a range with this as the maximum.
# For other searches, this value is not needed.
# ------------------------------------------------------------------------
room_id_upper_bound = 20000000
# ------------------------------------------------------------------------
# Maximum number of pages to loop over in search,
# for a given area, room_type, and number of guests
# search_max_pages = 10
# ------------------------------------------------------------------------
search_max_pages = 20
# ------------------------------------------------------------------------
# Maximum number of guests in search
# If set to 1 (recommended), searches will not include a "guests" parameter
# ------------------------------------------------------------------------
search_max_guests = 1
# ------------------------------------------------------------------------
# Maximum zoom level for bounding box search
# For search_max_guests = 1 or for larger areas, set to 8
# For search_max_guests > 1 and a city (as opposed to larger area),
# 6 may be sufficient
# For search_do_loop_over_prices = 0, increase further (eg 12)
# ------------------------------------------------------------------------
# search_max_rectangle_zoom = 6
search_max_rectangle_zoom = 12
# ------------------------------------------------------------------------
# Blur to add to rectangle boundary to avoid gaps
# - as a fraction of the rectangle width
# ------------------------------------------------------------------------
search_rectangle_edge_blur = 0.0
# ------------------------------------------------------------------------
# Set this to zero to not loop over various room types, but look for all
# room types at once.
# ------------------------------------------------------------------------
search_do_loop_over_room_types = 0
# ------------------------------------------------------------------------
# Set this to zero to not loop over various price ranges, but look for all
# prices at once: if you do this, set a high search_max_rectangle_zoom as
# there will be more listings in any given square
# As of May 2018, this variable is ignored: no loop over prices is done.
# ------------------------------------------------------------------------
search_do_loop_over_prices = 0
# ------------------------------------------------------------------------
# Time to wait, in seconds, when all proxies are used up, before restarting
# ------------------------------------------------------------------------
re_init_sleep_time = 60
[ACCOUNT]
# ------------------------------------------------------------------------
# Google geocoding API key, obtained from
# https://developers.google.com/maps/documentation/geocoding/start#get-a-key
# ------------------------------------------------------------------------
google_api_key = ALONGSTRINGOFLETTERSANDNUMBERS
# ------------------------------------------------------------------------
# End of configuration file
# ------------------------------------------------------------------------