-
Notifications
You must be signed in to change notification settings - Fork 1
/
Makefile.wikipedia
49 lines (37 loc) · 2.58 KB
/
Makefile.wikipedia
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
WIKIPEDIA_DIR?=data/wikipedia
WIKIDATA_DIR?=data/wikidata
PROCESSES?=6
clean:
rm -f $(WIKIPEDIA_DIR)/*.sql
rm -f $(WIKIPEDIA_DIR)/*.xml
$(WIKIPEDIA_DIR)/%wiki-latest-pages-articles-multistream.xml:
wget -nc --directory-prefix=$(WIKIPEDIA_DIR) https://dumps.wikimedia.org/$*wiki/latest/$*wiki-latest-pages-articles-multistream.xml.bz2
bunzip2 $(WIKIPEDIA_DIR)/$*wiki*bz2
%-wikipedia-files:
wget -nc --directory-prefix=$(WIKIPEDIA_DIR) https://dumps.wikimedia.org/$*wiki/latest/$*wiki-latest-categorylinks.sql.gz
wget -nc --directory-prefix=$(WIKIPEDIA_DIR) https://dumps.wikimedia.org/$*wiki/latest/$*wiki-latest-category.sql.gz
wget -nc --directory-prefix=$(WIKIPEDIA_DIR) https://dumps.wikimedia.org/$*wiki/latest/$*wiki-latest-page.sql.gz
wget -nc --directory-prefix=$(WIKIPEDIA_DIR) https://dumps.wikimedia.org/$*wiki/latest/$*wiki-latest-redirect.sql.gz
wget -nc --directory-prefix=$(WIKIPEDIA_DIR) https://dumps.wikimedia.org/$*wiki/latest/$*wiki-latest-page_props.sql.gz
gunzip $(WIKIPEDIA_DIR)/$*wiki*.gz
mysql2sqlite:
git clone https://github.com/dumblob/mysql2sqlite.git
$(WIKIPEDIA_DIR)/%-wikipedia.sqlite3: %-wikipedia-files mysql2sqlite
rm -f $@
-mysql2sqlite/mysql2sqlite $(WIKIPEDIA_DIR)/$*wiki-latest-category.sql | sqlite3 $@
-mysql2sqlite/mysql2sqlite $(WIKIPEDIA_DIR)/$*wiki-latest-categorylinks.sql | sqlite3 $@
-mysql2sqlite/mysql2sqlite $(WIKIPEDIA_DIR)/$*wiki-latest-page.sql | sqlite3 $@
-mysql2sqlite/mysql2sqlite $(WIKIPEDIA_DIR)/$*wiki-latest-redirect.sql | sqlite3 $@
-mysql2sqlite/mysql2sqlite $(WIKIPEDIA_DIR)/$*wiki-latest-page_props.sql | sqlite3 $@
rm $(WIKIPEDIA_DIR)/$*wiki*.sql
$(WIKIPEDIA_DIR)/%-fulltext.sqlite: $(WIKIPEDIA_DIR)/%wiki-latest-pages-articles-multistream.xml
extract-wiki-full-text-sqlite $< $@
rm $<
$(WIKIPEDIA_DIR)/%-wikipedia-tagged.sqlite: $(WIKIPEDIA_DIR)/%-fulltext.sqlite $(WIKIDATA_DIR)/%-wikipedia-ner-entities.pkl $(WIKIPEDIA_DIR)/%-wikipedia.sqlite3
tag-wiki-entities2sqlite --processes=$(PROCESSES) $^ $@
$(WIKIPEDIA_DIR)/%-redirects.pkl: $(WIKIDATA_DIR)/%-wikipedia-ner-entities.pkl $(WIKIPEDIA_DIR)/%-wikipedia.sqlite3
redirects2pkl $^ $@
sqlite-databases: $(WIKIPEDIA_DIR)/de-wikipedia.sqlite3 $(WIKIPEDIA_DIR)/fr-wikipedia.sqlite3 $(WIKIPEDIA_DIR)/en-wikipedia.sqlite3
fulltext: $(WIKIPEDIA_DIR)/de-fulltext.sqlite $(WIKIPEDIA_DIR)/fr-fulltext.sqlite $(WIKIPEDIA_DIR)/en-fulltext.sqlite
tagged: $(WIKIPEDIA_DIR)/de-wikipedia-tagged.sqlite $(WIKIPEDIA_DIR)/fr-wikipedia-tagged.sqlite $(WIKIPEDIA_DIR)/en-wikipedia-tagged.sqlite
redirects: $(WIKIPEDIA_DIR)/de-redirects.pkl $(WIKIPEDIA_DIR)/fr-redirects.pkl $(WIKIPEDIA_DIR)/en-redirects.pkl