-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathMakefile
53 lines (35 loc) · 2.57 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
yesterday != date -d yesterday +"%Y%m%d"
CXX ?= g++
CFLAGS = -O3 -std=c++17 -lpthread
main: src/main.cpp src/BooPHF.h
$(CXX) -o $@ $< $(CFLAGS)
### NECESSARY INTERMEDIATE FILES: --------------------------------
data/planet.pbf:
@echo "Missing data/planet.pbf: See https://wiki.openstreetmap.org/wiki/Planet.osm#Downloading for how to download an Osmium-compatible planet file." >&2; exit 1
# You could add a command here to download data/planet.pbf with a BitTorrent client
data/commonswiki-all-titles.gz: FORCE
curl -z data/commonswiki-all-titles.gz https://dumps.wikimedia.org/commonswiki/latest/commonswiki-latest-all-titles.gz -o data/commonswiki-all-titles.gz
data/commonswiki-cats.txt: data/commonswiki-all-titles.gz
# namespace 14 is "Category:"
zcat data/commonswiki-all-titles.gz | grep ^14 | cut -f2 > data/commonswiki-cats.txt
data/commonswiki-files.txt.gz: FORCE
curl -z data/commonswiki-files.txt.gz https://dumps.wikimedia.org/other/mediatitles/$(yesterday)/commonswiki-$(yesterday)-all-media-titles.gz -o data/commonswiki-files.txt.gz
data/commonswiki-files.txt: data/commonswiki-files.txt.gz
gunzip --force --keep data/commonswiki-files.txt.gz
data/planet-filtered.tsv: data/planet.pbf
osmium tags-filter -R data/planet.pbf 'nwr/wikimedia_commons' -f xml | ./xml2tsv.py > data/planet-filtered.tsv
data/planet-filtered.geojson: data/planet.pbf
osmium tags-filter -t data/planet.pbf 'nwr/wikimedia_commons' -o data/planet-filtered.pbf --overwrite
osmium export data/planet-filtered.pbf -c src/config.json -o data/planet-filtered.geojson -f jsonseq --overwrite
### OUTPUT FILES: ------------------------------------------------
out/cats.tsv: main data/commonswiki-cats.txt data/planet-filtered.tsv
# ./main.py Category: data/commonswiki-cats.txt data/planet-filtered.tsv > out/cats.tsv
./main Category: $$(wc -l data/commonswiki-cats.txt) data/planet-filtered.tsv $$(nproc) > out/cats.tsv
out/cats.geojson: main data/commonswiki-cats.txt data/planet-filtered.geojson
# ./main.py Category: data/commonswiki-cats.txt data/planet-filtered.geojson > out/cats.geojson
./main Category: $$(wc -l data/commonswiki-cats.txt) data/planet-filtered.geojson $$(nproc) > out/cats.geojson
out/files.tsv: main data/commonswiki-files.txt data/planet-filtered.tsv
./main File: $$(wc -l data/commonswiki-files.txt) data/planet-filtered.tsv $$(nproc) | grep -Fv ';File:' > out/files.tsv
out/files.geojson: main data/commonswiki-files.txt data/planet-filtered.geojson
./main File: $$(wc -l data/commonswiki-files.txt) data/planet-filtered.geojson $$(nproc) | grep -Fv ';File:' > out/files.geojson
FORCE: ;