forked from dismantl/CaseHarvester
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Makefile
259 lines (205 loc) · 8.62 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
CODE_SRC=src
PACKAGE_DIR=pkg
LIB_DIR=lib
DOCS_DIR=docs
SPIDER_DEPS=$(addprefix $(CODE_SRC)/,spider/scheduled_spider.py \
$(addprefix mjcs/,__init__.py spider.py config.py util.py run.py search.py \
session.py models/*.py))
SCRAPER_DEPS=$(addprefix $(CODE_SRC)/,scraper/scraper_lambda.py \
$(addprefix mjcs/,__init__.py scraper.py config.py util.py session.py models/*.py))
PARSER_DEPS=$(addprefix $(CODE_SRC)/,parser/parser_lambda.py \
$(addprefix mjcs/,__init__.py config.py util.py parser/*.py models/*.py))
SECRETS_FILE=secrets.json
STACK_PREFIX=caseharvester-stack
AWS_REGION=us-east-1
DB_NAME=mjcs
AWS_PROFILE=default
DOCKER_REPO_NAME=mjcs_spider
.PHONY: package $(addprefix package_,scraper parser) deploy \
deploy_production $(addprefix deploy_,static docker-repo spider scraper parser) \
$(addsuffix _production,$(addprefix deploy_,static docker-repo spider scraper parser)) \
test clean clean_all list_exports init init_production parser_notification \
parser_notification_production docker_image docker_image_production sync sync_prod docs
define package_f
$(eval component = $(1))
mkdir -p $(PACKAGE_DIR)/$(component)
pip3 install -r $(CODE_SRC)/$(component)/requirements.txt -t $(PACKAGE_DIR)/$(component)/
cp $(CODE_SRC)/$(component)/$(component)_lambda.py $(PACKAGE_DIR)/$(component)/
cp -r $(CODE_SRC)/mjcs $(PACKAGE_DIR)/$(component)/
find $(PACKAGE_DIR) -name *.pyc -delete
find $(PACKAGE_DIR) -name __pycache__ -delete
endef
define deploy_stack_f
$(eval component = $(1))
$(eval environment = $(2))
$(eval env_long = $(subst prod,production,$(subst dev,development,$(environment))))
aws cloudformation package --template-file cloudformation/stack-$(component).yaml \
--output-template-file cloudformation/stack-$(component)-output.yaml \
--s3-bucket $(STACK_PREFIX)-$(component)-$(environment)
aws cloudformation deploy --template-file cloudformation/stack-$(component)-output.yaml \
--stack-name $(STACK_PREFIX)-$(component)-$(environment) \
--capabilities CAPABILITY_IAM \
--capabilities CAPABILITY_NAMED_IAM \
--parameter-overrides \
EnvironmentType=$(environment) DatabaseName=$(DB_NAME) \
StaticStackName=$(STACK_PREFIX)-static-$(environment) \
DockerRepoStackName=$(STACK_PREFIX)-docker-repo-$(environment) \
ScraperStackName=$(STACK_PREFIX)-scraper-$(environment) \
AWSRegion=$(AWS_REGION) \
DockerRepoName=$(environment)_$(DOCKER_REPO_NAME) \
$(shell jq -r '.$(env_long) as $$x|$$x|keys[]|. + "=" + $$x[.]' $(SECRETS_FILE))
endef
define create_stack_bucket_f
$(eval component = $(1))
$(eval environment = $(2))
aws s3api create-bucket --bucket $(STACK_PREFIX)-$(component)-$(environment) --region $(AWS_REGION)
endef
define add_parser_notification_f
$(eval environment = $(1))
sleep 5 # so exports can propagate
aws cloudformation list-exports | awk '\
/$(STACK_PREFIX)-static-$(environment)-CaseDetailsBucketName/ { getline; bucket=$$2 };\
/$(STACK_PREFIX)-parser-$(environment)-ParserArn/ { getline; parser_arn=$$2 };\
END { print bucket, parser_arn }' | \
xargs printf "aws s3api put-bucket-notification-configuration --bucket %s \
--notification-configuration \'{\"LambdaFunctionConfigurations\": \
[{\"LambdaFunctionArn\":\"%s\",\"Events\":[\"s3:ObjectCreated:*\"]}]}\'" | bash
endef
define db_init_f
$(eval environment = $(1))
$(eval profile = $(or $(2),$(AWS_PROFILE),default))
DEV_MODE=1 python3 $(CODE_SRC)/case_harvester.py --environment $(environment) \
--profile $(profile) db_init --db-name $(DB_NAME) --secrets-file $(SECRETS_FILE)
endef
define create_docs_f
$(eval environment = $(1))
$(eval env_long = $(subst prod,production,$(subst dev,development,$(environment))))
aws cloudformation list-exports|grep \
$(STACK_PREFIX)-static-$(environment)-DatabaseHostname -A1|grep Value|\
awk '{print $$2}' | xargs \
docker run -v "$(abspath $(DOCS_DIR)):/output" schemaspy/schemaspy:snapshot \
-t pgsql \
-db $(DB_NAME) \
-s public \
-u $(shell jq '.$(env_long).DatabaseUsername' $(SECRETS_FILE)) \
-p $(shell jq '.$(env_long).DatabasePassword' $(SECRETS_FILE)) \
-host
endef
define push_docker_image_f
$(eval environment = $(1))
$(shell aws ecr get-login --region $(AWS_REGION) --no-include-email)
$(eval REPO_NAME = $(environment)_$(DOCKER_REPO_NAME))
$(eval AWS_ACCOUNT_ID = $(shell aws sts get-caller-identity | grep Account | cut -d'"' -f4))
find $(CODE_SRC) -name *.pyc -delete
find $(CODE_SRC) -name __pycache__ -delete
docker build -t $(REPO_NAME) .
$(eval REPO_URL = $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/$(REPO_NAME))
docker tag $(REPO_NAME):latest $(REPO_URL):latest
docker push $(REPO_URL)
endef
.package-scraper: $(SCRAPER_DEPS) $(CODE_SRC)/scraper/requirements.txt
$(call package_f,scraper)
cp -r $(LIB_DIR)/psycopg2 $(PACKAGE_DIR)/scraper/
touch $@
.package-parser: $(PARSER_DEPS) $(CODE_SRC)/parser/requirements.txt
$(call package_f,parser)
cp -r $(LIB_DIR)/psycopg2 $(PACKAGE_DIR)/parser/
touch $@
.create-stack-buckets:
$(foreach env,dev prod,\
$(foreach component,static docker-repo spider scraper parser,\
$(call create_stack_bucket_f,$(component),$(env))\
)\
)
touch $@
.deploy-static-dev: .create-stack-buckets cloudformation/stack-static.yaml $(SECRETS_FILE)
$(call deploy_stack_f,static,dev)
touch $@
.deploy-static-prod: .create-stack-buckets cloudformation/stack-static.yaml $(SECRETS_FILE)
$(call deploy_stack_f,static,prod)
touch $@
.deploy-docker-repo-dev: .create-stack-buckets .deploy-static-dev cloudformation/stack-docker-repo.yaml $(SECRETS_FILE)
$(call deploy_stack_f,docker-repo,dev)
touch $@
.deploy-docker-repo-prod: .create-stack-buckets .deploy-static-prod cloudformation/stack-docker-repo.yaml $(SECRETS_FILE)
$(call deploy_stack_f,docker-repo,prod)
touch $@
.push-docker-image-dev: .deploy-docker-repo-dev Dockerfile $(SPIDER_DEPS)
$(call push_docker_image_f,dev)
touch $@
.push-docker-image-prod: .deploy-docker-repo-prod Dockerfile $(SPIDER_DEPS)
$(call push_docker_image_f,prod)
touch $@
.deploy-spider-dev: .deploy-static-dev cloudformation/stack-spider.yaml $(SECRETS_FILE)
$(call deploy_stack_f,spider,dev)
touch $@
.deploy-spider-prod: .deploy-static-prod cloudformation/stack-spider.yaml $(SECRETS_FILE)
$(call deploy_stack_f,spider,prod)
touch $@
.deploy-scraper-dev: .deploy-static-dev .package-scraper cloudformation/stack-scraper.yaml $(SECRETS_FILE)
$(call deploy_stack_f,scraper,dev)
touch $@
.deploy-scraper-prod: .deploy-static-prod .package-scraper cloudformation/stack-scraper.yaml $(SECRETS_FILE)
$(call deploy_stack_f,scraper,prod)
touch $@
.deploy-parser-dev: .deploy-static-dev .package-parser cloudformation/stack-parser.yaml $(SECRETS_FILE)
$(call deploy_stack_f,parser,dev)
$(call add_parser_notification_f,dev)
touch $@
.deploy-parser-prod: .deploy-static-prod .package-parser cloudformation/stack-parser.yaml $(SECRETS_FILE)
$(call deploy_stack_f,parser,prod)
$(call add_parser_notification_f,prod)
touch $@
.init-dev: .deploy-static-dev .deploy-spider-dev .deploy-scraper-dev .deploy-parser-dev $(SECRETS_FILE)
$(call db_init_f,development)
touch $@
.init-prod: .deploy-static-prod .deploy-spider-prod .deploy-scraper-prod .deploy-parser-prod $(SECRETS_FILE)
$(call db_init_f,production)
touch $@
parser_notification:
$(call add_parser_notification_f,dev)
parser_notification_production:
$(call add_parser_notification_f,prod)
docker_image:
$(call push_docker_image_f,dev)
docker_image_production:
$(call push_docker_image_f,prod)
package_scraper: .package-scraper
package_parser: .package-parser
package: package_scraper package_parser
deploy_static: .deploy-static-dev
deploy_docker_repo: .deploy-docker-repo-dev
deploy_spider: .deploy-spider-dev .push-docker-image-dev
deploy_scraper: .deploy-scraper-dev
deploy_parser: .deploy-parser-dev
deploy: deploy_static deploy_docker_repo deploy_scraper deploy_parser deploy_spider
deploy_static_production: .deploy-static-prod
deploy_docker_repo_production: .deploy-docker-repo-prod
deploy_spider_production: .deploy-spider-prod .push-docker-image-prod
deploy_scraper_production: .deploy-scraper-prod
deploy_parser_production: .deploy-parser-prod
deploy_production: deploy_static_production deploy_docker_repo_production \
deploy_scraper_production deploy_parser_production deploy_spider_production
init: .init-dev
init_production: .init-prod
list_exports:
aws cloudformation list-exports
test:
pytest
sync:
rsync -av . mjcs:mjcs
sync_prod:
rsync -av . mjcs-prod:mjcs
docs:
rm -rf $(DOCS_DIR)
mkdir -p $(DOCS_DIR)
$(call create_docs_f,prod)
clean:
rm -rf $(PACKAGE_DIR)
rm -f .package-*
rm -f $(foreach component,static docker-repo spider scraper parser,cloudformation/stack-$(component)-output.yaml)
rm -rf $(DOCS_DIR)
clean_all: clean
rm -f .deploy-*
rm -f .create-stack-buckets
rm -f .init-*