-
Notifications
You must be signed in to change notification settings - Fork 6
/
Makefile
238 lines (201 loc) · 9.11 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
SHELL = /bin/bash
CONTROLLER_ENV_UID ?= $(shell id -u)
CONTROLLER_ENV_GID ?= $(shell id -g)
MANAGER_ENV_UID ?= $(shell id -u)
MANAGER_ENV_GID ?= $(shell id -g)
COMPOSE_PROFILES ?= with-kitodo-production,with-ocrd-controller
.EXPORT_ALL_VARIABLES:
# conditional assignment operator ?= creates deferred variables only, so use conditionals directly:
ifndef APP_DATA
APP_DATA != source .env && echo $$APP_DATA # ${PWD}/kitodo/data
endif
ifndef APP_KEY
APP_KEY != source .env && echo $$APP_KEY # ./kitodo/.ssh/id_rsa
endif
ifndef APP_PORT
APP_PORT != source .env && echo $$APP_PORT # 8080
endif
ifndef MANAGER_KEYS
MANAGER_KEYS != source .env && echo $$MANAGER_KEYS # ./ocrd/manager/.ssh/authorized_keys
endif
ifndef MANAGER_KEY
MANAGER_KEY != source .env && echo $$MANAGER_KEY # ./ocrd/manager/.ssh/id_rsa
endif
ifndef MANAGER_DATA
MANAGER_DATA != source .env && echo $$MANAGER_DATA # ./kitodo/data/metadata/
endif
ifndef MANAGER_WORKFLOWS
MANAGER_WORKFLOWS != source .env && echo $$MANAGER_WORKFLOWS # ./_modules/ocrd_manager/workflows
endif
ifndef CONTROLLER_KEYS
CONTROLLER_KEYS != source .env && echo $$CONTROLLER_KEYS # ./ocrd/controller/.ssh/authorized_keys
endif
ifndef CONTROLLER_DATA
CONTROLLER_DATA != source .env && echo $$CONTROLLER_DATA # ./ocrd/controller/data
endif
ifndef CONTROLLER_MODELS
CONTROLLER_MODELS != source .env && echo $$CONTROLLER_MODELS # ./ocrd/controller/models
endif
ifndef CONTROLLER_CONFIG
CONTROLLER_CONFIG != source .env && echo $$CONTROLLER_CONFIG # ./ocrd/controller/config
endif
ifndef CONTROLLER_PORT_SSH
CONTROLLER_PORT_SSH != source .env && echo $$CONTROLLER_PORT_SSH # ./ocrd/controller/config
endif
# removes (default) files and directories of prepare target
clean: clean-testdata
$(RM) -fr kitodo ocrd
# private SSH key for login from Production to Manager
prepare-keys: $(APP_KEY)
# public SSH keys for logins allowed on Manager
prepare-keys: $(MANAGER_KEYS)
# private SSH key for login from Manager to Controller
prepare-keys: $(MANAGER_KEY)
# public SSH keys for logins allowed on Controller
prepare-keys: $(CONTROLLER_KEYS)
# mount-point for default data volume of kitodo-app
prepare-keys: | $(MANAGER_DATA)/
# mount-point for default data volume of ocrd-controller
prepare-keys: | $(CONTROLLER_DATA)/
# mount-point for default model volume of ocrd-controller
prepare-keys: | $(CONTROLLER_MODELS)/
# mount-point for default config volume of ocrd-controller
prepare-keys: | $(CONTROLLER_CONFIG)/
# general data for Kitodo.Production
prepare-data: ./kitodo/before_startup.sh ./kitodo/overwrites/data
ifneq ($(findstring with-kitodo-production,$(COMPOSE_PROFILES)),)
# example data for Kitodo.Production (users, projects, processes, workflows, ...)
prepare-examples: ./kitodo/overwrites/sql
endif
ifneq ($(findstring with-ocrd-controller,$(COMPOSE_PROFILES)),)
# initial OCR model for Controller
prepare-examples: | $(CONTROLLER_MODELS)/ocrd-resources/ocrd-tesserocr-recognize/frak2021.traineddata
endif
prepare: prepare-keys prepare-data prepare-examples
./%/:
mkdir -p $@
# generate private SSH key for login from Production to Manager
$(APP_KEY):
mkdir -p $(@D)
ssh-keygen -t rsa -q -f $@ -P '' -C 'Kitodo.Production key'
ifeq ($(findstring with-kitodo-production,$(COMPOSE_PROFILES)),)
@echo >&2 " You should now install the private key $@"
@echo >&2 " to your own Kitodo.Production instance,"
@echo >&2 " or conversely, install the existing public key ~/.ssh/id_rsa.pub"
@echo >&2 " of your own Kitodo.Production instance"
@echo >&2 " to $(MANAGER_KEYS)."
endif
# generate private SSH key for login from Manager to Controller
$(MANAGER_KEY):
mkdir -p $(@D)
ssh-keygen -t rsa -q -f $@ -P '' -C 'OCR-D manager key'
ifeq ($(findstring with-ocrd-controller,$(COMPOSE_PROFILES)),)
@echo >&2 " You should now install the public key [email protected]"
@echo >&2 " to your own OCR-D Controller instance,"
@echo >&2 " or conversely, install the existing private key ~/.ssh/id_rsa"
@echo >&2 " for your OCR-D Controller instance to $@"
endif
# derive public SSH keys for logins allowed on Controller from private SSH key for login from Manager
$(CONTROLLER_KEYS): $(MANAGER_KEY)
mkdir -p $(@D)
cp $<.pub $@
# derive public SSH keys for logins allowed on Controller from private SSH key for login from Manager
$(MANAGER_KEYS): $(APP_KEY)
mkdir -p $(@D)
cp $<.pub $@
# copy before startup script (SSH configuration)
./kitodo/before_startup.sh:
cp ./_resources/kitodo/before_startup.sh $@
# copy prebuilt data for Production (scripts, OCR-D workflows)
./kitodo/overwrites/data: | ./kitodo/overwrites/
cp -r ./_resources/kitodo/data $@
ifeq ($(findstring with-kitodo-production,$(COMPOSE_PROFILES)),)
@echo >&2 " You should now copy $@/scripts"
@echo >&2 " to your own Kitodo.Production instance"
@echo >&2 " (typically under /usr/local/kitodo)."
endif
# copy examples for Production (users, projects, processes)
./kitodo/overwrites/sql:
cp -r ./_resources/kitodo-sample/* ./kitodo/overwrites/
# install initial OCR model for Controller
$(CONTROLLER_MODELS)/ocrd-resources/ocrd-tesserocr-recognize/frak2021.traineddata:
mkdir -p $(@D)
wget -O $@ https://ub-backup.bib.uni-mannheim.de/~stweil/tesstrain/frak2021/tessdata_best/frak2021-0.905.traineddata
build:
docker compose build $(SERVICES) --no-cache
rebuild:
docker compose stop $(SERVICES)
docker compose up -d --build $(SERVICES)
start:
docker compose up -d
down:
docker compose down -v
stop:
docker compose stop
config:
docker compose config
pull:
docker compose pull
status:
docker compose ps
$(APP_DATA)/metadata/testdata-kitodo:
mkdir -p $@/images
for page in {00000009..00000014}; do \
wget -P $@/images https://digital.slub-dresden.de/data/kitodo/LankDres_1760234508/LankDres_1760234508_tif/jpegs/$$page.tif.original.jpg; \
done
ifneq ($(findstring with-kitodo-production,$(COMPOSE_PROFILES)),)
test: test-kitodo
endif
test-kitodo: APP_CONTAINER != docker container ls -qf name=kitodo-app
test-kitodo: $(APP_DATA)/metadata/testdata-kitodo
$(if $(APP_CONTAINER),$(info using $(APP_CONTAINER)),$(error must run kitodo-app before test-kitodo))
# remove ocr directory if exist
rm -rf $(APP_DATA)/metadata/testdata-kitodo/ocr
# wait until Kitodo.Production directory structure is initialized
docker exec -t $(APP_CONTAINER) bash -c "/wait-for-it.sh -t 0 ocrd-controller:$$CONTROLLER_PORT_SSH"
docker exec -t $(APP_CONTAINER) bash -c "/wait-for-it.sh -t 0 kitodo-app:$$APP_PORT"
docker exec -t $(APP_CONTAINER) bash -c "/wait-for-it.sh -t 0 ocrd-database:27017"
# run asynchronous ocr processing, which should return within 5 seconds with exit status 1
timeout --preserve-status 5 docker exec $(APP_CONTAINER) bash -c '/usr/local/kitodo/scripts/script_ocr_process_dir.sh --proc-id "testdata-kitodo" --task-id 1'; test $$? = 1
# check with interval of 1 second if ocr folder exists. It fails if the ocr folder is not created within 5 minutes.
timeout 10m bash -c 'until test -s $(APP_DATA)/metadata/testdata-kitodo/ocr/alto/00000014.tif.original.xml; do sleep 5; done'
# rest if the alto directory and file exist
test -d $(APP_DATA)/metadata/testdata-kitodo/ocr/alto
test -s $(APP_DATA)/metadata/testdata-kitodo/ocr/alto/00000009.tif.original.xml
test test-production test-presentation clean-testdata: NETWORK=ocrd_kitodo_default
# if there is no shell override for MANAGER_DATA, then get it from the .env
# (we must download testdata here, the path must match the currently mounted volume)
test test-production test-presentation clean-testdata: DATA=$(or $(MANAGER_DATA),$(shell eval echo `sed -n s/^MANAGER_DATA=//p .env`))
test test-production test-presentation clean-testdata:
$(MAKE) -C _modules/ocrd_manager $@
define HELP
cat <<"EOF"
Targets:
- prepare: create directories, SSH key files and examples
- build: `docker compose build` all images
- start: `docker compose up` all containers (in detached mode)
- down: `docker compose down` all containers (i.e. stop and remove)
- stop: `docker compose stop` all containers (i.e. only stop)
- config: dump all the composed files
- pull: `docker compose pull` latest prebuilt Docker images for all services
- status: `docker compose ps` all running containers
- test: run an example workflow on example data on running containers
- clean-testdata: remove files created by `test`
- clean: remove files created by `prepare` and `test`
Variables:
- CONTROLLER_ENV_UID user id to use on the OCR-D Controller (default: $(CONTROLLER_ENV_UID))
- CONTROLLER_ENV_GID group id to use on the OCR-D Controller (default: $(CONTROLLER_ENV_GID))
- MANAGER_ENV_UID user id to use on the OCR-D Manager (default: $(MANAGER_ENV_UID))
- MANAGER_ENV_GID group id to use on the OCR-D Manager (default: $(MANAGER_ENV_GID))
- COMPOSE_PROFILES contains 'with-ocrd-controller' if OCR-D Controller service should be started/stopped here,
contains 'with-kitodo-production' if Kitodo.Production service should be started/stopped here,
comma-separated (default: $(COMPOSE_PROFILES))
- in addition, all variables defined in .env can be overridden via shell or make
EOF
endef
export HELP
help: ; @eval "$$HELP"
.PHONY: clean clean-testdata prepare prepare-keys prepare-examples build build-kitodo start down config pull status test test-kitodo help
# do not search for implicit rules here:
%.zip: ;
Makefile: ;