-
Notifications
You must be signed in to change notification settings - Fork 1
/
Makefile
114 lines (78 loc) · 5.95 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
.PHONY: install test concepts adaptation barriers-and-challenges capacity-building climate-related-hazards deforestation equity-and-just-transition fossil-fuels good-practice-and-opportunities greenhouse-gases international-cooperation loss-and-damage mitigation renewables response-measures technologies-adaptation technologies-mitigation vulnerable-groups split_spans_csvs sync_concepts_with_s3 concepts_non_ml concepts_with_ml concepts_classifiers concepts
include .env
install:
git init
poetry install
poetry run pre-commit install
poetry run ipython kernel install --user
test: install
poetry run python -m pytest -vvv
build:
docker-compose build
up:
docker-compose up
index_data:
poetry run python -m src.opensearch.index_data ${DOCS_DIR_GST} ${SCRAPER_CSV_PATH} ./concepts -i global-stocktake
## Elastic Beanstalk
archive:
git archive -v -o ebs_archive.zip --add-file=.env --format=zip HEAD
# needs explorer to be installed in the environment this is run in
cop28:
explorer gst --spans-csv-filename ${SPANS_CSV_FILENAME} -i ./concepts/5-COP28-GST-asks/input.xlsx -d ${DOCS_DIR_GST} -o ./concepts/5-COP28-GST-asks
adaptation:
explorer gst --spans-csv-filename ${SPANS_CSV_FILENAME} -i ./concepts/adaptation/input.xlsx -d ${DOCS_DIR_GST} -o ./concepts/adaptation
barriers-and-challenges:
explorer gst --spans-csv-filename ${SPANS_CSV_FILENAME} -i ./concepts/barriers-and-challenges/input.xlsx -d ${DOCS_DIR_GST} -o ./concepts/barriers-and-challenges
capacity-building:
explorer gst --spans-csv-filename ${SPANS_CSV_FILENAME} -i ./concepts/capacity-building/input.xlsx -d ${DOCS_DIR_GST} -o ./concepts/capacity-building
climate-related-hazards:
explorer gst --spans-csv-filename ${SPANS_CSV_FILENAME} -i ./concepts/climate-related-hazards/input.xlsx -d ${DOCS_DIR_GST} -o ./concepts/climate-related-hazards
deforestation:
explorer gst --spans-csv-filename ${SPANS_CSV_FILENAME} -i ./concepts/deforestation/input.xlsx -d ${DOCS_DIR_GST} -o ./concepts/deforestation
equity-and-just-transition:
explorer gst --spans-csv-filename ${SPANS_CSV_FILENAME} -i ./concepts/equity-and-just-transition/input.xlsx -d ${DOCS_DIR_GST} -o ./concepts/equity-and-just-transition
financial-flows:
explorer gst --spans-csv-filename ${SPANS_CSV_FILENAME} -t -i ./concepts/financial-flows/input.xlsx -d ${DOCS_DIR_GST} -o ./concepts/financial-flows
fossil-fuels:
explorer gst --spans-csv-filename ${SPANS_CSV_FILENAME} -i ./concepts/fossil-fuels/input.xlsx -d ${DOCS_DIR_GST} -o ./concepts/fossil-fuels
good-practice-and-opportunities:
explorer gst --spans-csv-filename ${SPANS_CSV_FILENAME} -i ./concepts/good-practice-and-opportunities/input.xlsx -d ${DOCS_DIR_GST} -o ./concepts/good-practice-and-opportunities
greenhouse-gases:
explorer gst --spans-csv-filename ${SPANS_CSV_FILENAME} -i ./concepts/greenhouse-gases/input.xlsx -d ${DOCS_DIR_GST} -o ./concepts/greenhouse-gases
international-cooperation:
explorer gst --spans-csv-filename ${SPANS_CSV_FILENAME} -i ./concepts/international-cooperation/input.xlsx -d ${DOCS_DIR_GST} -o ./concepts/international-cooperation
loss-and-damage:
explorer gst --spans-csv-filename ${SPANS_CSV_FILENAME} -i ./concepts/loss-and-damage/input.xlsx -d ${DOCS_DIR_GST} -o ./concepts/loss-and-damage
mitigation:
explorer gst --spans-csv-filename ${SPANS_CSV_FILENAME} -i ./concepts/mitigation/input.xlsx -d ${DOCS_DIR_GST} -o ./concepts/mitigation
renewables:
explorer gst --spans-csv-filename ${SPANS_CSV_FILENAME} -i ./concepts/renewables/input.xlsx -d ${DOCS_DIR_GST} -o ./concepts/renewables
response-measures:
explorer gst --spans-csv-filename ${SPANS_CSV_FILENAME} -i ./concepts/response-measures/input.xlsx -d ${DOCS_DIR_GST} -o ./concepts/response-measures
technologies-adaptation:
explorer gst --spans-csv-filename ${SPANS_CSV_FILENAME} -i ./concepts/technologies-br-adaptation-br/input.xlsx -d ${DOCS_DIR_GST} -o ./concepts/technologies-br-adaptation-br
technologies-mitigation:
explorer gst --spans-csv-filename ${SPANS_CSV_FILENAME} -i ./concepts/technologies-br-mitigation-br/input.xlsx -d ${DOCS_DIR_GST} -o ./concepts/technologies-br-mitigation-br
vulnerable-groups:
explorer gst --spans-csv-filename ${SPANS_CSV_FILENAME} -i ./concepts/vulnerable-groups/input.xlsx -d ${DOCS_DIR_GST} -o ./concepts/vulnerable-groups
train_sector_classifier:
poetry run python classifiers/trainer.py --argilla-dataset-name sector-text-classifier
train_instruments_classifier:
poetry run python classifiers/trainer.py --argilla-dataset-name policy-instrument-text-classifier
# NOTE: these should be run against the *best* model artifact, not the latest
run_sector_classifier:
poetry run python -m classifiers.run_on_full_dataset --spans-csv-filename ${SPANS_CSV_FILENAME} --wandb-artifact-name climatepolicyradar/sector-text-classifier/sector-text-classifier:latest --output-dir ./concepts/sectors --extra-output
run_instruments_classifier:
poetry run python -m classifiers.run_on_full_dataset --spans-csv-filename ${SPANS_CSV_FILENAME} --wandb-artifact-name climatepolicyradar/policy-instrument-text-classifier/policy-instrument-text-classifier:latest --output-dir ./concepts/policy-instruments --extra-output
# split spans csvs into smaller chunks that can be pushed to git
split_spans_csvs:
python src/data/split_spans_csvs.py ${SPANS_CSV_FILENAME}
# TODO: split_spans_csvs doesn't work as results are in subdirectories
concepts_non_ml: adaptation barriers-and-challenges capacity-building climate-related-hazards deforestation equity-and-just-transition fossil-fuels good-practice-and-opportunities greenhouse-gases international-cooperation loss-and-damage mitigation renewables response-measures technologies-adaptation technologies-mitigation vulnerable-groups
concepts_with_ml: financial-flows
concepts_classifiers: run_sector_classifier run_instruments_classifier
concepts: concepts_non_ml concepts_with_ml concepts_classifiers #split_spans_csvs
sync_concepts_to_s3:
aws s3 sync ./concepts s3://cpr-dataset-gst-concepts
sync_concepts_from_s3:
aws s3 sync s3://cpr-dataset-gst-concepts ./concepts