From 4cc3313f5336d3aa329c56df549e1058b40caf58 Mon Sep 17 00:00:00 2001 From: Hai Liang Wang Date: Sat, 24 Mar 2018 17:16:27 +0800 Subject: [PATCH] split codes into multi modules --- admin/eager.ewt.test.sh | 2 +- admin/eager.ewt.train.sh | 3 ++- admin/eager.thu.test.sh | 2 +- admin/eager.thu.train.sh | 2 +- admin/standard.ewt.test.sh | 2 +- admin/standard.ewt.train.sh | 2 +- admin/standard.thu.test.sh | 2 +- admin/standard.thu.train.sh | 2 +- app/{isprojective.py => common/utils.py} | 0 app/transitionparser/__init__.py | 0 app/{ => transitionparser}/configurations.py | 14 +++++++++++++ app/{ => transitionparser}/deciders.py | 2 +- app/{ => transitionparser}/eager.py | 20 +++++++++---------- app/{ => transitionparser}/oracles.py | 12 +++++++++++ .../parsers.py} | 4 ++-- app/{ => transitionparser}/standard.py | 10 ++++++---- 16 files changed, 54 insertions(+), 25 deletions(-) rename app/{isprojective.py => common/utils.py} (100%) create mode 100644 app/transitionparser/__init__.py rename app/{ => transitionparser}/configurations.py (95%) rename app/{ => transitionparser}/deciders.py (99%) rename app/{ => transitionparser}/eager.py (92%) rename app/{ => transitionparser}/oracles.py (94%) rename app/{transitionparser.py => transitionparser/parsers.py} (97%) rename app/{ => transitionparser}/standard.py (94%) diff --git a/admin/eager.ewt.test.sh b/admin/eager.ewt.test.sh index f88434c..ecd9bd8 100755 --- a/admin/eager.ewt.test.sh +++ b/admin/eager.ewt.test.sh @@ -11,7 +11,7 @@ baseDir=$(cd `dirname "$0"`;pwd) ####################### # variables ####################### -PY=$baseDir/../app/eager.py +PY=$baseDir/../app/transitionparser/eager.py MODEL=$baseDir/../tmp/eager.ewt.model TEST_DATA=$baseDir/../data/UD_English-EWT/en-ud-test.conllu TEST_RESULT=$baseDir/../tmp/en-ud-test.eager.results diff --git a/admin/eager.ewt.train.sh b/admin/eager.ewt.train.sh index 80a7d8a..fbe8604 100755 --- a/admin/eager.ewt.train.sh +++ b/admin/eager.ewt.train.sh @@ -11,7 +11,7 @@ baseDir=$(cd `dirname "$0"`;pwd) ####################### # variables ####################### -PY=$baseDir/../app/eager.py +PY=$baseDir/../app/transitionparser/eager.py TRAIN_DATA=$baseDir/../data/UD_English-EWT/en-ud-train.conllu MODEL=$baseDir/../tmp/eager.ewt.model EPOCH=10 @@ -23,4 +23,5 @@ LOG_VERBOSITY=0 # info # main [ -z "${BASH_SOURCE[0]}" -o "${BASH_SOURCE[0]}" = "$0" ] || return set -x +cd $baseDir/../app train $PY $LOG_VERBOSITY $MODEL $TRAIN_DATA $EPOCH diff --git a/admin/eager.thu.test.sh b/admin/eager.thu.test.sh index 87daf84..dd5bd22 100755 --- a/admin/eager.thu.test.sh +++ b/admin/eager.thu.test.sh @@ -11,7 +11,7 @@ baseDir=$(cd `dirname "$0"`;pwd) ####################### # variables ####################### -PY=$baseDir/../app/eager.py +PY=$baseDir/../app/transitionparser/eager.py MODEL=$baseDir/../tmp/eager.thu.model TEST_DATA=$baseDir/../data/evsam05/THU/dev.conllu TEST_RESULT=$baseDir/../tmp/en-ud-test.eager.results diff --git a/admin/eager.thu.train.sh b/admin/eager.thu.train.sh index cabba5d..a1e748d 100755 --- a/admin/eager.thu.train.sh +++ b/admin/eager.thu.train.sh @@ -11,7 +11,7 @@ baseDir=$(cd `dirname "$0"`;pwd) ####################### # variables ####################### -PY=$baseDir/../app/eager.py +PY=$baseDir/../app/transitionparser/eager.py TRAIN_DATA=$baseDir/../data/evsam05/THU/train.conllu MODEL=$baseDir/../tmp/eager.thu.model EPOCH=10 diff --git a/admin/standard.ewt.test.sh b/admin/standard.ewt.test.sh index 3d674f1..c198907 100755 --- a/admin/standard.ewt.test.sh +++ b/admin/standard.ewt.test.sh @@ -11,7 +11,7 @@ baseDir=$(cd `dirname "$0"`;pwd) ####################### # variables ####################### -PY=$baseDir/../app/standard.py +PY=$baseDir/../app/transitionparser/standard.py MODEL=$baseDir/../tmp/standard.ewt.model TEST_DATA=$baseDir/../data/UD_English-EWT/en-ud-test.conllu TEST_RESULT=$baseDir/../tmp/en-ud-test.standard.results diff --git a/admin/standard.ewt.train.sh b/admin/standard.ewt.train.sh index 9a2641d..df65c2b 100755 --- a/admin/standard.ewt.train.sh +++ b/admin/standard.ewt.train.sh @@ -11,7 +11,7 @@ baseDir=$(cd `dirname "$0"`;pwd) ####################### # variables ####################### -PY=$baseDir/../app/standard.py +PY=$baseDir/../app/transitionparser/standard.py TRAIN_DATA=$baseDir/../data/conll.example # TRAIN_DATA=$baseDir/../data/UD_English-EWT/en-ud-train.conllu # MODEL=$baseDir/../tmp/standard.ewt.model diff --git a/admin/standard.thu.test.sh b/admin/standard.thu.test.sh index e4b35b6..47e443b 100755 --- a/admin/standard.thu.test.sh +++ b/admin/standard.thu.test.sh @@ -11,7 +11,7 @@ baseDir=$(cd `dirname "$0"`;pwd) ####################### # variables ####################### -PY=$baseDir/../app/standard.py +PY=$baseDir/../app/transitionparser/standard.py MODEL=$baseDir/../tmp/eager.thu.model TEST_DATA=$baseDir/../data/evsam05/THU/dev.conllu TEST_RESULT=$baseDir/../tmp/en-ud-test.standard.results diff --git a/admin/standard.thu.train.sh b/admin/standard.thu.train.sh index 2a0b293..1a70163 100755 --- a/admin/standard.thu.train.sh +++ b/admin/standard.thu.train.sh @@ -11,7 +11,7 @@ baseDir=$(cd `dirname "$0"`;pwd) ####################### # variables ####################### -PY=$baseDir/../app/standard.py +PY=$baseDir/../app/transitionparser/standard.py TRAIN_DATA=$baseDir/../data/evsam05/THU/train.conllu MODEL=$baseDir/../tmp/standard.thu.model EPOCH=10 diff --git a/app/isprojective.py b/app/common/utils.py similarity index 100% rename from app/isprojective.py rename to app/common/utils.py diff --git a/app/transitionparser/__init__.py b/app/transitionparser/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/configurations.py b/app/transitionparser/configurations.py similarity index 95% rename from app/configurations.py rename to app/transitionparser/configurations.py index a8f9232..3cb0ee8 100644 --- a/app/configurations.py +++ b/app/transitionparser/configurations.py @@ -18,9 +18,23 @@ Author: Yoav Goldberg (yoav.goldberg@gmail.com) """ +from __future__ import print_function +from __future__ import division + +import os +import sys +curdir = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.join(curdir, os.path.pardir)) + +if sys.version_info[0] < 3: + reload(sys) + sys.setdefaultencoding("utf-8") + # raise "Must be using Python 3" + from absl import logging from common import * from common.deps import DependenciesCollection +from common.exceptions import * # learners class Configuration: # {{{ diff --git a/app/deciders.py b/app/transitionparser/deciders.py similarity index 99% rename from app/deciders.py rename to app/transitionparser/deciders.py index 3d04e20..a1500e6 100644 --- a/app/deciders.py +++ b/app/transitionparser/deciders.py @@ -22,7 +22,7 @@ import os import sys curdir = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(curdir) +sys.path.append(os.path.join(curdir, os.path.pardir)) if sys.version_info[0] < 3: reload(sys) diff --git a/app/eager.py b/app/transitionparser/eager.py similarity index 92% rename from app/eager.py rename to app/transitionparser/eager.py index 7c8d74b..a73d42f 100755 --- a/app/eager.py +++ b/app/transitionparser/eager.py @@ -24,7 +24,7 @@ import os import sys curdir = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(curdir) +sys.path.insert(0, os.path.join(curdir, os.path.pardir)) if sys.version_info[0] < 3: reload(sys) @@ -37,10 +37,11 @@ from ml import ml from pio import io -from oracles import * -from deciders import * -from transitionparser import * +from transitionparser.oracles import * +from transitionparser.deciders import * +from transitionparser.parsers import * from features import extractors +from common.utils import is_projective FLAGS = flags.FLAGS ''' @@ -52,14 +53,14 @@ flags.DEFINE_boolean('lazypop', True, 'Lazy pop.') # nopep8 flags.DEFINE_boolean('unlex', False, 'unlex') # nopep8 flags.DEFINE_string('feature_extarctor', 'eager.zhang', 'Feature Extarctor') # nopep8 -flags.DEFINE_string('model', os.path.join(curdir, os.path.pardir, "tmp", "eager.model"), 'Transition Parser Model.') # nopep8 +flags.DEFINE_string('model', os.path.join(curdir, os.path.pardir, os.path.pardir, "tmp", "eager.model"), 'Transition Parser Model.') # nopep8 ''' Train ''' flags.DEFINE_boolean('train', False, 'Train model with train data') # nopep8 flags.DEFINE_integer('epoch', 1, 'Train Epoch.') # nopep8 -flags.DEFINE_string('train_data', os.path.join(curdir, os.path.pardir, "data", "conll.example"), 'Train Data') # nopep8 +flags.DEFINE_string('train_data', os.path.join(curdir, os.path.pardir, os.path.pardir, "data", "conll.example"), 'Train Data') # nopep8 flags.DEFINE_string('externaltrainfile', None, 'External Train File.') # nopep8 # flags.DEFINE_string('modelfile', 'data/weights', 'Model File.') @@ -67,8 +68,8 @@ Test ''' flags.DEFINE_boolean('test', False, 'Evalutate with test data') # nopep8 -flags.DEFINE_string('test_data', os.path.join(curdir, os.path.pardir, "data", "conll.example"), 'Test data.') # nopep8 -flags.DEFINE_string('test_results', os.path.join(curdir, os.path.pardir, "tmp", "eager.test.results"), 'Save scores into disk.') # nopep8 +flags.DEFINE_string('test_data', os.path.join(curdir, os.path.pardir, os.path.pardir, "data", "conll.example"), 'Test data.') # nopep8 +flags.DEFINE_string('test_results', os.path.join(curdir, os.path.pardir, os.path.pardir, "tmp", "eager.test.results"), 'Save scores into disk.') # nopep8 def transform_conll_sents(conll_file_path): @@ -78,8 +79,7 @@ def transform_conll_sents(conll_file_path): sents = list(io.conll_to_sents(file(conll_file_path))) if FLAGS.only_projective: - import isprojective - sents = [s for s in sents if isprojective.is_projective(s)] + sents = [s for s in sents if is_projective(s)] if FLAGS.unlex: from shared.lemmatize import EnglishMinimalWordSmoother diff --git a/app/oracles.py b/app/transitionparser/oracles.py similarity index 94% rename from app/oracles.py rename to app/transitionparser/oracles.py index 699a5b6..18885d7 100644 --- a/app/oracles.py +++ b/app/transitionparser/oracles.py @@ -16,6 +16,18 @@ Transition based parsing (both arc-standard and arc-eager). Easily extended to support other variants. """ +from __future__ import print_function +from __future__ import division + +import os +import sys +curdir = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.join(curdir, os.path.pardir)) + +if sys.version_info[0] < 3: + reload(sys) + sys.setdefaultencoding("utf-8") + # raise "Must be using Python 3" from absl import logging from collections import defaultdict diff --git a/app/transitionparser.py b/app/transitionparser/parsers.py similarity index 97% rename from app/transitionparser.py rename to app/transitionparser/parsers.py index 4fa2e73..0d58385 100644 --- a/app/transitionparser.py +++ b/app/transitionparser/parsers.py @@ -22,7 +22,7 @@ import os import sys curdir = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(curdir) +sys.path.append(os.path.join(curdir, os.path.pardir)) if sys.version_info[0] < 3: reload(sys) @@ -39,7 +39,7 @@ from common import * from common.exceptions import * -from configurations import * +from transitionparser.configurations import * class TransitionBasedParser: """ diff --git a/app/standard.py b/app/transitionparser/standard.py similarity index 94% rename from app/standard.py rename to app/transitionparser/standard.py index 028688b..35b51ab 100755 --- a/app/standard.py +++ b/app/transitionparser/standard.py @@ -34,7 +34,7 @@ import os import sys curdir = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(curdir) +sys.path.insert(0, os.path.join(curdir, os.path.pardir)) if sys.version_info[0] < 3: reload(sys) @@ -47,8 +47,11 @@ from ml import ml from pio import io -from transitionparser import * +from transitionparser.oracles import * +from transitionparser.deciders import * +from transitionparser.parsers import * from features import extractors +from common.utils import is_projective FLAGS = flags.FLAGS ''' @@ -95,8 +98,7 @@ def transform_conll_sents(conll_file_path): sents = list(io.conll_to_sents(file(conll_file_path))) if FLAGS.only_projective: - import isprojective - sents = [s for s in sents if isprojective.is_projective(s)] + sents = [s for s in sents if is_projective(s)] if FLAGS.unlex: from shared.lemmatize import EnglishMinimalWordSmoother