split codes into multi modules

chatopera · Mar 24, 2018 · 4cc3313 · 4cc3313
1 parent 78434cc
commit 4cc3313
Show file tree

Hide file tree

Showing 16 changed files with 54 additions and 25 deletions.
diff --git a/admin/eager.ewt.test.sh b/admin/eager.ewt.test.sh
@@ -11,7 +11,7 @@ baseDir=$(cd `dirname "$0"`;pwd)
 #######################
 # variables
 #######################
-PY=$baseDir/../app/eager.py
+PY=$baseDir/../app/transitionparser/eager.py
 MODEL=$baseDir/../tmp/eager.ewt.model
 TEST_DATA=$baseDir/../data/UD_English-EWT/en-ud-test.conllu
 TEST_RESULT=$baseDir/../tmp/en-ud-test.eager.results

diff --git a/admin/eager.ewt.train.sh b/admin/eager.ewt.train.sh
@@ -11,7 +11,7 @@ baseDir=$(cd `dirname "$0"`;pwd)
 #######################
 # variables
 #######################
-PY=$baseDir/../app/eager.py
+PY=$baseDir/../app/transitionparser/eager.py
 TRAIN_DATA=$baseDir/../data/UD_English-EWT/en-ud-train.conllu
 MODEL=$baseDir/../tmp/eager.ewt.model
 EPOCH=10
@@ -23,4 +23,5 @@ LOG_VERBOSITY=0 # info
 # main 
 [ -z "${BASH_SOURCE[0]}" -o "${BASH_SOURCE[0]}" = "$0" ] || return
 set -x
+cd $baseDir/../app
 train $PY $LOG_VERBOSITY $MODEL $TRAIN_DATA $EPOCH 
diff --git a/admin/eager.thu.test.sh b/admin/eager.thu.test.sh
@@ -11,7 +11,7 @@ baseDir=$(cd `dirname "$0"`;pwd)
 #######################
 # variables
 #######################
-PY=$baseDir/../app/eager.py
+PY=$baseDir/../app/transitionparser/eager.py
 MODEL=$baseDir/../tmp/eager.thu.model
 TEST_DATA=$baseDir/../data/evsam05/THU/dev.conllu
 TEST_RESULT=$baseDir/../tmp/en-ud-test.eager.results

diff --git a/admin/eager.thu.train.sh b/admin/eager.thu.train.sh
@@ -11,7 +11,7 @@ baseDir=$(cd `dirname "$0"`;pwd)
 #######################
 # variables
 #######################
-PY=$baseDir/../app/eager.py
+PY=$baseDir/../app/transitionparser/eager.py
 TRAIN_DATA=$baseDir/../data/evsam05/THU/train.conllu
 MODEL=$baseDir/../tmp/eager.thu.model
 EPOCH=10

diff --git a/admin/standard.ewt.test.sh b/admin/standard.ewt.test.sh
@@ -11,7 +11,7 @@ baseDir=$(cd `dirname "$0"`;pwd)
 #######################
 # variables
 #######################
-PY=$baseDir/../app/standard.py
+PY=$baseDir/../app/transitionparser/standard.py
 MODEL=$baseDir/../tmp/standard.ewt.model
 TEST_DATA=$baseDir/../data/UD_English-EWT/en-ud-test.conllu
 TEST_RESULT=$baseDir/../tmp/en-ud-test.standard.results

diff --git a/admin/standard.ewt.train.sh b/admin/standard.ewt.train.sh
@@ -11,7 +11,7 @@ baseDir=$(cd `dirname "$0"`;pwd)
 #######################
 # variables
 #######################
-PY=$baseDir/../app/standard.py
+PY=$baseDir/../app/transitionparser/standard.py
 TRAIN_DATA=$baseDir/../data/conll.example
 # TRAIN_DATA=$baseDir/../data/UD_English-EWT/en-ud-train.conllu
 # MODEL=$baseDir/../tmp/standard.ewt.model

diff --git a/admin/standard.thu.test.sh b/admin/standard.thu.test.sh
@@ -11,7 +11,7 @@ baseDir=$(cd `dirname "$0"`;pwd)
 #######################
 # variables
 #######################
-PY=$baseDir/../app/standard.py
+PY=$baseDir/../app/transitionparser/standard.py
 MODEL=$baseDir/../tmp/eager.thu.model
 TEST_DATA=$baseDir/../data/evsam05/THU/dev.conllu
 TEST_RESULT=$baseDir/../tmp/en-ud-test.standard.results

diff --git a/admin/standard.thu.train.sh b/admin/standard.thu.train.sh
@@ -11,7 +11,7 @@ baseDir=$(cd `dirname "$0"`;pwd)
 #######################
 # variables
 #######################
-PY=$baseDir/../app/standard.py
+PY=$baseDir/../app/transitionparser/standard.py
 TRAIN_DATA=$baseDir/../data/evsam05/THU/train.conllu
 MODEL=$baseDir/../tmp/standard.thu.model
 EPOCH=10

diff --git a/app/isprojective.py → app/common/utils.py b/app/isprojective.py → app/common/utils.py
diff --git a/app/transitionparser/__init__.py b/app/transitionparser/__init__.py
diff --git a/app/configurations.py → app/transitionparser/configurations.py b/app/configurations.py → app/transitionparser/configurations.py
@@ -18,9 +18,23 @@
 
 Author: Yoav Goldberg ([email protected])
 """
+from __future__ import print_function
+from __future__ import division
+
+import os
+import sys
+curdir = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(os.path.join(curdir, os.path.pardir))
+
+if sys.version_info[0] < 3:
+    reload(sys)
+    sys.setdefaultencoding("utf-8")
+    # raise "Must be using Python 3"
+
 from absl import logging
 from common import *
 from common.deps import DependenciesCollection
+from common.exceptions import *
 
 # learners
 class Configuration:  # {{{

diff --git a/app/deciders.py → app/transitionparser/deciders.py b/app/deciders.py → app/transitionparser/deciders.py
@@ -22,7 +22,7 @@
 import os
 import sys
 curdir = os.path.dirname(os.path.abspath(__file__))
-sys.path.append(curdir)
+sys.path.append(os.path.join(curdir, os.path.pardir))
 
 if sys.version_info[0] < 3:
     reload(sys)

diff --git a/app/eager.py → app/transitionparser/eager.py b/app/eager.py → app/transitionparser/eager.py
@@ -24,7 +24,7 @@
 import os
 import sys
 curdir = os.path.dirname(os.path.abspath(__file__))
-sys.path.append(curdir)
+sys.path.insert(0, os.path.join(curdir, os.path.pardir))
 
 if sys.version_info[0] < 3:
     reload(sys)
@@ -37,10 +37,11 @@
 
 from ml import ml
 from pio import io
-from oracles import *
-from deciders import *
-from transitionparser import *
+from transitionparser.oracles import *
+from transitionparser.deciders import *
+from transitionparser.parsers import *
 from features import extractors
+from common.utils import is_projective
 
 FLAGS = flags.FLAGS
 '''
@@ -52,23 +53,23 @@
 flags.DEFINE_boolean('lazypop', True, 'Lazy pop.')   # nopep8
 flags.DEFINE_boolean('unlex', False, 'unlex')   # nopep8
 flags.DEFINE_string('feature_extarctor', 'eager.zhang', 'Feature Extarctor')  # nopep8
-flags.DEFINE_string('model', os.path.join(curdir, os.path.pardir, "tmp", "eager.model"), 'Transition Parser Model.')  # nopep8
+flags.DEFINE_string('model', os.path.join(curdir, os.path.pardir, os.path.pardir, "tmp", "eager.model"), 'Transition Parser Model.')  # nopep8
 
 '''
 Train
 '''
 flags.DEFINE_boolean('train', False, 'Train model with train data')  # nopep8
 flags.DEFINE_integer('epoch', 1, 'Train Epoch.')  # nopep8
-flags.DEFINE_string('train_data', os.path.join(curdir, os.path.pardir, "data", "conll.example"), 'Train Data')  # nopep8
+flags.DEFINE_string('train_data', os.path.join(curdir, os.path.pardir, os.path.pardir, "data", "conll.example"), 'Train Data')  # nopep8
 flags.DEFINE_string('externaltrainfile', None, 'External Train File.')  # nopep8
 # flags.DEFINE_string('modelfile', 'data/weights', 'Model File.')
 
 '''
 Test
 '''
 flags.DEFINE_boolean('test', False, 'Evalutate with test data')  # nopep8
-flags.DEFINE_string('test_data', os.path.join(curdir, os.path.pardir, "data", "conll.example"), 'Test data.')  # nopep8
-flags.DEFINE_string('test_results', os.path.join(curdir, os.path.pardir, "tmp", "eager.test.results"), 'Save scores into disk.')  # nopep8
+flags.DEFINE_string('test_data', os.path.join(curdir, os.path.pardir, os.path.pardir, "data", "conll.example"), 'Test data.')  # nopep8
+flags.DEFINE_string('test_results', os.path.join(curdir, os.path.pardir, os.path.pardir, "tmp", "eager.test.results"), 'Save scores into disk.')  # nopep8
 
 
 def transform_conll_sents(conll_file_path):
@@ -78,8 +79,7 @@ def transform_conll_sents(conll_file_path):
     sents = list(io.conll_to_sents(file(conll_file_path)))
 
     if FLAGS.only_projective:
-        import isprojective
-        sents = [s for s in sents if isprojective.is_projective(s)]
+        sents = [s for s in sents if is_projective(s)]
 
     if FLAGS.unlex:
         from shared.lemmatize import EnglishMinimalWordSmoother

diff --git a/app/oracles.py → app/transitionparser/oracles.py b/app/oracles.py → app/transitionparser/oracles.py
@@ -16,6 +16,18 @@
 Transition based parsing (both arc-standard and arc-eager).
 Easily extended to support other variants.
 """
+from __future__ import print_function
+from __future__ import division
+
+import os
+import sys
+curdir = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(os.path.join(curdir, os.path.pardir))
+
+if sys.version_info[0] < 3:
+    reload(sys)
+    sys.setdefaultencoding("utf-8")
+    # raise "Must be using Python 3"
 
 from absl import logging
 from collections import defaultdict

diff --git a/app/transitionparser.py → app/transitionparser/parsers.py b/app/transitionparser.py → app/transitionparser/parsers.py
@@ -22,7 +22,7 @@
 import os
 import sys
 curdir = os.path.dirname(os.path.abspath(__file__))
-sys.path.append(curdir)
+sys.path.append(os.path.join(curdir, os.path.pardir))
 
 if sys.version_info[0] < 3:
     reload(sys)
@@ -39,7 +39,7 @@
 
 from common import *
 from common.exceptions import *
-from configurations import *
+from transitionparser.configurations import *
 
 class TransitionBasedParser:
     """

diff --git a/app/standard.py → app/transitionparser/standard.py b/app/standard.py → app/transitionparser/standard.py
@@ -34,7 +34,7 @@
 import os
 import sys
 curdir = os.path.dirname(os.path.abspath(__file__))
-sys.path.append(curdir)
+sys.path.insert(0, os.path.join(curdir, os.path.pardir))
 
 if sys.version_info[0] < 3:
     reload(sys)
@@ -47,8 +47,11 @@
 
 from ml import ml
 from pio import io
-from transitionparser import *
+from transitionparser.oracles import *
+from transitionparser.deciders import *
+from transitionparser.parsers import *
 from features import extractors
+from common.utils import is_projective
 
 FLAGS = flags.FLAGS
 '''
@@ -95,8 +98,7 @@ def transform_conll_sents(conll_file_path):
     sents = list(io.conll_to_sents(file(conll_file_path)))
 
     if FLAGS.only_projective:
-        import isprojective
-        sents = [s for s in sents if isprojective.is_projective(s)]
+        sents = [s for s in sents if is_projective(s)]
 
     if FLAGS.unlex:
         from shared.lemmatize import EnglishMinimalWordSmoother