Generate native package wrappers

Fixes google#255
xiaoyangegege · Aug 22, 2017 · 205bbd3 · 205bbd3
1 parent 9d80504
commit 205bbd3
Show file tree

Hide file tree

Showing 11 changed files with 236 additions and 167 deletions.
diff --git a/Makefile b/Makefile
@@ -68,6 +68,8 @@ COMPILER_STMT_PASS_FILES := $(patsubst %,$(PY_DIR)/grumpy/compiler/stmt_test.%of
 COMPILER_D_FILES := $(patsubst %,$(PY_DIR)/%.d,$(COMPILER_TESTS))
 COMPILER := $(COMPILER_BIN) $(COMPILER_SRCS) $(PYTHONPARSER_SRCS)
 
+PKGC_BIN := build/bin/pkgc
+
 RUNNER_BIN := build/bin/grumprun
 RUNTIME_SRCS := $(addprefix build/src/grumpy/,$(notdir $(wildcard runtime/*.go)))
 RUNTIME := $(PKG_DIR)/grumpy.a
@@ -182,12 +184,25 @@ $(COMPILER_EXPR_VISITOR_PASS_FILES): $(PY_DIR)/grumpy/compiler/expr_visitor_test
 	@touch $@
 	@echo 'compiler/expr_visitor_test $* PASS'
 
+COMPILER_STMT_PASS_FILE_DEPS := \
+  $(PKG_DIR)/__python__/__go__/grumpy.a \
+  $(PKG_DIR)/__python__/__go__/os.a \
+  $(PKG_DIR)/__python__/__go__/runtime.a \
+  $(PKG_DIR)/__python__/__go__/time.a \
+  $(PKG_DIR)/__python__/__go__/unicode.a \
+  $(PKG_DIR)/__python__/sys.a \
+  $(PKG_DIR)/__python__/traceback.a
+
 # Does not depend on stdlibs since it makes minimal use of them.
-$(COMPILER_STMT_PASS_FILES): $(PY_DIR)/grumpy/compiler/stmt_test.%.pass: $(PY_DIR)/grumpy/compiler/stmt_test.py $(RUNNER_BIN) $(COMPILER) $(RUNTIME) $(PKG_DIR)/__python__/traceback.a
+$(COMPILER_STMT_PASS_FILES): $(PY_DIR)/grumpy/compiler/stmt_test.%.pass: $(PY_DIR)/grumpy/compiler/stmt_test.py $(RUNNER_BIN) $(COMPILER) $(RUNTIME) $(COMPILER_STMT_PASS_FILE_DEPS)
 	@$(PYTHON) $< --shard=$*
 	@touch $@
 	@echo 'compiler/stmt_test $* PASS'
 
+$(PKGC_BIN): tools/pkgc.go
+	@mkdir -p $(@D)
+	@go build -o $@ $<
+
 # ------------------------------------------------------------------------------
 # Grumpy runtime
 # ------------------------------------------------------------------------------
@@ -236,6 +251,22 @@ pylint: $(PYLINT_BIN) $(COMPILER_SRCS) $(PYTHONPARSER_SRCS) $(COMPILER_BIN) $(RU
 
 lint: golint pylint
 
+# ------------------------------------------------------------------------------
+# Native modules
+# ------------------------------------------------------------------------------
+
+$(PKG_DIR)/__python__/__go__/%.a: build/src/__python__/__go__/%/module.go $(RUNTIME)
+	@mkdir -p $(@D)
+	@go install __python__/__go__/$*
+
+build/src/__python__/__go__/%/module.go: $(PKGC_BIN) $(RUNTIME)
+	@mkdir -p $(@D)
+	@$(PKGC_BIN) $* > $@
+
+$(PKG_DIR)/__python__/__go__/grumpy.a: $(RUNTIME)
+
+.PRECIOUS: build/src/__python__/__go__/%/module.go $(PKG_DIR)/__python__/__go__/%.a
+
 # ------------------------------------------------------------------------------
 # Standard library
 # ------------------------------------------------------------------------------
@@ -294,6 +325,14 @@ $(ACCEPT_PASS_FILES): build/%_test.pass: %_test.py $(RUNTIME) $(STDLIB) $(RUNNER
 	@touch $@
 	@echo '$*_test PASS'
 
+NATIVE_TEST_DEPS := \
+  $(PKG_DIR)/__python__/__go__/encoding/csv.a \
+  $(PKG_DIR)/__python__/__go__/image.a \
+  $(PKG_DIR)/__python__/__go__/math.a \
+  $(PKG_DIR)/__python__/__go__/strings.a
+
+build/testing/native_test.pass: $(NATIVE_TEST_DEPS)
+
 $(ACCEPT_PY_PASS_FILES): build/%_py.pass: %.py $(PY_DIR)/weetest.py
 	@mkdir -p $(@D)
 	@$(PYTHON) $<

diff --git a/compiler/imputil.py b/compiler/imputil.py
@@ -86,15 +86,17 @@ def visit_Import(self, node):
     imports = []
     for alias in node.names:
       if alias.name.startswith(_NATIVE_MODULE_PREFIX):
-        msg = 'for native imports use "from \'__go__/xyz\' import ..." syntax'
-        raise util.ImportError(node, msg)
-      imp = self._resolve_import(node, alias.name)
-      if alias.asname:
-        imp.add_binding(Import.MODULE, alias.asname, imp.name.count('.'))
+        imp = Import(alias.name, is_native=True)
+        asname = alias.asname if alias.asname else alias.name.split('/')[-1]
+        imp.add_binding(Import.MODULE, asname, 0)
       else:
-        parts = alias.name.split('.')
-        imp.add_binding(Import.MODULE, parts[0],
-                        imp.name.count('.') - len(parts) + 1)
+        imp = self._resolve_import(node, alias.name)
+        if alias.asname:
+          imp.add_binding(Import.MODULE, alias.asname, imp.name.count('.'))
+        else:
+          parts = alias.name.split('.')
+          imp.add_binding(Import.MODULE, parts[0],
+                          imp.name.count('.') - len(parts) + 1)
       imports.append(imp)
     return imports
 
@@ -106,7 +108,7 @@ def visit_ImportFrom(self, node):
       return []
 
     if not node.level and node.module.startswith(_NATIVE_MODULE_PREFIX):
-      imp = Import(node.module[len(_NATIVE_MODULE_PREFIX):], is_native=True)
+      imp = Import(node.module, is_native=True)
       for alias in node.names:
         asname = alias.asname or alias.name
         imp.add_binding(Import.MEMBER, asname, alias.name)
@@ -211,6 +213,7 @@ def calc(modname, script):
     deps.add(modname)
     for imp in collect_imports(modname, script, gopath):
       if imp.is_native:
+        deps.add(imp.name)
         continue
       parts = imp.name.split('.')
       calc(imp.name, imp.script)

diff --git a/compiler/imputil_test.py b/compiler/imputil_test.py
@@ -140,10 +140,6 @@ def testImportAs(self):
     imp.add_binding(imputil.Import.MODULE, 'bar', 0)
     self._check_imports('import foo as bar', [imp])
 
-  def testImportNativeRaises(self):
-    self.assertRaises(util.ImportError, self.importer.visit,
-                      pythonparser.parse('import __go__.fmt').body[0])
-
   def testImportFrom(self):
     imp = copy.deepcopy(self.baz2_import)
     imp.add_binding(imputil.Import.MODULE, 'baz', 1)
@@ -186,18 +182,18 @@ def testImportFromFuture(self):
     self._check_imports('from __future__ import print_function', [])
 
   def testImportFromNative(self):
-    imp = imputil.Import('fmt', is_native=True)
+    imp = imputil.Import('__go__/fmt', is_native=True)
     imp.add_binding(imputil.Import.MEMBER, 'Printf', 'Printf')
     self._check_imports('from "__go__/fmt" import Printf', [imp])
 
   def testImportFromNativeMultiple(self):
-    imp = imputil.Import('fmt', is_native=True)
+    imp = imputil.Import('__go__/fmt', is_native=True)
     imp.add_binding(imputil.Import.MEMBER, 'Printf', 'Printf')
     imp.add_binding(imputil.Import.MEMBER, 'Println', 'Println')
     self._check_imports('from "__go__/fmt" import Printf, Println', [imp])
 
   def testImportFromNativeAs(self):
-    imp = imputil.Import('fmt', is_native=True)
+    imp = imputil.Import('__go__/fmt', is_native=True)
     imp.add_binding(imputil.Import.MEMBER, 'foo', 'Printf')
     self._check_imports('from "__go__/fmt" import Printf as foo', [imp])
 

diff --git a/compiler/stmt.py b/compiler/stmt.py
@@ -284,27 +284,7 @@ def visit_ImportFrom(self, node):
       raise util.LateFutureError(node)
 
     for imp in self.block.root.importer.visit(node):
-      if imp.is_native:
-        values = [b.value for b in imp.bindings]
-        with self._import_native(imp.name, values) as mod:
-          for binding in imp.bindings:
-            # Strip the 'type_' prefix when populating the module. This means
-            # that, e.g. 'from "__go__/foo" import type_Bar' will populate foo
-            # with a member called Bar, not type_Bar (although the symbol in
-            # the importing module will still be type_Bar unless aliased). This
-            # bends the semantics of import but makes native module contents
-            # more sensible.
-            name = binding.value
-            if name.startswith(_NATIVE_TYPE_PREFIX):
-              name = name[len(_NATIVE_TYPE_PREFIX):]
-            with self.block.alloc_temp() as member:
-              self.writer.write_checked_call2(
-                  member, 'πg.GetAttr(πF, {}, {}, nil)',
-                  mod.expr, self.block.root.intern(name))
-              self.block.bind_var(
-                  self.writer, binding.alias, member.expr)
-      else:
-        self._import_and_bind(imp)
+      self._import_and_bind(imp)
 
   def visit_Module(self, node):
     self._visit_each(node.body)
@@ -659,36 +639,6 @@ def _import_and_bind(self, imp):
                 mod.expr, self.block.root.intern(binding.value))
             self.block.bind_var(self.writer, binding.alias, member.expr)
 
-  def _import_native(self, name, values):
-    reflect_package = self.block.root.add_native_import('reflect')
-    package = self.block.root.add_native_import(name)
-    mod = self.block.alloc_temp()
-    with self.block.alloc_temp('map[string]*πg.Object') as members:
-      self.writer.write_tmpl('$members = map[string]*πg.Object{}',
-                             members=members.name)
-      for v in values:
-        module_attr = v
-        with self.block.alloc_temp() as wrapped:
-          if v.startswith(_NATIVE_TYPE_PREFIX):
-            module_attr = v[len(_NATIVE_TYPE_PREFIX):]
-            with self.block.alloc_temp(
-                '{}.{}'.format(package.alias, module_attr)) as type_:
-              self.writer.write_checked_call2(
-                  wrapped, 'πg.WrapNative(πF, {}.ValueOf({}))',
-                  reflect_package.alias, type_.expr)
-              self.writer.write('{} = {}.Type().ToObject()'.format(
-                  wrapped.name, wrapped.expr))
-          else:
-            self.writer.write_checked_call2(
-                wrapped, 'πg.WrapNative(πF, {}.ValueOf({}.{}))',
-                reflect_package.alias, package.alias, v)
-          self.writer.write('{}[{}] = {}'.format(
-              members.name, util.go_str(module_attr), wrapped.expr))
-      mod_name = imputil._NATIVE_MODULE_PREFIX + '.' + name  # pylint: disable=protected-access
-      self.writer.write_checked_call2(mod, 'πg.ImportNativeModule(πF, {}, {})',
-                                      util.go_str(mod_name), members.expr)
-    return mod
-
   def _tie_target(self, target, value):
     if isinstance(target, ast.Name):
       self._assign_target(target, value)

diff --git a/compiler/stmt_test.py b/compiler/stmt_test.py
@@ -328,15 +328,9 @@ def testImportGrumpy(self):
         from "__go__/grumpy" import Assert
         Assert(__frame__(), True, 'bad')""")))
 
-  def testImportNativeModuleRaises(self):
-    regexp = (r'for native imports use '
-              r'"from \'__go__/xyz\' import \.\.\." syntax')
-    self.assertRaisesRegexp(util.ImportError, regexp, _ParseAndVisit,
-                            'import "__go__/foo"')
-
   def testImportNativeType(self):
     self.assertEqual((0, "<type 'Duration'>\n"), _GrumpRun(textwrap.dedent("""\
-        from "__go__/time" import type_Duration as Duration
+        from "__go__/time" import Duration
         print Duration""")))
 
   def testImportWildcardMemberRaises(self):

diff --git a/lib/os/__init__.py b/lib/os/__init__.py
@@ -17,15 +17,15 @@
 # pylint: disable=g-multiple-import
 from '__go__/io/ioutil' import ReadDir
 from '__go__/os' import (Chdir, Chmod, Environ, Getpid as getpid, Getwd, Pipe,
-    type_ProcAttr as ProcAttr, Remove, StartProcess, Stat, Stdout, Stdin,
+    ProcAttr, Remove, StartProcess, Stat, Stdout, Stdin,
     Stderr, Mkdir)
 from '__go__/path/filepath' import Separator
 from '__go__/grumpy' import (NewFileFromFD, StartThread, ToNative)
 from '__go__/reflect' import MakeSlice
 from '__go__/runtime' import GOOS
 from '__go__/syscall' import (Close, SYS_FCNTL, Syscall, F_GETFD, Wait4,
-    type_WaitStatus as WaitStatus, WNOHANG)
-from '__go__/sync' import type_WaitGroup as WaitGroup
+    WaitStatus, WNOHANG)
+from '__go__/sync' import WaitGroup
 from '__go__/time' import Second
 import _syscall
 from os import path

diff --git a/lib/select_.py b/lib/select_.py
@@ -15,8 +15,8 @@
 from '__go__/syscall' import (
     FD_SETSIZE as _FD_SETSIZE,
     Select as _Select,
-    type_FdSet as _FdSet,
-    type_Timeval as _Timeval
+    FdSet as _FdSet,
+    Timeval as _Timeval
 )
 import _syscall
 import math