all: introduce context-based interpreter

This CL introduces py.Context and implements a context-based interpreter.
This enables multi-context execution.

Co-authored-by: Drew O'Meara <[email protected]>

Author: drew-512

builtin/builtin.go

@@ -162,7 +162,16 @@ func init() {
 		"Warning":                   py.Warning,
 		"ZeroDivisionError":         py.ZeroDivisionError,
 	}
-	py.NewModule("builtins", builtin_doc, methods, globals)
+
+	py.RegisterModule(&py.ModuleImpl{
+		Info: py.ModuleInfo{
+			Name:  "builtins",
+			Doc:   builtin_doc,
+			Flags: py.ShareModule,
+		},
+		Methods: methods,
+		Globals: globals,
+	})
 }
 
 const print_doc = `print(value, ..., sep=' ', end='\\n', file=sys.stdout, flush=False)
@@ -178,18 +187,22 @@ func builtin_print(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Obje
 	var (
 		sepObj py.Object = py.String(" ")
 		endObj py.Object = py.String("\n")
-		file   py.Object = py.MustGetModule("sys").Globals["stdout"]
 		flush  py.Object
 	)
+	sysModule, err := self.(*py.Module).Context.GetModule("sys")
+	if err != nil {
+		return nil, err
+	}
+	stdout := sysModule.Globals["stdout"]
 	kwlist := []string{"sep", "end", "file", "flush"}
-	err := py.ParseTupleAndKeywords(nil, kwargs, "|ssOO:print", kwlist, &sepObj, &endObj, &file, &flush)
+	err = py.ParseTupleAndKeywords(nil, kwargs, "|ssOO:print", kwlist, &sepObj, &endObj, &stdout, &flush)
 	if err != nil {
 		return nil, err
 	}
 	sep := sepObj.(py.String)
 	end := endObj.(py.String)
 
-	write, err := py.GetAttrString(file, "write")
+	write, err := py.GetAttrString(stdout, "write")
 	if err != nil {
 		return nil, err
 	}
@@ -219,7 +232,7 @@ func builtin_print(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Obje
 	}
 
 	if shouldFlush, _ := py.MakeBool(flush); shouldFlush == py.True {
-		fflush, err := py.GetAttrString(file, "flush")
+		fflush, err := py.GetAttrString(stdout, "flush")
 		if err == nil {
 			return py.Call(fflush, nil, nil)
 		}
@@ -449,7 +462,7 @@ func builtin___build_class__(self py.Object, args py.Tuple, kwargs py.StringDict
 	}
 	// fmt.Printf("Calling %v with %v and %v\n", fn.Name, fn.Globals, ns)
 	// fmt.Printf("Code = %#v\n", fn.Code)
-	cell, err = py.VmRun(fn.Globals, ns, fn.Code, fn.Closure)
+	cell, err = fn.Context.RunCode(fn.Code, fn.Globals, ns, fn.Closure)
 	if err != nil {
 		return nil, err
 	}
@@ -750,7 +763,7 @@ func builtin_compile(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Ob
 	}
 
 	// if dont_inherit.(py.Int) != 0 {
-		// PyEval_MergeCompilerFlags(&cf)
+	// PyEval_MergeCompilerFlags(&cf)
 	// }
 
 	// switch string(startstr.(py.String)) {
@@ -782,7 +795,7 @@ func builtin_compile(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Ob
 		return nil, err
 	}
 	// result = py.CompileStringExFlags(str, filename, start[mode], &cf, optimize)
-	result, err = compile.Compile(str, string(filename.(py.String)), string(startstr.(py.String)), int(supplied_flags.(py.Int)), dont_inherit.(py.Int) != 0)
+	result, err = compile.Compile(str, string(filename.(py.String)), py.CompileMode(startstr.(py.String)), int(supplied_flags.(py.Int)), dont_inherit.(py.Int) != 0)
 	if err != nil {
 		return nil, err
 	}

compile/compile.go

@@ -89,33 +89,36 @@ func init() {
 	py.Compile = Compile
 }
 
-// Compile(source, filename, mode, flags, dont_inherit) -> code object
+// Compile(src, srcDesc, compileMode, flags, dont_inherit) -> code object
 //
 // Compile the source string (a Python module, statement or expression)
-// into a code object that can be executed by exec() or eval().
-// The filename will be used for run-time error messages.
-// The mode must be 'exec' to compile a module, 'single' to compile a
-// single (interactive) statement, or 'eval' to compile an expression.
+// into a code object that can be executed.
+//
+// srcDesc is used for run-time error messages and is typically a file system pathname,
+//
+// See py.CompileMode for compile mode options.
+//
 // The flags argument, if present, controls which future statements influence
 // the compilation of the code.
+//
 // The dont_inherit argument, if non-zero, stops the compilation inheriting
 // the effects of any future statements in effect in the code calling
 // compile; if absent or zero these statements do influence the compilation,
 // in addition to any features explicitly specified.
-func Compile(str, filename, mode string, futureFlags int, dont_inherit bool) (py.Object, error) {
+func Compile(src, srcDesc string, mode py.CompileMode, futureFlags int, dont_inherit bool) (*py.Code, error) {
 	// Parse Ast
-	Ast, err := parser.ParseString(str, mode)
+	Ast, err := parser.ParseString(src, mode)
 	if err != nil {
 		return nil, err
 	}
 	// Make symbol table
-	SymTable, err := symtable.NewSymTable(Ast, filename)
+	SymTable, err := symtable.NewSymTable(Ast, srcDesc)
 	if err != nil {
 		return nil, err
 	}
 	c := newCompiler(nil, compilerScopeModule)
-	c.Filename = filename
-	err = c.compileAst(Ast, filename, futureFlags, dont_inherit, SymTable)
+	c.Filename = srcDesc
+	err = c.compileAst(Ast, srcDesc, futureFlags, dont_inherit, SymTable)
 	if err != nil {
 		return nil, err
 	}

compile/compile_data_test.go

@@ -12,7 +12,7 @@ import (
 
 var compileTestData = []struct {
 	in            string
-	mode          string // exec, eval or single
+	mode          py.CompileMode
 	out           *py.Code
 	exceptionType *py.Type
 	errString     string

compile/compile_test.go

@@ -163,7 +163,7 @@ func EqCode(t *testing.T, name string, a, b *py.Code) {
 func TestCompile(t *testing.T) {
 	for _, test := range compileTestData {
 		// log.Printf(">>> %s", test.in)
-		codeObj, err := Compile(test.in, "<string>", test.mode, 0, true)
+		code, err := Compile(test.in, "<string>", test.mode, 0, true)
 		if err != nil {
 			if test.exceptionType == nil {
 				t.Errorf("%s: Got exception %v when not expecting one", test.in, err)
@@ -196,17 +196,12 @@ func TestCompile(t *testing.T) {
 			}
 		} else {
 			if test.out == nil {
-				if codeObj != nil {
-					t.Errorf("%s: Expecting nil *py.Code but got %T", test.in, codeObj)
+				if code != nil {
+					t.Errorf("%s: Expecting nil *py.Code but got %T", test.in, code)
 				}
 			} else {
-				code, ok := codeObj.(*py.Code)
-				if !ok {
-					t.Errorf("%s: Expecting *py.Code but got %T", test.in, codeObj)
-				} else {
-					//t.Logf("Testing %q", test.in)
-					EqCode(t, test.in, test.out, code)
-				}
+				//t.Logf("Testing %q", test.in)
+				EqCode(t, test.in, test.out, code)
 			}
 		}
 	}

go.mod

@@ -3,6 +3,6 @@ module github.com/go-python/gpython
 go 1.16
 
 require (
-	github.com/gopherjs/gopherwasm v1.0.0
-	github.com/peterh/liner v1.1.0
+	github.com/gopherjs/gopherwasm v1.1.0
+	github.com/peterh/liner v1.2.2
 )

go.sum

@@ -2,7 +2,13 @@ github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c h1:16eHWuMGvCjSf
 github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gopherjs/gopherwasm v1.0.0 h1:32nge/RlujS1Im4HNCJPp0NbBOAeBXFuT1KonUuLl+Y=
 github.com/gopherjs/gopherwasm v1.0.0/go.mod h1:SkZ8z7CWBz5VXbhJel8TxCmAcsQqzgWGR/8nMhyhZSI=
+github.com/gopherjs/gopherwasm v1.1.0 h1:fA2uLoctU5+T3OhOn2vYP0DVT6pxc7xhTlBB1paATqQ=
+github.com/gopherjs/gopherwasm v1.1.0/go.mod h1:SkZ8z7CWBz5VXbhJel8TxCmAcsQqzgWGR/8nMhyhZSI=
 github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4=
 github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/peterh/liner v1.1.0 h1:f+aAedNJA6uk7+6rXsYBnhdo4Xux7ESLe+kcuVUF5os=
 github.com/peterh/liner v1.1.0/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0=
+github.com/peterh/liner v1.2.2 h1:aJ4AOodmL+JxOZZEL2u9iJf8omNRpqHc/EbrK+3mAXw=
+github.com/peterh/liner v1.2.2/go.mod h1:xFwJyiKIXJZUKItq5dGHZSTBRAuG/CpeNpWLyiNRNwI=
+golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1 h1:kwrAHlwJ0DUBZwQ238v+Uod/3eZ8B2K5rYsUHBQvzmI=
+golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

importlib/importlib.go

@@ -7,15 +7,18 @@
 package py
 
 import (
-	"log"
-
-	"github.com/go-python/gpython/marshal"
+	"github.com/go-python/gpython/py"
 )
 
 // Load the frozen module
 func init() {
-	_, err := marshal.LoadFrozenModule("importlib", data)
-	log.Fatalf("Failed to load importlib: %v", err)
+
+	py.RegisterModule(&py.ModuleImpl{
+		Info: py.ModuleInfo{
+			Name: "importlib",
+		},
+		CodeBuf: data,
+	})
 }
 
 // Auto-generated by Modules/_freeze_importlib.c

main.go

@@ -12,22 +12,13 @@ import (
 	"runtime"
 	"runtime/pprof"
 
-	_ "github.com/go-python/gpython/builtin"
+	"github.com/go-python/gpython/repl"
 	"github.com/go-python/gpython/repl/cli"
 
-	//_ "github.com/go-python/gpython/importlib"
-	"io/ioutil"
 	"log"
 	"os"
-	"strings"
 
-	"github.com/go-python/gpython/compile"
-	"github.com/go-python/gpython/marshal"
-	_ "github.com/go-python/gpython/math"
 	"github.com/go-python/gpython/py"
-	pysys "github.com/go-python/gpython/sys"
-	_ "github.com/go-python/gpython/time"
-	"github.com/go-python/gpython/vm"
 )
 
 // Globals
@@ -48,33 +39,14 @@ Full options:
 	flag.PrintDefaults()
 }
 
-// Exit with the message
-func fatal(message string, args ...interface{}) {
-	if !strings.HasSuffix(message, "\n") {
-		message += "\n"
-	}
-	syntaxError()
-	fmt.Fprintf(os.Stderr, message, args...)
-	os.Exit(1)
-}
-
 func main() {
 	flag.Usage = syntaxError
 	flag.Parse()
 	args := flag.Args()
-	py.MustGetModule("sys").Globals["argv"] = pysys.MakeArgv(args)
-	if len(args) == 0 {
-
-		fmt.Printf("Python 3.4.0 (%s, %s)\n", commit, date)
-		fmt.Printf("[Gpython %s]\n", version)
-		fmt.Printf("- os/arch: %s/%s\n", runtime.GOOS, runtime.GOARCH)
-		fmt.Printf("- go version: %s\n", runtime.Version())
 
-		cli.RunREPL()
-		return
-	}
-	prog := args[0]
-	// fmt.Printf("Running %q\n", prog)
+	opts := py.DefaultContextOpts()
+	opts.SysArgs = flag.Args()
+	ctx := py.NewContext(opts)
 
 	if *cpuprofile != "" {
 		f, err := os.Create(*cpuprofile)
@@ -88,41 +60,23 @@ func main() {
 		defer pprof.StopCPUProfile()
 	}
 
-	// FIXME should be using ImportModuleLevelObject() here
-	f, err := os.Open(prog)
-	if err != nil {
-		log.Fatalf("Failed to open %q: %v", prog, err)
-	}
-	var obj py.Object
-	if strings.HasSuffix(prog, ".pyc") {
-		obj, err = marshal.ReadPyc(f)
-		if err != nil {
-			log.Fatalf("Failed to marshal %q: %v", prog, err)
-		}
-	} else if strings.HasSuffix(prog, ".py") {
-		str, err := ioutil.ReadAll(f)
-		if err != nil {
-			log.Fatalf("Failed to read %q: %v", prog, err)
-		}
-		obj, err = compile.Compile(string(str), prog, "exec", 0, true)
+	// IF no args, enter REPL mode
+	if len(args) == 0 {
+
+		fmt.Printf("Python 3.4.0 (%s, %s)\n", commit, date)
+		fmt.Printf("[Gpython %s]\n", version)
+		fmt.Printf("- os/arch: %s/%s\n", runtime.GOOS, runtime.GOARCH)
+		fmt.Printf("- go version: %s\n", runtime.Version())
+
+		replCtx := repl.New(ctx)
+		cli.RunREPL(replCtx)
+
+	} else {
+		_, err := py.RunFile(ctx, args[0], py.CompileOpts{}, nil)
 		if err != nil {
-			log.Fatalf("Can't compile %q: %v", prog, err)
+			py.TracebackDump(err)
+			log.Fatal(err)
 		}
-	} else {
-		log.Fatalf("Can't execute %q", prog)
-	}
-	if err = f.Close(); err != nil {
-		log.Fatalf("Failed to close %q: %v", prog, err)
-	}
-	code := obj.(*py.Code)
-	module := py.NewModule("__main__", "", nil, nil)
-	module.Globals["__file__"] = py.String(prog)
-	res, err := vm.Run(module.Globals, module.Globals, code, nil)
-	if err != nil {
-		py.TracebackDump(err)
-		log.Fatal(err)
 	}
-	// fmt.Printf("Return = %v\n", res)
-	_ = res
 
 }

marshal/marshal.go

@@ -6,7 +6,6 @@
 package marshal
 
 import (
-	"bytes"
 	"encoding/binary"
 	"errors"
 	"fmt"
@@ -15,7 +14,6 @@ import (
 	"strconv"
 
 	"github.com/go-python/gpython/py"
-	"github.com/go-python/gpython/vm"
 )
 
 const (
@@ -454,23 +452,6 @@ func ReadPyc(r io.Reader) (obj py.Object, err error) {
 	return ReadObject(r)
 }
 
-// Unmarshals a frozen module
-func LoadFrozenModule(name string, data []byte) (*py.Module, error) {
-	r := bytes.NewBuffer(data)
-	obj, err := ReadObject(r)
-	if err != nil {
-		return nil, err
-	}
-	code := obj.(*py.Code)
-	module := py.NewModule(name, "", nil, nil)
-	_, err = vm.Run(module.Globals, module.Globals, code, nil)
-	if err != nil {
-		py.TracebackDump(err)
-		return nil, err
-	}
-	return module, nil
-}
-
 const dump_doc = `dump(value, file[, version])
 
 Write the value on the open file. The value must be a supported type.
@@ -634,5 +615,14 @@ func init() {
 	globals := py.StringDict{
 		"version": py.Int(MARSHAL_VERSION),
 	}
-	py.NewModule("marshal", module_doc, methods, globals)
+
+	py.RegisterModule(&py.ModuleImpl{
+		Info: py.ModuleInfo{
+			Name:  "marshal",
+			Doc:   module_doc,
+			Flags: py.ShareModule,
+		},
+		Globals: globals,
+		Methods: methods,
+	})
 }